From 43fc4ba891174691b5c30b437587761e4b3f3bfb Mon Sep 17 00:00:00 2001 From: Yang Lei Date: Tue, 14 Jul 2015 12:10:43 -0400 Subject: [PATCH 01/11] swift --- exhibitor-core/.gitignore | 1 + exhibitor-core/build.gradle | 7 + .../backup/swift/SwiftBackupProvider.java | 359 ++++++++++++++++++ .../exhibitor/core/backup/swift/Throttle.java | 107 ++++++ .../config/swift/SwiftConfigArguments.java | 46 +++ .../SwiftConfigAutoManageLockArguments.java | 42 ++ .../config/swift/SwiftConfigProvider.java | 136 +++++++ .../core/config/swift/SwiftPseudoLock.java | 73 ++++ exhibitor-standalone/.gitignore | 1 + exhibitor-standalone/build.gradle | 13 + .../exhibitor/standalone/ExhibitorCLI.java | 22 ++ .../standalone/ExhibitorCreator.java | 60 ++- gradle.properties | 1 + 13 files changed, 866 insertions(+), 2 deletions(-) create mode 100644 exhibitor-core/.gitignore create mode 100644 exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/SwiftBackupProvider.java create mode 100644 exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/Throttle.java create mode 100644 exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftConfigArguments.java create mode 100644 exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftConfigAutoManageLockArguments.java create mode 100644 exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftConfigProvider.java create mode 100644 exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftPseudoLock.java create mode 100644 exhibitor-standalone/.gitignore diff --git a/exhibitor-core/.gitignore b/exhibitor-core/.gitignore new file mode 100644 index 00000000..ae3c1726 --- /dev/null +++ b/exhibitor-core/.gitignore @@ -0,0 +1 @@ +/bin/ diff --git a/exhibitor-core/build.gradle b/exhibitor-core/build.gradle index a1470e6b..c03e2421 100644 --- a/exhibitor-core/build.gradle +++ b/exhibitor-core/build.gradle @@ -1,4 +1,5 @@ apply plugin: 'nebula.provided-base' +apply plugin: 'eclipse' dependencies { compile('org.apache.zookeeper:zookeeper:3.4.5') { @@ -27,6 +28,12 @@ dependencies { compile "com.sun.jersey:jersey-server:${jerseyVersion}" compile "com.sun.jersey:jersey-json:${jerseyVersion}" + // Swift and Jcloud + compile "org.apache.jclouds.driver:jclouds-slf4j:${jcloudsVersion}" + compile "org.apache.jclouds.driver:jclouds-sshj:${jcloudsVersion}" + compile "org.apache.jclouds.api:openstack-swift:${jcloudsVersion}" + compile "ch.qos.logback:logback-classic:1.0.13" + testCompile "org.apache.curator:curator-test:${curatorVersion}" testCompile "org.mockito:mockito-core:${mockitoVersion}" } diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/SwiftBackupProvider.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/SwiftBackupProvider.java new file mode 100644 index 00000000..49acce91 --- /dev/null +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/SwiftBackupProvider.java @@ -0,0 +1,359 @@ +/* + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.exhibitor.core.backup.swift; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; +import com.google.common.io.Files; +import com.netflix.exhibitor.core.Exhibitor; +import com.netflix.exhibitor.core.activity.ActivityLog; +import com.netflix.exhibitor.core.backup.BackupConfigSpec; +import com.netflix.exhibitor.core.backup.BackupMetaData; +import com.netflix.exhibitor.core.backup.BackupProvider; +import com.netflix.exhibitor.core.backup.BackupStream; + +import org.apache.curator.RetryLoop; +import org.apache.curator.RetryPolicy; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.curator.utils.CloseableUtils; +import org.jclouds.io.Payload; +import org.jclouds.io.payloads.ByteArrayPayload; +import org.jclouds.openstack.swift.v1.SwiftApi; +import org.jclouds.openstack.swift.v1.domain.ObjectList; +import org.jclouds.openstack.swift.v1.domain.SwiftObject; +import org.jclouds.openstack.swift.v1.features.ContainerApi; +import org.jclouds.openstack.swift.v1.features.ObjectApi; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static com.netflix.exhibitor.core.config.DefaultProperties.asInt; + +public class SwiftBackupProvider implements BackupProvider +{ + private final SwiftApi swiftApi; + + private static final BackupConfigSpec CONFIG_THROTTLE = new BackupConfigSpec("throttle", "Throttle (bytes/ms)", "Data throttling. Maximum bytes per millisecond.", Integer.toString(1024 * 1024), BackupConfigSpec.Type.INTEGER); + private static final BackupConfigSpec CONFIG_CONTAINER= new BackupConfigSpec("container-name", "Swift Container Name", "The Swfit containerto use", "", BackupConfigSpec.Type.STRING); + private static final BackupConfigSpec CONFIG_KEY_PREFIX = new BackupConfigSpec("key-prefix", "Swfit Key Prefix", "The prefix for Swift backup keys", "exhibitor-backup", BackupConfigSpec.Type.STRING); + private static final BackupConfigSpec CONFIG_MAX_RETRIES = new BackupConfigSpec("max-retries", "Max Retries", "Maximum retries when uploading/downloading S3 data", "3", BackupConfigSpec.Type.INTEGER); + private static final BackupConfigSpec CONFIG_RETRY_SLEEP_MS = new BackupConfigSpec("retry-sleep-ms", "Retry Sleep (ms)", "Sleep time in milliseconds when retrying", "1000", BackupConfigSpec.Type.INTEGER); + + private static final List CONFIGS = Arrays.asList(CONFIG_THROTTLE, CONFIG_CONTAINER, CONFIG_KEY_PREFIX, CONFIG_MAX_RETRIES, CONFIG_RETRY_SLEEP_MS); + + private static final int MIN_SWIFT_PART_SIZE = 5 * (1024 * 1024); + + @VisibleForTesting + static final String SEPARATOR = "/"; + private static final String SEPARATOR_REPLACEMENT = "_"; + + public SwiftBackupProvider(SwiftApi swiftApi) throws Exception + { + this.swiftApi = swiftApi; + } + + + @Override + public List getConfigs() + { + return CONFIGS; + } + + @Override + public boolean isValidConfig(Exhibitor exhibitor, Map configValues) + { + String containerName = (configValues != null) ? configValues.get(CONFIG_CONTAINER.getKey()) : null; + return (containerName != null) && (containerName.trim().length() > 0); + } + + private ObjectApi getObjectApi(String containerName) + { + String region = swiftApi.getConfiguredRegions().iterator().next(); + + ContainerApi containerApi = swiftApi.getContainerApi(region); + if (containerApi.get(containerName) ==null) + containerApi.create(containerName); + + return swiftApi.getObjectApi(region, containerName); + + } + + @Override + public UploadResult uploadBackup(Exhibitor exhibitor, BackupMetaData backup, File source, final Map configValues) throws Exception + { + List availableBackups = getAvailableBackups(exhibitor, configValues); + if ( availableBackups.contains(backup) ) + { + return UploadResult.DUPLICATE; + } + + RetryPolicy retryPolicy = makeRetryPolicy(configValues); + Throttle throttle = makeThrottle(configValues); + + String key = toKey(backup, configValues); + ObjectApi objectApi = getObjectApi(configValues.get(CONFIG_CONTAINER.getKey())); + + if ( source.length() < MIN_SWIFT_PART_SIZE ) + { + byte[] bytes = Files.toByteArray(source); + Payload payload = new ByteArrayPayload(bytes); + objectApi.put(key, payload); + } + else + { + multiPartUpload(source, configValues, retryPolicy, throttle, key); + } + + UploadResult result = UploadResult.SUCCEEDED; + for ( BackupMetaData existing : availableBackups ) + { + if ( existing.getName().equals(backup.getName()) ) + { + deleteBackup(exhibitor, existing, configValues); + result = UploadResult.REPLACED_OLD_VERSION; + } + } + return result; + } + + private void multiPartUpload(File source, Map configValues, RetryPolicy retryPolicy, Throttle throttle, String key) throws Exception + { + // TODO how to support multipart on jcloud swift? + ObjectApi objectApi = getObjectApi(configValues.get(CONFIG_CONTAINER.getKey())); + byte[] bytes = Files.toByteArray(source); + Payload payload = new ByteArrayPayload(bytes); + objectApi.put(key, payload); + + } + + @Override + public BackupStream getBackupStream(Exhibitor exhibitor, BackupMetaData backup, Map configValues) throws Exception + { + long startMs = System.currentTimeMillis(); + RetryPolicy retryPolicy = makeRetryPolicy(configValues); + int retryCount = 0; + ObjectApi objectApi = getObjectApi(configValues.get(CONFIG_CONTAINER.getKey())); + SwiftObject object = null; + while ( object == null ) + { + try + { + object = objectApi.get(toKey(backup, configValues)); + } + catch ( Exception e) + { + /*TODO any fastpath on failure? + if (some condition) + { + exhibitor.getLog().add(ActivityLog.Type.ERROR, "Swift client error: " + ActivityLog.getExceptionMessage(e)); + return null; + }*/ + if ( !retryPolicy.allowRetry(retryCount++, System.currentTimeMillis() - startMs, RetryLoop.getDefaultRetrySleeper()) ) + { + exhibitor.getLog().add(ActivityLog.Type.ERROR, "Retries exhausted: " + ActivityLog.getExceptionMessage(e)); + return null; + } + } + } + + final Throttle throttle = makeThrottle(configValues); + final InputStream in = object.getPayload().openStream(); + final InputStream wrappedstream = new InputStream() + { + @Override + public void close() throws IOException + { + in.close(); + } + + @Override + public int read() throws IOException + { + throttle.throttle(1); + return in.read(); + } + + @Override + public int read(byte[] b) throws IOException + { + int bytesRead = in.read(b); + if ( bytesRead > 0 ) + { + throttle.throttle(bytesRead); + } + return bytesRead; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException + { + int bytesRead = in.read(b, off, len); + if ( bytesRead > 0 ) + { + throttle.throttle(bytesRead); + } + return bytesRead; + } + }; + + return new BackupStream() + { + @Override + public InputStream getStream() + { + return wrappedstream; + } + + @Override + public void close() throws IOException + { + in.close(); + } + }; + } + + @Override + public void downloadBackup(Exhibitor exhibitor, BackupMetaData backup, OutputStream destination, Map configValues) throws Exception + { + byte[] buffer = new byte[MIN_SWIFT_PART_SIZE]; + + long startMs = System.currentTimeMillis(); + RetryPolicy retryPolicy = makeRetryPolicy(configValues); + int retryCount = 0; + boolean done = false; + + while ( !done ) + { + Throttle throttle = makeThrottle(configValues); + InputStream in = null; + try + { + ObjectApi objectApi = getObjectApi(configValues.get(CONFIG_CONTAINER.getKey())); + SwiftObject object = objectApi.get(toKey(backup, configValues)); + in = object.getPayload().openStream(); + + for(;;) + { + int bytesRead = in.read(buffer); + if ( bytesRead < 0 ) + { + break; + } + + throttle.throttle(bytesRead); + destination.write(buffer, 0, bytesRead); + } + + done = true; + } + catch ( Exception e ) + { + if ( !retryPolicy.allowRetry(retryCount++, System.currentTimeMillis() - startMs, RetryLoop.getDefaultRetrySleeper()) ) + { + done = true; + } + } + finally + { + CloseableUtils.closeQuietly(in); + } + } + } + + + @Override + public List getAvailableBackups(Exhibitor exhibitor, Map configValues) throws Exception + { + String keyPrefix = getKeyPrefix(configValues); + List completeList = Lists.newArrayList(); + + ObjectApi objectApi = getObjectApi(configValues.get(CONFIG_CONTAINER.getKey())); + + ObjectList list = objectApi.list(); + for (int i=0; i< list.size(); i++) + { + SwiftObject obj = list.get(i); + String name = obj.getName(); + if (name.startsWith(keyPrefix)) + completeList.add(new BackupMetaData(name, obj.getLastModified().getTime())); + + } + return completeList; + } + + @Override + public void deleteBackup(Exhibitor exhibitor, BackupMetaData backup, Map configValues) throws Exception + { + ObjectApi objectApi = getObjectApi(configValues.get(CONFIG_CONTAINER.getKey())); + objectApi.delete(toKey(backup, configValues)); + } + + private Throttle makeThrottle(final Map configValues) + { + return new Throttle(this.getClass().getCanonicalName(), new Throttle.ThroughputFunction() + { + public int targetThroughput() + { + return Math.max(asInt(configValues.get(CONFIG_THROTTLE.getKey())), Integer.MAX_VALUE); + } + }); + } + + private ExponentialBackoffRetry makeRetryPolicy(Map configValues) + { + return new ExponentialBackoffRetry(asInt(configValues.get(CONFIG_RETRY_SLEEP_MS.getKey())), asInt(configValues.get(CONFIG_MAX_RETRIES.getKey()))); + } + + + private String toKey(BackupMetaData backup, Map configValues) + { + String name = backup.getName().replace(SEPARATOR, SEPARATOR_REPLACEMENT); + String prefix = getKeyPrefix(configValues); + + return prefix + SEPARATOR + name + SEPARATOR + backup.getModifiedDate(); + } + + private String getKeyPrefix(Map configValues) + { + String prefix = configValues.get(CONFIG_KEY_PREFIX.getKey()); + if ( prefix != null ) + { + prefix = prefix.replace(SEPARATOR, SEPARATOR_REPLACEMENT); + } + + if ( (prefix == null) || (prefix.length() == 0)) + { + prefix = "exhibitor-backup"; + } + return prefix; + } + + private static BackupMetaData fromKey(String key) + { + String[] parts = key.split("\\" + SEPARATOR); + if ( parts.length != 3 ) + { + return null; + } + return new BackupMetaData(parts[1], Long.parseLong(parts[2])); + } +} diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/Throttle.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/Throttle.java new file mode 100644 index 00000000..a4197b2c --- /dev/null +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/Throttle.java @@ -0,0 +1,107 @@ +/* + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.exhibitor.core.backup.swift; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Throttle +{ + private static final Logger logger = LoggerFactory.getLogger(Throttle.class); + + private final String name; + private final ThroughputFunction fun; + + // the bytes that had been handled the last time we delayed to throttle, + // and the time in milliseconds when we last throttled + private long bytesAtLastDelay; + private long timeAtLastDelay = System.currentTimeMillis(); // JLZ - need to initialize + + // current target bytes of throughput per millisecond + private int targetBytesPerMS = -1; + + public Throttle(String name, ThroughputFunction fun) + { + this.name = name; + this.fun = fun; + } + + /** @param currentBytes Bytes of throughput since the beginning of the task. */ + public void throttle(long currentBytes) + { + throttleDelta(currentBytes - bytesAtLastDelay); + } + + /** @param bytesDelta Bytes of throughput since the last call to throttle*(). */ + public void throttleDelta(long bytesDelta) + { + int newTargetBytesPerMS = fun.targetThroughput(); + if ( newTargetBytesPerMS < 1 ) + // throttling disabled + { + return; + } + + // if the target changed, log + if ( newTargetBytesPerMS != targetBytesPerMS ) + { + logger.debug("{} target throughput now {} bytes/ms.", this, newTargetBytesPerMS); + } + targetBytesPerMS = newTargetBytesPerMS; + + // time passed since last delay + long msSinceLast = System.currentTimeMillis() - timeAtLastDelay; + // the excess bytes in this period + long excessBytes = bytesDelta - msSinceLast * targetBytesPerMS; + + // the time to delay to recap the deficit + long timeToDelay = excessBytes / Math.max(1, targetBytesPerMS); + if (timeToDelay > 0) + { + if ( logger.isTraceEnabled() ) + { + logger.trace(String.format("%s actual throughput was %d bytes in %d ms: throttling for %d ms", + this, bytesDelta, msSinceLast, timeToDelay)); + } + try + { + Thread.sleep(timeToDelay); + } + catch (InterruptedException e) + { + throw new AssertionError(e); + } + } + bytesAtLastDelay += bytesDelta; + timeAtLastDelay = System.currentTimeMillis(); + } + + @Override + public String toString() + { + return "Throttle(for=" + name + ")"; + } + + public interface ThroughputFunction + { + /** + * @return The instantaneous target throughput in bytes per millisecond. Targets less + * than or equal to zero will disable throttling. + */ + public int targetThroughput(); + } +} diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftConfigArguments.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftConfigArguments.java new file mode 100644 index 00000000..6b10f88c --- /dev/null +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftConfigArguments.java @@ -0,0 +1,46 @@ +/* + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.exhibitor.core.config.swift; + +public class SwiftConfigArguments +{ + private final String containerName; + private final String key; + private final SwiftConfigAutoManageLockArguments lockArguments; + + public SwiftConfigArguments(String containerName, String key, + SwiftConfigAutoManageLockArguments lockArguments) { + super(); + this.containerName = containerName; + this.key = key; + this.lockArguments = lockArguments; + } + + public String getContainerName() { + return containerName; + } + + public String getKey() { + return key; + } + + public SwiftConfigAutoManageLockArguments getLockArguments() { + return lockArguments; + } + + + } diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftConfigAutoManageLockArguments.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftConfigAutoManageLockArguments.java new file mode 100644 index 00000000..d4d9a22d --- /dev/null +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftConfigAutoManageLockArguments.java @@ -0,0 +1,42 @@ +/* + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.exhibitor.core.config.swift; + +import com.netflix.exhibitor.core.config.AutoManageLockArguments; +import java.util.concurrent.TimeUnit; + +public class SwiftConfigAutoManageLockArguments extends AutoManageLockArguments +{ + private final int settlingMs; + + public SwiftConfigAutoManageLockArguments(String prefix) + { + super(prefix); + settlingMs = (int)TimeUnit.SECONDS.toMillis(5); // TODO - get this right + } + + public SwiftConfigAutoManageLockArguments(String prefix, int timeoutMs, int pollingMs, int settlingMs) + { + super(prefix, timeoutMs, pollingMs); + this.settlingMs = settlingMs; + } + + public int getSettlingMs() + { + return settlingMs; + } +} diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftConfigProvider.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftConfigProvider.java new file mode 100644 index 00000000..2e0caf02 --- /dev/null +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftConfigProvider.java @@ -0,0 +1,136 @@ +/* + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.exhibitor.core.config.swift; + +import com.netflix.exhibitor.core.config.ConfigCollection; +import com.netflix.exhibitor.core.config.ConfigProvider; +import com.netflix.exhibitor.core.config.LoadedInstanceConfig; +import com.netflix.exhibitor.core.config.PropertyBasedInstanceConfig; +import com.netflix.exhibitor.core.config.PseudoLock; + +import org.jclouds.io.Payload; +import org.jclouds.io.payloads.ByteArrayPayload; +import org.jclouds.openstack.swift.v1.SwiftApi; +import org.jclouds.openstack.swift.v1.domain.SwiftObject; +import org.jclouds.openstack.swift.v1.features.ContainerApi; +import org.jclouds.openstack.swift.v1.features.ObjectApi; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +public class SwiftConfigProvider implements ConfigProvider +{ + private final SwiftApi swiftApi; + private final String hostname; + private final SwiftConfigArguments arguments; + private final ObjectApi objectApi; + private final Properties defaults; + + public SwiftConfigProvider(SwiftApi swiftApi, SwiftConfigArguments arguments,String hostname, Properties defaults) throws Exception + { + this.swiftApi = swiftApi; + this.hostname = hostname; + this.arguments = arguments; + this.objectApi = initialize(arguments.getContainerName()); + this.defaults = defaults; + } + + private ObjectApi initialize (String containerName) + { + String region = swiftApi.getConfiguredRegions().iterator().next(); + + ContainerApi containerApi = swiftApi.getContainerApi(region); + if (containerApi.get(containerName) ==null) + containerApi.create(containerName); + + return swiftApi.getObjectApi(region, containerName); + + } + + @Override + public void start() throws Exception + { + // NOP + } + + @Override + public void close() throws IOException + { + + swiftApi.close(); + } + + @Override + public PseudoLock newPseudoLock() throws Exception + { + return new SwiftPseudoLock + ( + objectApi, + arguments.getLockArguments().getPrefix(), + arguments.getLockArguments().getTimeoutMs(), + arguments.getLockArguments().getPollingMs(), + arguments.getLockArguments().getSettlingMs() + ); + } + + @Override + public LoadedInstanceConfig loadConfig() throws Exception + { + Properties properties = new Properties(); + SwiftObject obj= objectApi.get(this.arguments.getKey()); + + if (obj !=null) + { + System.out.println("name=" +obj.getName() + ",meta="+ obj.getMetadata()+ ",headers="+obj.getHeaders()); + InputStream is = null; + try{ + is = obj.getPayload().openStream(); + properties.load(is); + is.close(); + }finally{ + if (is !=null) + is.close(); + } + } + + PropertyBasedInstanceConfig config = new PropertyBasedInstanceConfig(properties, defaults); + return new LoadedInstanceConfig(config, obj.getLastModified().getTime()); + } + + + @Override + public LoadedInstanceConfig storeConfig(ConfigCollection config, long compareVersion) throws Exception + { + String name =this.arguments.getKey(); + SwiftObject obj= objectApi.get(name); + if ( obj!=null && obj.getLastModified().getTime() != compareVersion ) + return null; + + PropertyBasedInstanceConfig propertyBasedInstanceConfig = new PropertyBasedInstanceConfig(config); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + propertyBasedInstanceConfig.getProperties().store(out, "Auto-generated by Exhibitor " + hostname); + + byte[] bytes = out.toByteArray(); + Payload payload = new ByteArrayPayload(bytes); + objectApi.put(name, payload); + + return new LoadedInstanceConfig(propertyBasedInstanceConfig, obj.getLastModified().getTime()); + } + +} diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftPseudoLock.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftPseudoLock.java new file mode 100644 index 00000000..0a8e0caa --- /dev/null +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftPseudoLock.java @@ -0,0 +1,73 @@ +/* + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.exhibitor.core.config.swift; + +import com.netflix.exhibitor.core.config.PseudoLockBase; + +import java.util.ArrayList; +import java.util.List; + +import org.jclouds.io.Payload; +import org.jclouds.io.payloads.ByteArrayPayload; +import org.jclouds.openstack.swift.v1.domain.ObjectList; +import org.jclouds.openstack.swift.v1.features.ObjectApi; + +public class SwiftPseudoLock extends PseudoLockBase +{ + private final ObjectApi objectApi; + + public SwiftPseudoLock(ObjectApi objectApi, String lockPrefix, int timeoutMs, int pollingMs) + { + super(lockPrefix, timeoutMs, pollingMs); + this.objectApi = objectApi; + } + + public SwiftPseudoLock(ObjectApi objectApi, String lockPrefix, int timeoutMs, int pollingMs, int settlingMs) + { + super(lockPrefix, timeoutMs, pollingMs, settlingMs); + this.objectApi = objectApi; + } + + + @Override + protected void createFile(String key, byte[] contents) throws Exception + { + Payload payload = new ByteArrayPayload(contents); + objectApi.put(key, payload); + } + + @Override + protected void deleteFile(String key) throws Exception + { + objectApi.delete(key); + } + + @Override + protected List getFileNames(String lockPrefix) throws Exception + { + // TODO find a better way to get files by prefix + ObjectList list = objectApi.list(); + List names = new ArrayList(); + for (int i=0; i< list.size(); i++) + { + String name = list.get(i).getName(); + if (name.startsWith(lockPrefix)) + names.add(name); + } + return names; + } +} diff --git a/exhibitor-standalone/.gitignore b/exhibitor-standalone/.gitignore new file mode 100644 index 00000000..ae3c1726 --- /dev/null +++ b/exhibitor-standalone/.gitignore @@ -0,0 +1 @@ +/bin/ diff --git a/exhibitor-standalone/build.gradle b/exhibitor-standalone/build.gradle index 77f05111..2b58e1a1 100644 --- a/exhibitor-standalone/build.gradle +++ b/exhibitor-standalone/build.gradle @@ -1,5 +1,6 @@ apply plugin: 'nebula.provided-base' apply plugin: 'application' +apply plugin: 'eclipse' mainClassName = 'com.netflix.exhibitor.application.ExhibitorMain' @@ -17,3 +18,15 @@ dependencies { provided 'javax.servlet:servlet-api:' + servletVersion } + +task fatJar(type: Jar) { + manifest { + attributes ( + 'Main-Class': 'com.netflix.exhibitor.application.ExhibitorMain', + 'Implementation-Version': project.version + ) + } + baseName = project.name + '-all' + from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } } + with jar +} \ No newline at end of file diff --git a/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCLI.java b/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCLI.java index 3ee5febb..01103ed8 100644 --- a/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCLI.java +++ b/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCLI.java @@ -79,6 +79,14 @@ private OptionSection(String sectionName, Options options) public static final String ZOOKEEPER_CONFIG_POLLING = "zkconfigpollms"; public static final String NONE_CONFIG_DIRECTORY = "noneconfigdir"; public static final String INITIAL_CONFIG_FILE = "defaultconfig"; + + public static final String SWIFT_PROVIDER= "swiftprovider"; + public static final String SWIFT_IDENTITY= "swiftidentity"; + public static final String SWIFT_API_KEY = "swiftapikey"; + public static final String SWIFT_AUTH_URL = "swiftauthurl"; + public static final String SWIFT_BACKUP = "swiftbackup"; + public static final String SWIFT_CONFIG = "swiftconfig"; + public static final String SWIFT_CONFIG_PREFIX = "swiftconfigprefix"; public static final String FILESYSTEMBACKUP = "filesystembackup"; public static final String TIMEOUT = "timeout"; @@ -112,6 +120,7 @@ private OptionSection(String sectionName, Options options) public static final String DEFAULT_ZOOKEEPER_CONFIG_RETRY = "1000:3"; public static final String DEFAULT_ZOOKEEPER_CONFIG_POLLING = "10000"; public static final String DEFAULT_ZOOKEEPER_CONFIG_EXHIBITOR_URI_PATH = "/"; + public static final String DEFAULT_SWIFT_PROVIDER = "exhibitor.properties"; public ExhibitorCLI() { @@ -138,6 +147,10 @@ public ExhibitorCLI() s3ConfigOptions.addOption(null, S3_CONFIG, true, "The bucket name and key to store the config (s3credentials may be provided as well). Argument is [bucket name]:[key]."); s3ConfigOptions.addOption(null, S3_CONFIG_PREFIX, true, "When using AWS S3 shared config files, the prefix to use for values such as locks. Default is " + DEFAULT_PREFIX); + Options swiftConfigOptions = new Options(); + swiftConfigOptions.addOption(null, SWIFT_CONFIG, true, "The container name and key to store the config. Argument is [container name]:[key]."); + swiftConfigOptions.addOption(null, SWIFT_CONFIG_PREFIX, true, "When using Swift shared config files, the prefix to use for values such as locks. Default is " + DEFAULT_PREFIX); + Options zookeeperConfigOptions = new Options(); zookeeperConfigOptions.addOption(null, ZOOKEEPER_CONFIG_INITIAL_CONNECT_STRING, true, "The initial connection string for ZooKeeper shared config storage. E.g: \"host1:2181,host2:2181...\""); zookeeperConfigOptions.addOption(null, ZOOKEEPER_CONFIG_EXHIBITOR_PORT, true, "Used if the ZooKeeper shared config is also running Exhibitor. This is the port that Exhibitor is listening on. IMPORTANT: if this value is not set it implies that Exhibitor is not being used on the ZooKeeper shared config."); @@ -152,12 +165,19 @@ public ExhibitorCLI() Options backupOptions = new Options(); backupOptions.addOption(null, S3_BACKUP, true, "If true, enables AWS S3 backup of ZooKeeper log files (s3credentials may be provided as well)."); backupOptions.addOption(null, FILESYSTEMBACKUP, true, "If true, enables file system backup of ZooKeeper log files."); + backupOptions.addOption(null, SWIFT_BACKUP, true, "If true, enables Swift backup of ZooKeeper."); Options s3Options = new Options(); s3Options.addOption(null, S3_CREDENTIALS, true, "Optional credentials to use for s3backup or s3config. Argument is the path to an AWS credential properties file with two properties: " + PropertyBasedS3Credential.PROPERTY_S3_KEY_ID + " and " + PropertyBasedS3Credential.PROPERTY_S3_SECRET_KEY); s3Options.addOption(null, S3_REGION, true, "Optional region for S3 calls (e.g. \"eu-west-1\"). Will be used to set the S3 client's endpoint."); s3Options.addOption(null, S3_PROXY, true, "Optional configuration used when when connecting to S3 via a proxy. Argument is the path to an AWS credential properties file with four properties (only host, port and protocol are required if using a proxy): " + PropertyBasedS3ClientConfig.PROPERTY_S3_PROXY_HOST + ", " + PropertyBasedS3ClientConfig.PROPERTY_S3_PROXY_PORT + ", " + PropertyBasedS3ClientConfig.PROPERTY_S3_PROXY_USERNAME + ", " + PropertyBasedS3ClientConfig.PROPERTY_S3_PROXY_PASSWORD); + Options swiftOptions = new Options(); + swiftOptions.addOption(null, SWIFT_PROVIDER, true, "Optional provider for jclouds to use for swiftbackup or swiftconfig. Default is "+ DEFAULT_SWIFT_PROVIDER+"."); + swiftOptions.addOption(null, SWIFT_IDENTITY, true, "Optional identify for jcloud to use for swiftbackup or swiftconfig."); + swiftOptions.addOption(null, SWIFT_API_KEY, true, "Optional api key to use for swiftbackup or swiftconfig."); + swiftOptions.addOption(null, SWIFT_AUTH_URL, true, "Optional authentication url to use for swiftbackup or swiftconfig."); + generalOptions = new Options(); generalOptions.addOption(null, TIMEOUT, true, "Connection timeout (ms) for ZK connections. Default is 30000."); generalOptions.addOption(null, LOGLINES, true, "Max lines of logging to keep in memory for display. Default is 1000."); @@ -181,6 +201,8 @@ public ExhibitorCLI() options = new Options(); addAll("S3 Options", s3Options); addAll("Configuration Options for Type \"s3\"", s3ConfigOptions); + addAll("Swift Options", swiftOptions); + addAll("Configuration Options for Type \"swift\"", swiftConfigOptions); addAll("Configuration Options for Type \"zookeeper\"", zookeeperConfigOptions); addAll("Configuration Options for Type \"file\"", fileConfigOptions); addAll("Configuration Options for Type \"none\"", noneConfigOptions); diff --git a/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCreator.java b/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCreator.java index da90e1da..d58bd12a 100644 --- a/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCreator.java +++ b/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCreator.java @@ -17,12 +17,15 @@ package com.netflix.exhibitor.standalone; import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import com.google.inject.Module; import com.netflix.exhibitor.core.ExhibitorArguments; import com.netflix.exhibitor.core.backup.BackupProvider; import com.netflix.exhibitor.core.backup.filesystem.FileSystemBackupProvider; import com.netflix.exhibitor.core.backup.s3.S3BackupProvider; +import com.netflix.exhibitor.core.backup.swift.SwiftBackupProvider; import com.netflix.exhibitor.core.config.AutoManageLockArguments; import com.netflix.exhibitor.core.config.ConfigProvider; import com.netflix.exhibitor.core.config.DefaultProperties; @@ -35,12 +38,16 @@ import com.netflix.exhibitor.core.config.s3.S3ConfigArguments; import com.netflix.exhibitor.core.config.s3.S3ConfigAutoManageLockArguments; import com.netflix.exhibitor.core.config.s3.S3ConfigProvider; +import com.netflix.exhibitor.core.config.swift.SwiftConfigArguments; +import com.netflix.exhibitor.core.config.swift.SwiftConfigAutoManageLockArguments; +import com.netflix.exhibitor.core.config.swift.SwiftConfigProvider; import com.netflix.exhibitor.core.config.zookeeper.ZookeeperConfigProvider; import com.netflix.exhibitor.core.s3.PropertyBasedS3ClientConfig; import com.netflix.exhibitor.core.s3.PropertyBasedS3Credential; import com.netflix.exhibitor.core.s3.S3ClientFactoryImpl; import com.netflix.exhibitor.core.servo.ServoRegistration; import com.netflix.servo.jmx.JmxMonitorRegistry; + import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.ParseException; @@ -58,6 +65,9 @@ import org.apache.zookeeper.common.PathUtils; import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Id; +import org.jclouds.ContextBuilder; +import org.jclouds.logging.slf4j.config.SLF4JLoggingModule; +import org.jclouds.openstack.swift.v1.SwiftApi; import org.mortbay.jetty.security.BasicAuthenticator; import org.mortbay.jetty.security.Constraint; import org.mortbay.jetty.security.ConstraintMapping; @@ -66,6 +76,7 @@ import org.mortbay.jetty.security.SecurityHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.io.BufferedInputStream; import java.io.Closeable; import java.io.File; @@ -117,6 +128,8 @@ public ExhibitorCreator(String[] args) throws Exception } checkMutuallyExclusive(cli, commandLine, S3_BACKUP, FILESYSTEMBACKUP); + checkMutuallyExclusive(cli, commandLine, S3_BACKUP, SWIFT_BACKUP); + checkMutuallyExclusive(cli, commandLine, FILESYSTEMBACKUP, SWIFT_BACKUP); String s3Region = commandLine.getOptionValue(S3_REGION, null); PropertyBasedS3Credential awsCredentials = null; @@ -131,6 +144,23 @@ public ExhibitorCreator(String[] args) throws Exception awsClientConfig = new PropertyBasedS3ClientConfig(new File(commandLine.getOptionValue(S3_PROXY))); } + Iterable modules = ImmutableSet.of( + new SLF4JLoggingModule()); + + String provider = commandLine.getOptionValue(SWIFT_PROVIDER,"openstack-swift"); + String identity = commandLine.getOptionValue(SWIFT_IDENTITY,null); + String apikey = commandLine.getOptionValue(SWIFT_API_KEY,null); + String authUrl = commandLine.getOptionValue(SWIFT_AUTH_URL,null); + + SwiftApi swiftApi = null; + + if (identity!=null && apikey !=null && authUrl !=null) + swiftApi = ContextBuilder.newBuilder(provider) + .endpoint(authUrl) + .credentials(identity, apikey) + .modules(modules) + .buildApi(SwiftApi.class); + BackupProvider backupProvider = null; if ( "true".equalsIgnoreCase(commandLine.getOptionValue(S3_BACKUP)) ) { @@ -139,8 +169,13 @@ public ExhibitorCreator(String[] args) throws Exception else if ( "true".equalsIgnoreCase(commandLine.getOptionValue(FILESYSTEMBACKUP)) ) { backupProvider = new FileSystemBackupProvider(); + } + else if ( "true".equalsIgnoreCase(commandLine.getOptionValue(SWIFT_BACKUP)) ) + { + backupProvider = new SwiftBackupProvider(swiftApi); } + int timeoutMs = Integer.parseInt(commandLine.getOptionValue(TIMEOUT, "30000")); int logWindowSizeLines = Integer.parseInt(commandLine.getOptionValue(LOGLINES, "1000")); int configCheckMs = Integer.parseInt(commandLine.getOptionValue(CONFIGCHECKMS, "30000")); @@ -155,7 +190,7 @@ else if ( "true".equalsIgnoreCase(commandLine.getOptionValue(FILESYSTEMBACKUP)) throw new MissingConfigurationTypeException("Configuration type (-" + SHORT_CONFIG_TYPE + " or --" + CONFIG_TYPE + ") must be specified", cli); } - ConfigProvider configProvider = makeConfigProvider(configType, cli, commandLine, awsCredentials, awsClientConfig, backupProvider, useHostname, s3Region); + ConfigProvider configProvider = makeConfigProvider(configType, cli, commandLine, awsCredentials, awsClientConfig, backupProvider, useHostname, s3Region, swiftApi); if ( configProvider == null ) { throw new ExhibitorCreatorExit(cli); @@ -278,7 +313,7 @@ public String getRemoteAuthSpec() return remoteAuthSpec; } - private ConfigProvider makeConfigProvider(String configType, ExhibitorCLI cli, CommandLine commandLine, PropertyBasedS3Credential awsCredentials, PropertyBasedS3ClientConfig awsClientConfig, BackupProvider backupProvider, String useHostname, String s3Region) throws Exception + private ConfigProvider makeConfigProvider(String configType, ExhibitorCLI cli, CommandLine commandLine, PropertyBasedS3Credential awsCredentials, PropertyBasedS3ClientConfig awsClientConfig, BackupProvider backupProvider, String useHostname, String s3Region, SwiftApi swiftApi) throws Exception { Properties defaultProperties = makeDefaultProperties(commandLine, backupProvider); @@ -300,6 +335,10 @@ else if ( configType.equals("none") ) log.warn("Warning: you have intentionally turned off shared configuration. This mode is meant for special purposes only. Please verify that this is your intent."); configProvider = getNoneProvider(commandLine, defaultProperties); } + else if ( configType.equals("swift") ) + { + configProvider = getSwiftProvider(cli, commandLine, swiftApi, useHostname,defaultProperties); + } else { configProvider = null; @@ -528,6 +567,12 @@ private ConfigProvider getS3Provider(ExhibitorCLI cli, CommandLine commandLine, return new S3ConfigProvider(new S3ClientFactoryImpl(), awsCredentials, awsClientConfig, getS3Arguments(cli, commandLine.getOptionValue(S3_CONFIG), prefix), hostname, defaultProperties, s3Region); } + private ConfigProvider getSwiftProvider(ExhibitorCLI cli, CommandLine commandLine, SwiftApi swiftApi, String hostname, Properties defaultProperties) throws Exception + { + String prefix = cli.getOptions().hasOption(SWIFT_CONFIG_PREFIX) ? commandLine.getOptionValue(SWIFT_CONFIG_PREFIX) : DEFAULT_PREFIX; + return new SwiftConfigProvider(swiftApi, getSwiftArguments(cli, commandLine.getOptionValue(SWIFT_CONFIG), prefix), hostname, defaultProperties); + } + private void checkMutuallyExclusive(ExhibitorCLI cli, CommandLine commandLine, String option1, String option2) throws ExhibitorCreatorExit { if ( commandLine.hasOption(option1) && commandLine.hasOption(option2) ) @@ -548,6 +593,17 @@ private S3ConfigArguments getS3Arguments(ExhibitorCLI cli, String value, String return new S3ConfigArguments(parts[0].trim(), parts[1].trim(), new S3ConfigAutoManageLockArguments(prefix + "-lock-")); } + private SwiftConfigArguments getSwiftArguments(ExhibitorCLI cli, String value, String prefix) throws ExhibitorCreatorExit + { + String[] parts = value.split(":"); + if ( parts.length != 2 ) + { + log.error("Bad swiftconfig argument: " + value); + throw new ExhibitorCreatorExit(cli); + } + return new SwiftConfigArguments(parts[0].trim(), parts[1].trim(), new SwiftConfigAutoManageLockArguments(prefix + "-lock-")); + } + private CuratorFramework makeCurator(final String connectString, int baseSleepTimeMs, int maxRetries, int exhibitorPort, String exhibitorRestPath, int pollingMs) { List hostnames = Lists.newArrayList(); diff --git a/gradle.properties b/gradle.properties index 06085ea0..caba3d1d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -12,3 +12,4 @@ jacksonVersion=1.8.3 luceneVersion=3.6.0 awsVersion=1.9.3 mockitoVersion=1.8.5 +jcloudsVersion=1.9.0 \ No newline at end of file From 732bbaefd7e245a8d605ef5c5097005d8896e3e1 Mon Sep 17 00:00:00 2001 From: Yang Lei Date: Tue, 14 Jul 2015 20:06:11 -0400 Subject: [PATCH 02/11] revise build fat jar --- build.gradle | 1 + .../backup/swift/SwiftBackupProvider.java | 45 +++++++++---------- .../config/swift/SwiftConfigProvider.java | 32 ++++++++++--- .../core/config/swift/SwiftPseudoLock.java | 15 +++++++ exhibitor-standalone/build.gradle | 12 ----- shadow/build.gradle | 44 ++++++++++++++++++ 6 files changed, 107 insertions(+), 42 deletions(-) create mode 100644 shadow/build.gradle diff --git a/build.gradle b/build.gradle index 0c952302..ebb74dd5 100644 --- a/build.gradle +++ b/build.gradle @@ -13,6 +13,7 @@ subprojects { repositories { jcenter() + mavenLocal() } sourceCompatibility = 1.6 diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/SwiftBackupProvider.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/SwiftBackupProvider.java index 49acce91..e3090a10 100644 --- a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/SwiftBackupProvider.java +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/SwiftBackupProvider.java @@ -66,6 +66,11 @@ public class SwiftBackupProvider implements BackupProvider static final String SEPARATOR = "/"; private static final String SEPARATOR_REPLACEMENT = "_"; + /** + * + * @param swiftApi the SwiftApi + * @throws Exception + */ public SwiftBackupProvider(SwiftApi swiftApi) throws Exception { this.swiftApi = swiftApi; @@ -100,29 +105,22 @@ private ObjectApi getObjectApi(String containerName) @Override public UploadResult uploadBackup(Exhibitor exhibitor, BackupMetaData backup, File source, final Map configValues) throws Exception { + System.out.println("uploadBackup:"+backup); List availableBackups = getAvailableBackups(exhibitor, configValues); if ( availableBackups.contains(backup) ) { return UploadResult.DUPLICATE; } - RetryPolicy retryPolicy = makeRetryPolicy(configValues); - Throttle throttle = makeThrottle(configValues); - String key = toKey(backup, configValues); ObjectApi objectApi = getObjectApi(configValues.get(CONFIG_CONTAINER.getKey())); - if ( source.length() < MIN_SWIFT_PART_SIZE ) - { - byte[] bytes = Files.toByteArray(source); - Payload payload = new ByteArrayPayload(bytes); - objectApi.put(key, payload); - } - else - { - multiPartUpload(source, configValues, retryPolicy, throttle, key); - } - + byte[] bytes = Files.toByteArray(source); + Payload payload = new ByteArrayPayload(bytes); + objectApi.put(key, payload); + + //TODO find a way to do multiparts + UploadResult result = UploadResult.SUCCEEDED; for ( BackupMetaData existing : availableBackups ) { @@ -135,20 +133,13 @@ public UploadResult uploadBackup(Exhibitor exhibitor, BackupMetaData backup, Fil return result; } - private void multiPartUpload(File source, Map configValues, RetryPolicy retryPolicy, Throttle throttle, String key) throws Exception - { - // TODO how to support multipart on jcloud swift? - ObjectApi objectApi = getObjectApi(configValues.get(CONFIG_CONTAINER.getKey())); - byte[] bytes = Files.toByteArray(source); - Payload payload = new ByteArrayPayload(bytes); - objectApi.put(key, payload); - - } @Override public BackupStream getBackupStream(Exhibitor exhibitor, BackupMetaData backup, Map configValues) throws Exception { - long startMs = System.currentTimeMillis(); + System.out.println("getBackupStream:"+backup); + + long startMs = System.currentTimeMillis(); RetryPolicy retryPolicy = makeRetryPolicy(configValues); int retryCount = 0; ObjectApi objectApi = getObjectApi(configValues.get(CONFIG_CONTAINER.getKey())); @@ -161,6 +152,7 @@ public BackupStream getBackupStream(Exhibitor exhibitor, BackupMetaData backup, } catch ( Exception e) { + e.printStackTrace(); /*TODO any fastpath on failure? if (some condition) { @@ -234,6 +226,8 @@ public void close() throws IOException @Override public void downloadBackup(Exhibitor exhibitor, BackupMetaData backup, OutputStream destination, Map configValues) throws Exception { + System.out.println("downloadBackup:"+backup); + byte[] buffer = new byte[MIN_SWIFT_PART_SIZE]; long startMs = System.currentTimeMillis(); @@ -267,6 +261,7 @@ public void downloadBackup(Exhibitor exhibitor, BackupMetaData backup, OutputStr } catch ( Exception e ) { + e.printStackTrace(); if ( !retryPolicy.allowRetry(retryCount++, System.currentTimeMillis() - startMs, RetryLoop.getDefaultRetrySleeper()) ) { done = true; @@ -283,6 +278,8 @@ public void downloadBackup(Exhibitor exhibitor, BackupMetaData backup, OutputStr @Override public List getAvailableBackups(Exhibitor exhibitor, Map configValues) throws Exception { + System.out.println("getAvailableBackups:"+getKeyPrefix(configValues)); + String keyPrefix = getKeyPrefix(configValues); List completeList = Lists.newArrayList(); diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftConfigProvider.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftConfigProvider.java index 2e0caf02..b97f8167 100644 --- a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftConfigProvider.java +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftConfigProvider.java @@ -32,6 +32,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.util.Date; import java.util.Properties; public class SwiftConfigProvider implements ConfigProvider @@ -42,6 +43,14 @@ public class SwiftConfigProvider implements ConfigProvider private final ObjectApi objectApi; private final Properties defaults; + /** + * + * @param swiftApi SwiftApi + * @param arguments args + * @param hostname this VM's hostname + * @param defaults default props + * @throws Exception + */ public SwiftConfigProvider(SwiftApi swiftApi, SwiftConfigArguments arguments,String hostname, Properties defaults) throws Exception { this.swiftApi = swiftApi; @@ -91,13 +100,15 @@ public PseudoLock newPseudoLock() throws Exception @Override public LoadedInstanceConfig loadConfig() throws Exception - { + { + Date lastModified; Properties properties = new Properties(); SwiftObject obj= objectApi.get(this.arguments.getKey()); if (obj !=null) { - System.out.println("name=" +obj.getName() + ",meta="+ obj.getMetadata()+ ",headers="+obj.getHeaders()); + lastModified = obj.getLastModified(); + InputStream is = null; try{ is = obj.getPayload().openStream(); @@ -108,19 +119,24 @@ public LoadedInstanceConfig loadConfig() throws Exception is.close(); } } + else + { + lastModified = new Date(0L); + } PropertyBasedInstanceConfig config = new PropertyBasedInstanceConfig(properties, defaults); - return new LoadedInstanceConfig(config, obj.getLastModified().getTime()); + return new LoadedInstanceConfig(config, lastModified.getTime()); } @Override public LoadedInstanceConfig storeConfig(ConfigCollection config, long compareVersion) throws Exception { - String name =this.arguments.getKey(); - SwiftObject obj= objectApi.get(name); + SwiftObject obj= objectApi.get(arguments.getKey()); if ( obj!=null && obj.getLastModified().getTime() != compareVersion ) + { return null; + } PropertyBasedInstanceConfig propertyBasedInstanceConfig = new PropertyBasedInstanceConfig(config); ByteArrayOutputStream out = new ByteArrayOutputStream(); @@ -128,7 +144,11 @@ public LoadedInstanceConfig storeConfig(ConfigCollection config, long compareVer byte[] bytes = out.toByteArray(); Payload payload = new ByteArrayPayload(bytes); - objectApi.put(name, payload); + objectApi.put(arguments.getKey(), payload); + if (obj == null) + { + obj = objectApi.get(arguments.getKey()); + } return new LoadedInstanceConfig(propertyBasedInstanceConfig, obj.getLastModified().getTime()); } diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftPseudoLock.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftPseudoLock.java index 0a8e0caa..3c5425e2 100644 --- a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftPseudoLock.java +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/swift/SwiftPseudoLock.java @@ -30,12 +30,27 @@ public class SwiftPseudoLock extends PseudoLockBase { private final ObjectApi objectApi; + /** + * + * @param objectApi ObjectApi + * @param lockPrefix key prefix + * @param timeoutMs max age for locks + * @param pollingMs how often to poll S3 + */ public SwiftPseudoLock(ObjectApi objectApi, String lockPrefix, int timeoutMs, int pollingMs) { super(lockPrefix, timeoutMs, pollingMs); this.objectApi = objectApi; } + /** + * + * @param objectApi ObjectApi + * @param lockPrefix key prefix + * @param timeoutMs max age for locks + * @param pollingMs how often to poll S3 + * @param settlingMs how long to wait to reach consistency + */ public SwiftPseudoLock(ObjectApi objectApi, String lockPrefix, int timeoutMs, int pollingMs, int settlingMs) { super(lockPrefix, timeoutMs, pollingMs, settlingMs); diff --git a/exhibitor-standalone/build.gradle b/exhibitor-standalone/build.gradle index 2b58e1a1..21e87a7d 100644 --- a/exhibitor-standalone/build.gradle +++ b/exhibitor-standalone/build.gradle @@ -18,15 +18,3 @@ dependencies { provided 'javax.servlet:servlet-api:' + servletVersion } - -task fatJar(type: Jar) { - manifest { - attributes ( - 'Main-Class': 'com.netflix.exhibitor.application.ExhibitorMain', - 'Implementation-Version': project.version - ) - } - baseName = project.name + '-all' - from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } } - with jar -} \ No newline at end of file diff --git a/shadow/build.gradle b/shadow/build.gradle new file mode 100644 index 00000000..9f0e9d34 --- /dev/null +++ b/shadow/build.gradle @@ -0,0 +1,44 @@ +buildscript { + repositories { + jcenter() + mavenCentral() + } + dependencies { + classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.1' + } +} + +apply plugin: 'java' +apply plugin: 'com.github.johnrengelman.shadow' + +group = 'exhibitor' +archivesBaseName = 'exhibitor' +version = '1.5.6-SNAPSHOT' + +repositories { + jcenter() + mavenLocal() + mavenCentral() + maven { + url "https://repository.jboss.org/nexus/content/groups/public/" + } +} + +dependencies { + compile 'com.netflix.exhibitor:exhibitor-standalone:1.5.6-SNAPSHOT' +} + +jar { + manifest { + attributes ( + 'Main-Class': 'com.netflix.exhibitor.application.ExhibitorMain', + 'Implementation-Version': project.version + ) + } +} + +shadowJar { + mergeServiceFiles() +} + +assemble.dependsOn shadowJar From 00783f342fbaaf94ebeda592df00317ddfb81dd9 Mon Sep 17 00:00:00 2001 From: Yang Lei Date: Wed, 15 Jul 2015 17:46:26 -0400 Subject: [PATCH 03/11] remove tracing --- .../core/backup/swift/SwiftBackupProvider.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/SwiftBackupProvider.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/SwiftBackupProvider.java index e3090a10..c04c3bc3 100644 --- a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/SwiftBackupProvider.java +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/SwiftBackupProvider.java @@ -87,7 +87,8 @@ public List getConfigs() public boolean isValidConfig(Exhibitor exhibitor, Map configValues) { String containerName = (configValues != null) ? configValues.get(CONFIG_CONTAINER.getKey()) : null; - return (containerName != null) && (containerName.trim().length() > 0); + boolean valid = (containerName != null) && (containerName.trim().length() > 0); + return valid; } private ObjectApi getObjectApi(String containerName) @@ -105,7 +106,6 @@ private ObjectApi getObjectApi(String containerName) @Override public UploadResult uploadBackup(Exhibitor exhibitor, BackupMetaData backup, File source, final Map configValues) throws Exception { - System.out.println("uploadBackup:"+backup); List availableBackups = getAvailableBackups(exhibitor, configValues); if ( availableBackups.contains(backup) ) { @@ -137,8 +137,6 @@ public UploadResult uploadBackup(Exhibitor exhibitor, BackupMetaData backup, Fil @Override public BackupStream getBackupStream(Exhibitor exhibitor, BackupMetaData backup, Map configValues) throws Exception { - System.out.println("getBackupStream:"+backup); - long startMs = System.currentTimeMillis(); RetryPolicy retryPolicy = makeRetryPolicy(configValues); int retryCount = 0; @@ -226,8 +224,6 @@ public void close() throws IOException @Override public void downloadBackup(Exhibitor exhibitor, BackupMetaData backup, OutputStream destination, Map configValues) throws Exception { - System.out.println("downloadBackup:"+backup); - byte[] buffer = new byte[MIN_SWIFT_PART_SIZE]; long startMs = System.currentTimeMillis(); @@ -278,8 +274,6 @@ public void downloadBackup(Exhibitor exhibitor, BackupMetaData backup, OutputStr @Override public List getAvailableBackups(Exhibitor exhibitor, Map configValues) throws Exception { - System.out.println("getAvailableBackups:"+getKeyPrefix(configValues)); - String keyPrefix = getKeyPrefix(configValues); List completeList = Lists.newArrayList(); @@ -292,6 +286,8 @@ public List getAvailableBackups(Exhibitor exhibitor, Map Date: Wed, 15 Jul 2015 19:37:33 -0400 Subject: [PATCH 04/11] add doc --- README.md | 3 ++ .../exhibitor/standalone/ExhibitorCLI.java | 2 +- .../standalone/ExhibitorCreator.java | 2 +- swift/README_swift.md | 38 +++++++++++++++++++ swift/sample_config.properties | 11 ++++++ swift/sample_run.sh | 3 ++ 6 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 swift/README_swift.md create mode 100644 swift/sample_config.properties create mode 100644 swift/sample_run.sh diff --git a/README.md b/README.md index 6922b8cc..a7ab7fa1 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,9 @@ Exhibitor is a supervisor system for ZooKeeper. Please see the doc at https://github.com/Netflix/exhibitor/wiki +[OpenStack Swift Object Storage as Backup and Config Provider](swift/README_Swift.md) + + ## BUILDING Exhibitor is built via Gradle (http://www.gradle.org). To build from the command line: diff --git a/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCLI.java b/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCLI.java index 01103ed8..4c9e41d7 100644 --- a/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCLI.java +++ b/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCLI.java @@ -120,7 +120,7 @@ private OptionSection(String sectionName, Options options) public static final String DEFAULT_ZOOKEEPER_CONFIG_RETRY = "1000:3"; public static final String DEFAULT_ZOOKEEPER_CONFIG_POLLING = "10000"; public static final String DEFAULT_ZOOKEEPER_CONFIG_EXHIBITOR_URI_PATH = "/"; - public static final String DEFAULT_SWIFT_PROVIDER = "exhibitor.properties"; + public static final String DEFAULT_SWIFT_PROVIDER = "openstack-swift"; public ExhibitorCLI() { diff --git a/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCreator.java b/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCreator.java index d58bd12a..4a510812 100644 --- a/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCreator.java +++ b/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCreator.java @@ -147,7 +147,7 @@ public ExhibitorCreator(String[] args) throws Exception Iterable modules = ImmutableSet.of( new SLF4JLoggingModule()); - String provider = commandLine.getOptionValue(SWIFT_PROVIDER,"openstack-swift"); + String provider = commandLine.getOptionValue(SWIFT_PROVIDER,DEFAULT_SWIFT_PROVIDER); String identity = commandLine.getOptionValue(SWIFT_IDENTITY,null); String apikey = commandLine.getOptionValue(SWIFT_API_KEY,null); String authUrl = commandLine.getOptionValue(SWIFT_AUTH_URL,null); diff --git a/swift/README_swift.md b/swift/README_swift.md new file mode 100644 index 00000000..7579c6b5 --- /dev/null +++ b/swift/README_swift.md @@ -0,0 +1,38 @@ +Netflix Exhibitor on OpenStack Swift Object Storage +================ + + +## Build + + git clone https://github.com/yanglei99/exhibitor + + ./gradlew clean build install + cd shadow + ../gradlew shadowJar + +* copy exhibitor-1.5.6-SNAPSHOT-all.jar from build/libs to your target zookeeper nodes + + +## Installation + +* upload shadow/build/libs/exhibitor-*-all.jar to the target ZooKeeper nodes + +* optional, revise[sample_config.properties](sample_config.properties) and upload to object storage + +* revise [sample_run.sh](sample_run.sh), and upload to target ZooKeeper nodes + + +## Run + + +Use the run script above. + + +## Verification + +Access http://:8090/exhibitor/v1/ui/index.html + + +## Known Issue + +* Do not support zookeeper repo installation. Download yourself then revise the location of zookeeper installation either in the property file or through config panel. \ No newline at end of file diff --git a/swift/sample_config.properties b/swift/sample_config.properties new file mode 100644 index 00000000..dfd172e0 --- /dev/null +++ b/swift/sample_config.properties @@ -0,0 +1,11 @@ +#Auto-generated by Exhibitor mesos.master-2201 +#Wed Jul 15 16:14:00 CDT 2015 +com.netflix.exhibitor-rolling-hostnames= +com.netflix.exhibitor-rolling.zookeeper-data-directory=/var/zookeeper/ +com.netflix.exhibitor.zookeeper-data-directory=/var/zookeeper/ +com.netflix.exhibitor.zookeeper-install-directory=/root/zookeeper-3.4.6/ +com.netflix.exhibitor.log-index-directory=/var/log/zookeeper_index +com.netflix.exhibitor-rolling.log-index-directory=/var/log/zookeeper_index +com.netflix.exhibitor-rolling.backup-extra=throttle\=0&container-name\=myexhibitor&key-prefix\=mybackup&max-retries\=3&retry-sleep-ms\=200 +com.netflix.exhibitor-rolling.zookeeper-install-directory=/root/zookeeper-3.4.6/ +com.netflix.exhibitor.backup-extra=throttle\=0&container-name\=myexhibitor&key-prefix\=mybackup&max-retries\=3&retry-sleep-ms\=200 diff --git a/swift/sample_run.sh b/swift/sample_run.sh new file mode 100644 index 00000000..fe6c1d22 --- /dev/null +++ b/swift/sample_run.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +java -Djclouds.keystone.credential-type=tempAuthCredentials -jar exhibitor-1.5.6-SNAPSHOT-all.jar -c swift --hostname $(hostname) --port 8090 --swiftbackup true --swiftidentity --swiftapikey --swiftauthurl https://.objectstorage.softlayer.net/auth/v1.0 --swiftconfig myexhibitor:myconfig.properties \ No newline at end of file From 77b0ad2a0b2da2cfa5812f970d37d8d3928e2b3f Mon Sep 17 00:00:00 2001 From: Yang Lei Date: Wed, 15 Jul 2015 19:39:31 -0400 Subject: [PATCH 05/11] fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a7ab7fa1..366caad5 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Exhibitor is a supervisor system for ZooKeeper. Please see the doc at https://github.com/Netflix/exhibitor/wiki -[OpenStack Swift Object Storage as Backup and Config Provider](swift/README_Swift.md) +[OpenStack Swift Object Storage as Backup and Config Provider](swift/README_swift.md) ## BUILDING From 0651620799904ed8515f91a995573ab59967a080 Mon Sep 17 00:00:00 2001 From: Yang Lei Date: Wed, 15 Jul 2015 19:55:44 -0400 Subject: [PATCH 06/11] revise readme --- swift/README_swift.md | 14 +++++++------- swift/sample_config.properties | 4 +--- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/swift/README_swift.md b/swift/README_swift.md index 7579c6b5..369717b3 100644 --- a/swift/README_swift.md +++ b/swift/README_swift.md @@ -4,28 +4,28 @@ Netflix Exhibitor on OpenStack Swift Object Storage ## Build - git clone https://github.com/yanglei99/exhibitor + git clone https://github.com/yanglei99/exhibitor.git ./gradlew clean build install + cd shadow + ../gradlew shadowJar -* copy exhibitor-1.5.6-SNAPSHOT-all.jar from build/libs to your target zookeeper nodes - ## Installation * upload shadow/build/libs/exhibitor-*-all.jar to the target ZooKeeper nodes -* optional, revise[sample_config.properties](sample_config.properties) and upload to object storage +* optional, revise [sample_config.properties](sample_config.properties) and upload to the object storage as default settings -* revise [sample_run.sh](sample_run.sh), and upload to target ZooKeeper nodes +* revise [sample_run.sh](sample_run.sh), and upload to the target ZooKeeper nodes ## Run -Use the run script above. +Use the revised run script above. ## Verification @@ -35,4 +35,4 @@ Access http://:8090/exhibitor/v1/ui/index.html ## Known Issue -* Do not support zookeeper repo installation. Download yourself then revise the location of zookeeper installation either in the property file or through config panel. \ No newline at end of file +* Do not support zookeeper repo installation. Download zookeeper jar and unzip. Make sure to revise the location of zookeeper installation either in the property file or through config panel. \ No newline at end of file diff --git a/swift/sample_config.properties b/swift/sample_config.properties index dfd172e0..861f2211 100644 --- a/swift/sample_config.properties +++ b/swift/sample_config.properties @@ -1,6 +1,4 @@ -#Auto-generated by Exhibitor mesos.master-2201 -#Wed Jul 15 16:14:00 CDT 2015 -com.netflix.exhibitor-rolling-hostnames= +# A sample com.netflix.exhibitor-rolling.zookeeper-data-directory=/var/zookeeper/ com.netflix.exhibitor.zookeeper-data-directory=/var/zookeeper/ com.netflix.exhibitor.zookeeper-install-directory=/root/zookeeper-3.4.6/ From 6aa962e9b36ed046cf96f65bb91e0ddd867f779c Mon Sep 17 00:00:00 2001 From: Yang Lei Date: Wed, 15 Jul 2015 21:17:26 -0400 Subject: [PATCH 07/11] add config descriptions for swift --- .../backup/swift/SwiftBackupProvider.java | 6 ++--- swift/README_swift.md | 25 +++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/SwiftBackupProvider.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/SwiftBackupProvider.java index c04c3bc3..24e6fef0 100644 --- a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/SwiftBackupProvider.java +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/SwiftBackupProvider.java @@ -53,9 +53,9 @@ public class SwiftBackupProvider implements BackupProvider private final SwiftApi swiftApi; private static final BackupConfigSpec CONFIG_THROTTLE = new BackupConfigSpec("throttle", "Throttle (bytes/ms)", "Data throttling. Maximum bytes per millisecond.", Integer.toString(1024 * 1024), BackupConfigSpec.Type.INTEGER); - private static final BackupConfigSpec CONFIG_CONTAINER= new BackupConfigSpec("container-name", "Swift Container Name", "The Swfit containerto use", "", BackupConfigSpec.Type.STRING); - private static final BackupConfigSpec CONFIG_KEY_PREFIX = new BackupConfigSpec("key-prefix", "Swfit Key Prefix", "The prefix for Swift backup keys", "exhibitor-backup", BackupConfigSpec.Type.STRING); - private static final BackupConfigSpec CONFIG_MAX_RETRIES = new BackupConfigSpec("max-retries", "Max Retries", "Maximum retries when uploading/downloading S3 data", "3", BackupConfigSpec.Type.INTEGER); + private static final BackupConfigSpec CONFIG_CONTAINER= new BackupConfigSpec("container-name", "Swift Container Name", "The Swift container to use", "", BackupConfigSpec.Type.STRING); + private static final BackupConfigSpec CONFIG_KEY_PREFIX = new BackupConfigSpec("key-prefix", "Swift Key Prefix", "The prefix for Swift backup keys", "exhibitor-backup", BackupConfigSpec.Type.STRING); + private static final BackupConfigSpec CONFIG_MAX_RETRIES = new BackupConfigSpec("max-retries", "Max Retries", "Maximum retries when uploading/downloading Swift data", "3", BackupConfigSpec.Type.INTEGER); private static final BackupConfigSpec CONFIG_RETRY_SLEEP_MS = new BackupConfigSpec("retry-sleep-ms", "Retry Sleep (ms)", "Sleep time in milliseconds when retrying", "1000", BackupConfigSpec.Type.INTEGER); private static final List CONFIGS = Arrays.asList(CONFIG_THROTTLE, CONFIG_CONTAINER, CONFIG_KEY_PREFIX, CONFIG_MAX_RETRIES, CONFIG_RETRY_SLEEP_MS); diff --git a/swift/README_swift.md b/swift/README_swift.md index 369717b3..ac8f441c 100644 --- a/swift/README_swift.md +++ b/swift/README_swift.md @@ -32,6 +32,31 @@ Use the revised run script above. Access http://:8090/exhibitor/v1/ui/index.html +## Configuration Details + +Type | Key | Default | Details +--- | --- | --- | --- +Swift Options | | | +| swiftconfig || The container name and key to store the config. Argument is [container name]:[key]. +| swiftconfigprefix |exhibitor-| When using Swift shared config files, the prefix to use for values such as locks. +--- | --- | --- +Swift Config Options ||| +|swiftprovider |openstack-swift | Optional provider for jclouds to use for swiftbackup or swiftconfig. +|swiftidentity | | Optional identify for jcloud to use for swiftbackup or swiftconfig. +|swiftapikey | | Optional api key to use for swiftbackup or swiftconfig. +|swiftauthur l| | Optional authentication url to use for swiftbackup or swiftconfig. +--- | --- | --- +Swift Backup Options ||| +|swiftbackup||If true, enables Swift backup of ZooKeeper +--- | --- | --- +Swift Backup Configuration ||| +|Throttle (bytes/ms)|Integer.toString(1024 * 1024)|Data throttling. Maximum bytes per millisecond +|Swift Container Name||The Swift container to use +|Swfit Key Prefix|exhibitor-backup|The prefix for Swift backup keys +|Max Retries|3|Maximum retries when uploading/downloading Swift data +|Retry Sleep (ms)|1000|Sleep time in milliseconds when retrying + + ## Known Issue From 38b86a928436ebf97c1805b95982696d49b2300f Mon Sep 17 00:00:00 2001 From: Yang Lei Date: Wed, 15 Jul 2015 21:22:39 -0400 Subject: [PATCH 08/11] revise readme --- swift/README_swift.md | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/swift/README_swift.md b/swift/README_swift.md index ac8f441c..db31c308 100644 --- a/swift/README_swift.md +++ b/swift/README_swift.md @@ -35,21 +35,18 @@ Access http://:8090/exhibitor/v1/ui/index.html ## Configuration Details Type | Key | Default | Details ---- | --- | --- | --- -Swift Options | | | +--- | --- | :---: | --- +**Swift Options** | | | | swiftconfig || The container name and key to store the config. Argument is [container name]:[key]. | swiftconfigprefix |exhibitor-| When using Swift shared config files, the prefix to use for values such as locks. ---- | --- | --- -Swift Config Options ||| +**Swift Config Options** ||| |swiftprovider |openstack-swift | Optional provider for jclouds to use for swiftbackup or swiftconfig. |swiftidentity | | Optional identify for jcloud to use for swiftbackup or swiftconfig. |swiftapikey | | Optional api key to use for swiftbackup or swiftconfig. |swiftauthur l| | Optional authentication url to use for swiftbackup or swiftconfig. ---- | --- | --- -Swift Backup Options ||| +**Swift Backup Options** ||| |swiftbackup||If true, enables Swift backup of ZooKeeper ---- | --- | --- -Swift Backup Configuration ||| +**Swift Backup Configuration** ||| |Throttle (bytes/ms)|Integer.toString(1024 * 1024)|Data throttling. Maximum bytes per millisecond |Swift Container Name||The Swift container to use |Swfit Key Prefix|exhibitor-backup|The prefix for Swift backup keys From 318686bba35bfe5099d5e421777db971a500e8f2 Mon Sep 17 00:00:00 2001 From: Yang Lei Date: Wed, 15 Jul 2015 21:23:51 -0400 Subject: [PATCH 09/11] revise readme --- swift/README_swift.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/swift/README_swift.md b/swift/README_swift.md index db31c308..68aa0c12 100644 --- a/swift/README_swift.md +++ b/swift/README_swift.md @@ -37,21 +37,21 @@ Access http://:8090/exhibitor/v1/ui/index.html Type | Key | Default | Details --- | --- | :---: | --- **Swift Options** | | | -| swiftconfig || The container name and key to store the config. Argument is [container name]:[key]. -| swiftconfigprefix |exhibitor-| When using Swift shared config files, the prefix to use for values such as locks. + | swiftconfig || The container name and key to store the config. Argument is [container name]:[key]. + | swiftconfigprefix |exhibitor-| When using Swift shared config files, the prefix to use for values such as locks. **Swift Config Options** ||| -|swiftprovider |openstack-swift | Optional provider for jclouds to use for swiftbackup or swiftconfig. -|swiftidentity | | Optional identify for jcloud to use for swiftbackup or swiftconfig. -|swiftapikey | | Optional api key to use for swiftbackup or swiftconfig. -|swiftauthur l| | Optional authentication url to use for swiftbackup or swiftconfig. + |swiftprovider |openstack-swift | Optional provider for jclouds to use for swiftbackup or swiftconfig. + |swiftidentity | | Optional identify for jcloud to use for swiftbackup or swiftconfig. + |swiftapikey | | Optional api key to use for swiftbackup or swiftconfig. + |swiftauthur l| | Optional authentication url to use for swiftbackup or swiftconfig. **Swift Backup Options** ||| -|swiftbackup||If true, enables Swift backup of ZooKeeper + |swiftbackup||If true, enables Swift backup of ZooKeeper **Swift Backup Configuration** ||| -|Throttle (bytes/ms)|Integer.toString(1024 * 1024)|Data throttling. Maximum bytes per millisecond -|Swift Container Name||The Swift container to use -|Swfit Key Prefix|exhibitor-backup|The prefix for Swift backup keys -|Max Retries|3|Maximum retries when uploading/downloading Swift data -|Retry Sleep (ms)|1000|Sleep time in milliseconds when retrying + |Throttle (bytes/ms)|Integer.toString(1024 * 1024)|Data throttling. Maximum bytes per millisecond + |Swift Container Name||The Swift container to use + |Swfit Key Prefix|exhibitor-backup|The prefix for Swift backup keys + |Max Retries|3|Maximum retries when uploading/downloading Swift data + |Retry Sleep (ms)|1000|Sleep time in milliseconds when retrying From 92ebbfe8b50660b8fe8e5799cd2f2a9c0490aaea Mon Sep 17 00:00:00 2001 From: Yang Lei Date: Wed, 15 Jul 2015 21:26:10 -0400 Subject: [PATCH 10/11] revise readme --- swift/README_swift.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/swift/README_swift.md b/swift/README_swift.md index 68aa0c12..91c611e1 100644 --- a/swift/README_swift.md +++ b/swift/README_swift.md @@ -43,15 +43,15 @@ Type | Key | Default | Details |swiftprovider |openstack-swift | Optional provider for jclouds to use for swiftbackup or swiftconfig. |swiftidentity | | Optional identify for jcloud to use for swiftbackup or swiftconfig. |swiftapikey | | Optional api key to use for swiftbackup or swiftconfig. - |swiftauthur l| | Optional authentication url to use for swiftbackup or swiftconfig. + |swiftauthur | | Optional authentication url to use for swiftbackup or swiftconfig. **Swift Backup Options** ||| |swiftbackup||If true, enables Swift backup of ZooKeeper **Swift Backup Configuration** ||| - |Throttle (bytes/ms)|Integer.toString(1024 * 1024)|Data throttling. Maximum bytes per millisecond - |Swift Container Name||The Swift container to use - |Swfit Key Prefix|exhibitor-backup|The prefix for Swift backup keys - |Max Retries|3|Maximum retries when uploading/downloading Swift data - |Retry Sleep (ms)|1000|Sleep time in milliseconds when retrying +Throttle (bytes/ms)||Integer.toString(1024 * 1024)|Data throttling. Maximum bytes per millisecond +Swift Container Name|||The Swift container to use +Swfit Key Prefix||exhibitor-backup|The prefix for Swift backup keys +Max Retries||3|Maximum retries when uploading/downloading Swift data +Retry Sleep (ms)||1000|Sleep time in milliseconds when retrying From 5e64257b1549359c639631ef8ae246ef2df60409 Mon Sep 17 00:00:00 2001 From: Yang Lei Date: Thu, 16 Jul 2015 12:24:12 -0400 Subject: [PATCH 11/11] fix backupmetadata --- .../exhibitor/core/backup/swift/SwiftBackupProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/SwiftBackupProvider.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/SwiftBackupProvider.java index 24e6fef0..ba309ef4 100644 --- a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/SwiftBackupProvider.java +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/backup/swift/SwiftBackupProvider.java @@ -285,7 +285,7 @@ public List getAvailableBackups(Exhibitor exhibitor, Map