diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 83fa51e..6b43825 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -16,6 +16,7 @@ def versions = [ kotlin : '1.8.20', + coroutines : '1.7.1', errorProne : '2.5.1', gjf : '1.7', nullawayPlugin: '0.8.0', @@ -36,12 +37,12 @@ def build = [ errorProneTestHelpers: "com.google.errorprone:error_prone_test_helpers:${versions.errorProne}", nullAway : 'com.uber.nullaway:nullaway:0.8.0', gradlePlugins : [ - android : '7.4.0', + android : '8.1.4', kotlin : "${versions.kotlin}", errorProne: '2.0.1', nullAway : '1.3.0', protobuf : '0.8.12', - spotless : '4.3.0', + spotless : '6.25.0', mavenPublish : '0.27.0', dokka : '1.6.10', ] @@ -64,7 +65,10 @@ def test = [ ] def kotlin = [ - stdLibJdk8 : "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${versions.kotlin}" + stdLibJdk8 : "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${versions.kotlin}", + coroutinesCore: "org.jetbrains.kotlinx:kotlinx-coroutines-core:${versions.coroutines}", + coroutinesAndroid: "org.jetbrains.kotlinx:kotlinx-coroutines-android:${versions.coroutines}", + coroutinesGuava: "org.jetbrains.kotlinx:kotlinx-coroutines-guava:${versions.coroutines}" ] def external = [ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 60a3091..bb8069a 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ #Tue Dec 01 12:24:16 PST 2020 distributionBase=GRADLE_USER_HOME -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip distributionPath=wrapper/dists zipStorePath=wrapper/dists zipStoreBase=GRADLE_USER_HOME diff --git a/simplestore/build.gradle b/simplestore/build.gradle index bdef651..5be3f7f 100644 --- a/simplestore/build.gradle +++ b/simplestore/build.gradle @@ -39,6 +39,9 @@ android { dependencies { implementation deps.external.findBugs implementation deps.kotlin.stdLibJdk8 + implementation deps.kotlin.coroutinesCore + implementation deps.kotlin.coroutinesAndroid + implementation deps.kotlin.coroutinesGuava api deps.external.guavaAndroid testImplementation deps.test.junit diff --git a/simplestore/src/main/java/com/uber/simplestore/DirectoryProvider.kt b/simplestore/src/main/java/com/uber/simplestore/DirectoryProvider.kt new file mode 100644 index 0000000..3deebce --- /dev/null +++ b/simplestore/src/main/java/com/uber/simplestore/DirectoryProvider.kt @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2020. Uber Technologies + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.uber.simplestore + +import java.io.File + +interface DirectoryProvider { + fun cacheDirectoryPath(): File + fun filesDirectoryPath(): File +} \ No newline at end of file diff --git a/simplestore/src/main/java/com/uber/simplestore/NamespaceConfig.kt b/simplestore/src/main/java/com/uber/simplestore/NamespaceConfig.kt new file mode 100644 index 0000000..a168bb7 --- /dev/null +++ b/simplestore/src/main/java/com/uber/simplestore/NamespaceConfig.kt @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2019. Uber Technologies + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.uber.simplestore + +/** Configure how the store accesses a namespace. */ +sealed class NamespaceConfig { + /** + * Opens a namespace as performance & integrity critical. + * + * Bypasses future memory use optimizations. + */ + object CRITICAL : NamespaceConfig() + + /** + * Use the cache directory. + * + * Hides errors due to data corruption by returning a miss. + */ + object CACHE : NamespaceConfig() + + /** Default settings. */ + object DEFAULT : NamespaceConfig() +} \ No newline at end of file diff --git a/simplestore/src/main/java/com/uber/simplestore/SimpleStore.kt b/simplestore/src/main/java/com/uber/simplestore/SimpleStore.kt new file mode 100644 index 0000000..dbabf74 --- /dev/null +++ b/simplestore/src/main/java/com/uber/simplestore/SimpleStore.kt @@ -0,0 +1,144 @@ +/* + * Copyright (C) 2019. Uber Technologies + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.uber.simplestore + +import com.google.common.annotations.Beta +import com.google.common.util.concurrent.ListenableFuture +import java.io.Closeable + +/** Fast, reliable storage. */ +interface SimpleStore : Closeable { + + /** + * Retrieve a byte[]-backed String. + * + * @param key to fetch from + */ + suspend fun getString(key: String): String + + /** + * Stores a String as a byte[]. + * + * @param key to store to + * @param value to write + */ + suspend fun putString(key: String, value: String?): String + + /** + * Retrieve a byte[] from disk. + * + * @param key to read from + * @return value if present, empty array if absent + */ + suspend fun get(key: String): ByteArray + + /** + * Stores a byte[] on disk. + * + * @param key to store to + * @param value to store + */ + suspend fun put(key: String, value: ByteArray?): ByteArray + + /** + * Removes a key from memory & disk. + * + * @param key to remove + * @return when complete + */ + suspend fun remove(key: String) + + /** + * Determine if a key exists in storage. + * + * @param key to check + * @return if key is set + */ + suspend fun contains(key: String): Boolean + + /** Delete all keys in this direct namespace. */ + suspend fun clear() + + /** + * Recursively delete all keys in this scope and child scopes. Fails all outstanding operations on + * the stores. + */ + @Beta + suspend fun deleteAllNow() + + /** Fails all outstanding operations then releases the memory cache. */ + override fun close() +} + +/** + * Extension functions to provide ListenableFuture compatibility for Java callers + */ +fun SimpleStore.getStringFuture(key: String): ListenableFuture { + return kotlinx.coroutines.guava.asListenableFuture( + kotlinx.coroutines.GlobalScope.async { getString(key) } + ) +} + +fun SimpleStore.putStringFuture(key: String, value: String?): ListenableFuture { + return kotlinx.coroutines.guava.asListenableFuture( + kotlinx.coroutines.GlobalScope.async { putString(key, value) } + ) +} + +fun SimpleStore.getFuture(key: String): ListenableFuture { + return kotlinx.coroutines.guava.asListenableFuture( + kotlinx.coroutines.GlobalScope.async { get(key) } + ) +} + +fun SimpleStore.putFuture(key: String, value: ByteArray?): ListenableFuture { + return kotlinx.coroutines.guava.asListenableFuture( + kotlinx.coroutines.GlobalScope.async { put(key, value) } + ) +} + +fun SimpleStore.removeFuture(key: String): ListenableFuture { + return kotlinx.coroutines.guava.asListenableFuture( + kotlinx.coroutines.GlobalScope.async { + remove(key) + null + } + ) +} + +fun SimpleStore.containsFuture(key: String): ListenableFuture { + return kotlinx.coroutines.guava.asListenableFuture( + kotlinx.coroutines.GlobalScope.async { contains(key) } + ) +} + +fun SimpleStore.clearFuture(): ListenableFuture { + return kotlinx.coroutines.guava.asListenableFuture( + kotlinx.coroutines.GlobalScope.async { + clear() + null + } + ) +} + +fun SimpleStore.deleteAllNowFuture(): ListenableFuture { + return kotlinx.coroutines.guava.asListenableFuture( + kotlinx.coroutines.GlobalScope.async { + deleteAllNow() + null + } + ) +} \ No newline at end of file diff --git a/simplestore/src/main/java/com/uber/simplestore/SimpleStoreConfig.kt b/simplestore/src/main/java/com/uber/simplestore/SimpleStoreConfig.kt new file mode 100644 index 0000000..c4adb06 --- /dev/null +++ b/simplestore/src/main/java/com/uber/simplestore/SimpleStoreConfig.kt @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2019. Uber Technologies + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.uber.simplestore + +import com.uber.simplestore.executors.StorageExecutors +import java.util.concurrent.Executor + +/** + * Configure executors used by SimpleStore. + * + * Set may only be called once, and should be called before any use of stores. + */ +object SimpleStoreConfig { + + private val writeLock = Any() + + @Volatile + private var ioExecutor: Executor? = null + + @Volatile + private var computationExecutor: Executor? = null + + fun getIOExecutor(): Executor { + if (ioExecutor == null) { + synchronized(writeLock) { + if (ioExecutor == null) { + ioExecutor = StorageExecutors.ioExecutor() + } + } + } + return ioExecutor!! + } + + /** + * Override the executor used for IO operations. + * + * @param executor to set, null unsets. + */ + fun setIOExecutor(executor: Executor?) { + synchronized(writeLock) { + ioExecutor = executor + } + } + + fun getComputationExecutor(): Executor { + if (computationExecutor == null) { + synchronized(writeLock) { + if (computationExecutor == null) { + computationExecutor = StorageExecutors.computationExecutor() + } + } + } + return computationExecutor!! + } + + /** + * Override the executor used for computation. + * + * @param executor to set, null unsets. + */ + fun setComputationExecutor(executor: Executor?) { + synchronized(writeLock) { + computationExecutor = executor + } + } +} \ No newline at end of file diff --git a/simplestore/src/main/java/com/uber/simplestore/SimpleStoreHelpers.kt b/simplestore/src/main/java/com/uber/simplestore/SimpleStoreHelpers.kt new file mode 100644 index 0000000..e9b85ec --- /dev/null +++ b/simplestore/src/main/java/com/uber/simplestore/SimpleStoreHelpers.kt @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2019. Uber Technologies + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.uber.simplestore + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope + +/** Useful wrappers for common storage operations. */ +object SimpleStoreHelpers { + + /** + * Prefetch specified keys into the memory cache. + * + * @param store to warm + * @param keys to fetch + */ + suspend fun prefetch(store: SimpleStore, vararg keys: String) { + coroutineScope { + keys.map { key -> + async { store.get(key) } + }.awaitAll() + } + } + + /** + * Java-compatible version that returns ListenableFuture + */ + fun prefetchFuture(store: SimpleStore, vararg keys: String) = + kotlinx.coroutines.guava.asListenableFuture( + kotlinx.coroutines.GlobalScope.async { prefetch(store, *keys) } + ) +} \ No newline at end of file diff --git a/simplestore/src/main/java/com/uber/simplestore/StoreClosedException.kt b/simplestore/src/main/java/com/uber/simplestore/StoreClosedException.kt new file mode 100644 index 0000000..12fdf8f --- /dev/null +++ b/simplestore/src/main/java/com/uber/simplestore/StoreClosedException.kt @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2019. Uber Technologies + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.uber.simplestore + +/** Thrown when attempting to use a closed store. */ +class StoreClosedException : RuntimeException { + constructor() : super() + constructor(message: String) : super(message) +} \ No newline at end of file diff --git a/simplestore/src/main/java/com/uber/simplestore/executors/StorageExecutors.kt b/simplestore/src/main/java/com/uber/simplestore/executors/StorageExecutors.kt new file mode 100644 index 0000000..775ef4c --- /dev/null +++ b/simplestore/src/main/java/com/uber/simplestore/executors/StorageExecutors.kt @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2019. Uber Technologies + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.uber.simplestore.executors + +import android.os.Handler +import android.os.Looper +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.asCoroutineDispatcher +import java.util.concurrent.Executor +import java.util.concurrent.Executors + +/** Like MoreExecutors, but no Guava. */ +object StorageExecutors { + + private val MAIN_EXECUTOR = MainThreadExecutor() + + private var ioThreadCount = 0 + private val IO_EXECUTOR = Executors.newCachedThreadPool { r -> + Thread(r, "SimpleStoreIO-${ioThreadCount++}") + } + + private var compThreadCount = 0 + private val COMPUTATION_EXECUTOR = Executors.newFixedThreadPool(2) { r -> + Thread(r, "SimpleStoreComp-${compThreadCount++}") + } + + fun mainExecutor(): Executor = MAIN_EXECUTOR + + fun computationExecutor(): Executor = COMPUTATION_EXECUTOR + + fun ioExecutor(): Executor = IO_EXECUTOR + + // Coroutine dispatchers for modern Kotlin usage + fun mainDispatcher() = Dispatchers.Main + + fun computationDispatcher() = COMPUTATION_EXECUTOR.asCoroutineDispatcher() + + fun ioDispatcher() = IO_EXECUTOR.asCoroutineDispatcher() + + private class MainThreadExecutor : Executor { + private val handler = Handler(Looper.getMainLooper()) + + override fun execute(r: Runnable) { + handler.post(r) + } + } +} \ No newline at end of file diff --git a/simplestore/src/main/java/com/uber/simplestore/impl/AndroidDirectoryProvider.kt b/simplestore/src/main/java/com/uber/simplestore/impl/AndroidDirectoryProvider.kt new file mode 100644 index 0000000..b9820e7 --- /dev/null +++ b/simplestore/src/main/java/com/uber/simplestore/impl/AndroidDirectoryProvider.kt @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2020. Uber Technologies + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.uber.simplestore.impl + +import android.content.Context +import com.uber.simplestore.DirectoryProvider +import java.io.File + +class AndroidDirectoryProvider(private val context: Context) : DirectoryProvider { + + override fun cacheDirectoryPath(): File = context.cacheDir + + override fun filesDirectoryPath(): File = context.filesDir +} \ No newline at end of file diff --git a/simplestore/src/main/java/com/uber/simplestore/impl/SimpleStoreFactory.kt b/simplestore/src/main/java/com/uber/simplestore/impl/SimpleStoreFactory.kt new file mode 100644 index 0000000..793df9c --- /dev/null +++ b/simplestore/src/main/java/com/uber/simplestore/impl/SimpleStoreFactory.kt @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2019. Uber Technologies + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.uber.simplestore.impl + +import com.google.common.annotations.VisibleForTesting +import com.uber.simplestore.DirectoryProvider +import com.uber.simplestore.NamespaceConfig +import com.uber.simplestore.SimpleStore +import com.uber.simplestore.StoreClosedException + +/** + * Obtain a SimpleStore interface that can read and write into a namespace. + * + * Only one instance per namespace may exist at any time to guarentee FIFO-ordering within the + * namespace. A namespace is a set of /-delimited strings that refer to a logical location on disk. + * It is recommended a random UUID be used to both prevent collisions and to obfuscate the contents + * on disk from rooted users. + */ +object SimpleStoreFactory { + + private val namespacesLock = Any() + private val namespaces = mutableMapOf() + + /** + * Obtain a store for a namespace with default configuration. + * + * @param directoryProvider to store the files in + * @param namespace forward-slash delimited logical address + * @return open store + */ + fun create(directoryProvider: DirectoryProvider, namespace: String): SimpleStore { + return create(directoryProvider, namespace, NamespaceConfig.DEFAULT) + } + + /** + * Obtain a store for a namespace. + * + * @param directoryProvider to store the files in + * @param namespace forward-slash delimited logical address + * @param config to use + * @return open store + */ + fun create( + directoryProvider: DirectoryProvider, + namespace: String, + config: NamespaceConfig + ): SimpleStore { + return synchronized(namespacesLock) { + val store = namespaces[namespace] + when { + store != null -> { + if (!store.openIfClosed()) { + // Never let two references be issued. + throw IllegalStateException("namespace '$namespace' already open") + } + store + } + else -> { + val newStore = SimpleStoreImpl(directoryProvider, namespace, config) + namespaces[namespace] = newStore + newStore + } + } + } + } + + fun tombstone(store: SimpleStoreImpl) { + synchronized(namespacesLock) { + if (store.tombstone()) { + namespaces.remove(store.getNamespace()) + } + } + } + + fun flushAndClearRecursive(store: SimpleStoreImpl) { + synchronized(namespacesLock) { + store.failQueueThenRun(StoreClosedException("deleteAllNow")) { + store.clearCache() + } + val children = getOpenChildren(store.getNamespace()) + children.forEach { child -> + child.failQueueThenRun(StoreClosedException("parent deleteAllNow")) { + child.clearCache() + } + } + store.moveAway() + } + } + + @VisibleForTesting + fun getOpenChildren(scope: String): List { + return synchronized(namespacesLock) { + namespaces.entries + .filter { (key, _) -> key.startsWith(scope) && key != scope } + .map { it.value } + .toList() + } + } + + @VisibleForTesting + fun crashIfAnyOpen() { + synchronized(namespacesLock) { + namespaces.forEach { (key, value) -> + if (value.available.get() == 0) { + throw IllegalStateException("Leaked namespace $key") + } + } + } + } +} \ No newline at end of file diff --git a/simplestore/src/main/java/com/uber/simplestore/impl/SimpleStoreImpl.kt b/simplestore/src/main/java/com/uber/simplestore/impl/SimpleStoreImpl.kt new file mode 100644 index 0000000..289ca52 --- /dev/null +++ b/simplestore/src/main/java/com/uber/simplestore/impl/SimpleStoreImpl.kt @@ -0,0 +1,265 @@ +/* + * Copyright (C) 2019. Uber Technologies + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.uber.simplestore.impl + +import android.util.Log +import com.google.common.annotations.VisibleForTesting +import com.uber.simplestore.DirectoryProvider +import com.uber.simplestore.NamespaceConfig +import com.uber.simplestore.SimpleStore +import com.uber.simplestore.SimpleStoreConfig +import com.uber.simplestore.StoreClosedException +import com.uber.simplestore.executors.StorageExecutors +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.withContext +import java.io.File +import java.io.FileOutputStream +import java.io.IOException +import java.nio.charset.Charset +import java.nio.charset.StandardCharsets +import java.util.concurrent.Executor +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicReference + +/** Asynchronous storage implementation. */ +internal class SimpleStoreImpl( + directoryProvider: DirectoryProvider, + private val namespace: String, + config: NamespaceConfig +) : SimpleStore { + + companion object { + private const val OPEN = 0 + private const val CLOSED = 1 + private const val TOMBSTONED = 2 + private val EMPTY_BYTES = ByteArray(0) + private val STRING_ENCODING: Charset = StandardCharsets.UTF_16BE + } + + private var namespacedDirectory: File? = null + private val available = AtomicInteger(OPEN) + + // Only touch from the serial executor. + private val cache = mutableMapOf() + private val orderedIoExecutor: Executor = StorageExecutors.ioExecutor() + private val flush = AtomicReference(null) + + init { + orderedIoExecutor.execute { + val directory = when (config) { + is NamespaceConfig.CACHE -> directoryProvider.cacheDirectoryPath() + else -> directoryProvider.filesDirectoryPath() + } + namespacedDirectory = File("${directory.absolutePath}/simplestore/$namespace") + namespacedDirectory?.mkdirs() + } + } + + override suspend fun getString(key: String): String { + val bytes = get(key) + return if (bytes.isNotEmpty()) { + String(bytes, STRING_ENCODING) + } else { + "" + } + } + + override suspend fun putString(key: String, value: String?): String { + val bytes = when { + value.isNullOrEmpty() -> null + else -> value.toByteArray(STRING_ENCODING) + } + put(key, bytes) + return value ?: "" + } + + override suspend fun get(key: String): ByteArray { + requireOpen() + return withContext(Dispatchers.IO) { + val isDead = isDead() + if (isDead != null) { + throw isDead + } + + val value = cache[key] ?: run { + try { + readFile(key)?.let { bytes -> + if (bytes.isEmpty()) EMPTY_BYTES else bytes + } ?: EMPTY_BYTES + } catch (e: IOException) { + throw e + }.also { bytes -> + cache[key] = bytes + } + } + value + } + } + + override suspend fun put(key: String, value: ByteArray?): ByteArray { + requireOpen() + return withContext(Dispatchers.IO) { + val isDead = isDead() + if (isDead != null) { + throw isDead + } + + when { + value.isNullOrEmpty() -> { + cache[key] = EMPTY_BYTES + deleteFile(key) + EMPTY_BYTES + } + else -> { + cache[key] = value + try { + writeFile(key, value) + value + } catch (e: IOException) { + throw e + } + } + } + } + } + + override suspend fun contains(key: String): Boolean { + requireOpen() + val value = get(key) + return value.isNotEmpty() + } + + override suspend fun remove(key: String) { + put(key, null) + } + + override suspend fun clear() { + requireOpen() + withContext(Dispatchers.IO) { + val isDead = isDead() + if (isDead != null) { + throw isDead + } + + try { + namespacedDirectory?.listFiles { it.isFile }?.forEach { file -> + file.delete() + } + namespacedDirectory?.delete() + cache.clear() + } catch (e: Exception) { + throw e + } + } + } + + override suspend fun deleteAllNow() { + SimpleStoreFactory.flushAndClearRecursive(this) + + withContext(Dispatchers.IO) { + namespacedDirectory?.let { recursiveDelete(it) } + } + } + + override fun close() { + if (available.compareAndSet(OPEN, CLOSED)) { + orderedIoExecutor.execute { SimpleStoreFactory.tombstone(this) } + } + } + + /** Only call from the orderedIoExecutor. */ + fun clearCache() { + cache.clear() + } + + /** + * Cause all items in the queue to fail out, then run something before enabling the queue again + */ + fun failQueueThenRun(exception: Exception, runnable: Runnable) { + if (!flush.compareAndSet(null, exception)) { + throw IllegalStateException() + } + orderedIoExecutor.execute { + runnable.run() + flush.set(null) + } + } + + fun moveAway() { + val currentDir = namespacedDirectory ?: return + val newLocation = File("${currentDir.absolutePath}.bak") + if (!currentDir.renameTo(newLocation)) { + Log.e(javaClass.name, "moveAway rename failed") + return + } + namespacedDirectory = newLocation + } + + private fun recursiveDelete(directory: File) { + directory.listFiles()?.forEach { file -> + recursiveDelete(file) + } + directory.delete() + } + + private fun requireOpen() { + if (available.get() > OPEN) { + throw StoreClosedException() + } + } + + @VisibleForTesting + fun getOrderedExecutor(): Executor = orderedIoExecutor + + fun tombstone(): Boolean = available.compareAndSet(CLOSED, TOMBSTONED) + + fun getNamespace(): String = namespace + + fun openIfClosed(): Boolean = available.compareAndSet(CLOSED, OPEN) + + private fun isDead(): Exception? { + return when { + available.get() > CLOSED -> StoreClosedException() + else -> flush.get() + } + } + + private fun deleteFile(key: String) { + val baseFile = File(namespacedDirectory, key) + val file = AtomicFile(baseFile) + file.delete() + } + + private fun readFile(key: String): ByteArray? { + val baseFile = File(namespacedDirectory, key) + val file = AtomicFile(baseFile) + return if (baseFile.exists()) { + file.readFully() + } else { + null + } + } + + private fun writeFile(key: String, value: ByteArray) { + val baseFile = File(namespacedDirectory, key) + val file = AtomicFile(baseFile) + val writer = file.startWrite() + writer.write(value) + file.finishWrite(writer) + } +} \ No newline at end of file diff --git a/simplestore/src/main/java/com/uber/simplestore/primitive/PrimitiveSimpleStore.kt b/simplestore/src/main/java/com/uber/simplestore/primitive/PrimitiveSimpleStore.kt new file mode 100644 index 0000000..c9d8b77 --- /dev/null +++ b/simplestore/src/main/java/com/uber/simplestore/primitive/PrimitiveSimpleStore.kt @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2019. Uber Technologies + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.uber.simplestore.primitive + +import com.google.common.util.concurrent.ListenableFuture +import com.uber.simplestore.SimpleStore + +/** + * Store primitives on disk. + * + * All methods never return null in the suspend functions, [SimpleStore.contains] should be used for optionality. + * If the value is not set, the 0-byte primitive will be returned. + */ +interface PrimitiveSimpleStore : SimpleStore { + + suspend fun getInt(key: String): Int + + suspend fun put(key: String, value: Int): Int + + suspend fun getLong(key: String): Long + + suspend fun put(key: String, value: Long): Long + + suspend fun getBoolean(key: String): Boolean + + suspend fun put(key: String, value: Boolean): Boolean + + suspend fun getDouble(key: String): Double + + suspend fun put(key: String, value: Double): Double + + /** + * Retrieves a [java.nio.charset.StandardCharsets.UTF_16BE] string. + * + * @param key to fetch from + * @return value if present, otherwise "" + */ + override suspend fun getString(key: String): String + + /** + * Store string as [java.nio.charset.StandardCharsets.UTF_16BE]. + * + * Putting "" will remove the value from disk. + * + * @param key name + * @param value to store + * @return stored value + */ + suspend fun put(key: String, value: String): String + + override suspend fun remove(key: String) +} + +/** + * Extension functions to provide ListenableFuture compatibility for Java callers + */ +fun PrimitiveSimpleStore.getIntFuture(key: String): ListenableFuture { + return kotlinx.coroutines.guava.asListenableFuture( + kotlinx.coroutines.GlobalScope.async { getInt(key) } + ) +} + +fun PrimitiveSimpleStore.putIntFuture(key: String, value: Int): ListenableFuture { + return kotlinx.coroutines.guava.asListenableFuture( + kotlinx.coroutines.GlobalScope.async { put(key, value) } + ) +} + +fun PrimitiveSimpleStore.getLongFuture(key: String): ListenableFuture { + return kotlinx.coroutines.guava.asListenableFuture( + kotlinx.coroutines.GlobalScope.async { getLong(key) } + ) +} + +fun PrimitiveSimpleStore.putLongFuture(key: String, value: Long): ListenableFuture { + return kotlinx.coroutines.guava.asListenableFuture( + kotlinx.coroutines.GlobalScope.async { put(key, value) } + ) +} + +fun PrimitiveSimpleStore.getBooleanFuture(key: String): ListenableFuture { + return kotlinx.coroutines.guava.asListenableFuture( + kotlinx.coroutines.GlobalScope.async { getBoolean(key) } + ) +} + +fun PrimitiveSimpleStore.putBooleanFuture(key: String, value: Boolean): ListenableFuture { + return kotlinx.coroutines.guava.asListenableFuture( + kotlinx.coroutines.GlobalScope.async { put(key, value) } + ) +} + +fun PrimitiveSimpleStore.getDoubleFuture(key: String): ListenableFuture { + return kotlinx.coroutines.guava.asListenableFuture( + kotlinx.coroutines.GlobalScope.async { getDouble(key) } + ) +} + +fun PrimitiveSimpleStore.putDoubleFuture(key: String, value: Double): ListenableFuture { + return kotlinx.coroutines.guava.asListenableFuture( + kotlinx.coroutines.GlobalScope.async { put(key, value) } + ) +} + +fun PrimitiveSimpleStore.putStringFuture(key: String, value: String): ListenableFuture { + return kotlinx.coroutines.guava.asListenableFuture( + kotlinx.coroutines.GlobalScope.async { put(key, value) } + ) +} \ No newline at end of file diff --git a/simplestore/src/main/java/com/uber/simplestore/primitive/PrimitiveSimpleStoreFactory.kt b/simplestore/src/main/java/com/uber/simplestore/primitive/PrimitiveSimpleStoreFactory.kt new file mode 100644 index 0000000..677cdf4 --- /dev/null +++ b/simplestore/src/main/java/com/uber/simplestore/primitive/PrimitiveSimpleStoreFactory.kt @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2019. Uber Technologies + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.uber.simplestore.primitive + +import com.uber.simplestore.DirectoryProvider +import com.uber.simplestore.NamespaceConfig +import com.uber.simplestore.impl.SimpleStoreFactory + +object PrimitiveSimpleStoreFactory { + fun create(directoryProvider: DirectoryProvider, namespace: String): PrimitiveSimpleStore { + return create(directoryProvider, namespace, NamespaceConfig.DEFAULT) + } + + fun create( + directoryProvider: DirectoryProvider, + namespace: String, + config: NamespaceConfig + ): PrimitiveSimpleStore { + return PrimitiveSimpleStoreImpl( + SimpleStoreFactory.create(directoryProvider, namespace, config) + ) + } +} \ No newline at end of file diff --git a/simplestore/src/main/java/com/uber/simplestore/primitive/PrimitiveSimpleStoreImpl.kt b/simplestore/src/main/java/com/uber/simplestore/primitive/PrimitiveSimpleStoreImpl.kt new file mode 100644 index 0000000..1bdd6be --- /dev/null +++ b/simplestore/src/main/java/com/uber/simplestore/primitive/PrimitiveSimpleStoreImpl.kt @@ -0,0 +1,153 @@ +/* + * Copyright (C) 2019. Uber Technologies + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.uber.simplestore.primitive + +import com.uber.simplestore.SimpleStore + +internal class PrimitiveSimpleStoreImpl( + private val simpleStore: SimpleStore +) : PrimitiveSimpleStore { + + override suspend fun getString(key: String): String { + val value = simpleStore.getString(key) + return value.ifEmpty { "" } + } + + override suspend fun put(key: String, value: String): String { + return simpleStore.putString(key, value) + } + + override suspend fun putString(key: String, value: String?): String { + return simpleStore.putString(key, value) + } + + override suspend fun get(key: String): ByteArray { + return simpleStore.get(key) + } + + override suspend fun put(key: String, value: ByteArray?): ByteArray { + return simpleStore.put(key, value) + } + + override suspend fun contains(key: String): Boolean { + return simpleStore.contains(key) + } + + override suspend fun clear() { + simpleStore.clear() + } + + override suspend fun deleteAllNow() { + simpleStore.deleteAllNow() + } + + override fun close() { + simpleStore.close() + } + + override suspend fun getInt(key: String): Int { + val bytes = get(key) + return when { + bytes.isEmpty() || bytes.size != 4 -> 0 + else -> { + // decode big endian + bytes[0].toInt() shl 24 or + (bytes[1].toInt() and 0xFF) shl 16 or + (bytes[2].toInt() and 0xFF) shl 8 or + (bytes[3].toInt() and 0xFF) + } + } + } + + override suspend fun put(key: String, value: Int): Int { + val bytes = if (value != 0) { + // encode big endian + byteArrayOf( + (value shr 24).toByte(), + (value shr 16).toByte(), + (value shr 8).toByte(), + value.toByte() + ) + } else { + null + } + put(key, bytes) + return value + } + + override suspend fun getLong(key: String): Long { + val bytes = get(key) + return when { + bytes.isEmpty() || bytes.size != 8 -> 0L + else -> { + (bytes[0].toLong() and 0xFF) shl 56 or + (bytes[1].toLong() and 0xFF) shl 48 or + (bytes[2].toLong() and 0xFF) shl 40 or + (bytes[3].toLong() and 0xFF) shl 32 or + (bytes[4].toLong() and 0xFF) shl 24 or + (bytes[5].toLong() and 0xFF) shl 16 or + (bytes[6].toLong() and 0xFF) shl 8 or + (bytes[7].toLong() and 0xFF) + } + } + } + + override suspend fun put(key: String, value: Long): Long { + val bytes = if (value != 0L) { + val bytes = ByteArray(8) + var v = value + // encode big endian + for (i in 7 downTo 0) { + bytes[i] = (v and 0xffL).toByte() + v = v shr 8 + } + bytes + } else { + null + } + put(key, bytes) + return value + } + + override suspend fun getBoolean(key: String): Boolean { + val bytes = get(key) + return bytes.isNotEmpty() && bytes[0] > 0 + } + + override suspend fun put(key: String, value: Boolean): Boolean { + val bytes = if (value) { + byteArrayOf(1) + } else { + byteArrayOf(0) + } + put(key, bytes) + return value + } + + override suspend fun getDouble(key: String): Double { + val longValue = getLong(key) + return Double.fromBits(longValue) + } + + override suspend fun put(key: String, value: Double): Double { + put(key, value.toBits()) + return value + } + + override suspend fun remove(key: String) { + simpleStore.remove(key) + } +} \ No newline at end of file