Skip to content

Ignore empty-set environment values for config #2639

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AJIOB wants to merge 1 commit into
base: main
Choose a base branch
from
+81 −57
Open

Ignore empty-set environment values for config #2639

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
138 changes: 81 additions & 57 deletions src/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -695,9 +695,12 @@ pub struct EnvConfig {
basedirs: Option<Vec<String>>,
}

fn string_from_env_var(env_var_name: &str) -> Option<String> {
env::var(env_var_name).ok().filter(|s| !s.is_empty())
}

fn key_prefix_from_env_var(env_var_name: &str) -> String {
env::var(env_var_name)
.ok()
string_from_env_var(env_var_name)
.as_ref()
.map(|s| s.trim_end_matches('/'))
.filter(|s| !s.is_empty())
Expand All @@ -709,7 +712,7 @@ fn number_from_env_var<A: std::str::FromStr>(env_var_name: &str) -> Option<Resul
where
<A as FromStr>::Err: std::fmt::Debug,
{
let value = env::var(env_var_name).ok()?;
let value = string_from_env_var(env_var_name)?;

value
.parse::<A>()
Expand All @@ -718,8 +721,7 @@ where
}

fn bool_from_env_var(env_var_name: &str) -> Result<Option<bool>> {
env::var(env_var_name)
.ok()
string_from_env_var(env_var_name)
.map(|value| match value.to_lowercase().as_str() {
"true" | "on" | "1" => Ok(true),
"false" | "off" | "0" => Ok(false),
Expand All @@ -733,12 +735,12 @@ fn bool_from_env_var(env_var_name: &str) -> Result<Option<bool>> {

fn config_from_env() -> Result<EnvConfig> {
// ======= AWS =======
let s3 = if let Ok(bucket) = env::var("SCCACHE_BUCKET") {
let region = env::var("SCCACHE_REGION").ok();
let s3 = if let Some(bucket) = string_from_env_var("SCCACHE_BUCKET") {
let region = string_from_env_var("SCCACHE_REGION");
let no_credentials = bool_from_env_var("SCCACHE_S3_NO_CREDENTIALS")?.unwrap_or(false);
let use_ssl = bool_from_env_var("SCCACHE_S3_USE_SSL")?;
let server_side_encryption = bool_from_env_var("SCCACHE_S3_SERVER_SIDE_ENCRYPTION")?;
let endpoint = env::var("SCCACHE_ENDPOINT").ok();
let endpoint = string_from_env_var("SCCACHE_ENDPOINT");
let key_prefix = key_prefix_from_env_var("SCCACHE_S3_KEY_PREFIX");
let enable_virtual_host_style = bool_from_env_var("SCCACHE_S3_ENABLE_VIRTUAL_HOST_STYLE")?;

Expand All @@ -765,18 +767,18 @@ fn config_from_env() -> Result<EnvConfig> {

// ======= redis =======
let redis = match (
env::var("SCCACHE_REDIS").ok(),
env::var("SCCACHE_REDIS_ENDPOINT").ok(),
env::var("SCCACHE_REDIS_CLUSTER_ENDPOINTS").ok(),
string_from_env_var("SCCACHE_REDIS"),
string_from_env_var("SCCACHE_REDIS_ENDPOINT"),
string_from_env_var("SCCACHE_REDIS_CLUSTER_ENDPOINTS"),
) {
(None, None, None) => None,
(url, endpoint, cluster_endpoints) => {
let db = number_from_env_var("SCCACHE_REDIS_DB")
.transpose()?
.unwrap_or(DEFAULT_REDIS_DB);

let username = env::var("SCCACHE_REDIS_USERNAME").ok();
let password = env::var("SCCACHE_REDIS_PASSWORD").ok();
let username = string_from_env_var("SCCACHE_REDIS_USERNAME");
let password = string_from_env_var("SCCACHE_REDIS_PASSWORD");

let ttl = number_from_env_var("SCCACHE_REDIS_EXPIRATION")
.or_else(|| number_from_env_var("SCCACHE_REDIS_TTL"))
Expand Down Expand Up @@ -805,11 +807,11 @@ fn config_from_env() -> Result<EnvConfig> {
}

// ======= memcached =======
let memcached = if let Ok(url) =
env::var("SCCACHE_MEMCACHED").or_else(|_| env::var("SCCACHE_MEMCACHED_ENDPOINT"))
let memcached = if let Some(url) = string_from_env_var("SCCACHE_MEMCACHED")
.or_else(|| string_from_env_var("SCCACHE_MEMCACHED_ENDPOINT"))
{
let username = env::var("SCCACHE_MEMCACHED_USERNAME").ok();
let password = env::var("SCCACHE_MEMCACHED_PASSWORD").ok();
let username = string_from_env_var("SCCACHE_MEMCACHED_USERNAME");
let password = string_from_env_var("SCCACHE_MEMCACHED_PASSWORD");

let expiration = number_from_env_var("SCCACHE_MEMCACHED_EXPIRATION")
.transpose()?
Expand Down Expand Up @@ -837,35 +839,35 @@ fn config_from_env() -> Result<EnvConfig> {
}

// ======= GCP/GCS =======
if (env::var("SCCACHE_GCS_CREDENTIALS_URL").is_ok()
|| env::var("SCCACHE_GCS_OAUTH_URL").is_ok()
|| env::var("SCCACHE_GCS_KEY_PATH").is_ok())
&& env::var("SCCACHE_GCS_BUCKET").is_err()
if (string_from_env_var("SCCACHE_GCS_CREDENTIALS_URL").is_some()
|| string_from_env_var("SCCACHE_GCS_OAUTH_URL").is_some()
|| string_from_env_var("SCCACHE_GCS_KEY_PATH").is_some())
&& string_from_env_var("SCCACHE_GCS_BUCKET").is_none()
{
bail!(
"If setting GCS credentials, SCCACHE_GCS_BUCKET and an auth mechanism need to be set."
);
}

let gcs = env::var("SCCACHE_GCS_BUCKET").ok().map(|bucket| {
let gcs = string_from_env_var("SCCACHE_GCS_BUCKET").map(|bucket| {
let key_prefix = key_prefix_from_env_var("SCCACHE_GCS_KEY_PREFIX");

if env::var("SCCACHE_GCS_OAUTH_URL").is_ok() {
if string_from_env_var("SCCACHE_GCS_OAUTH_URL").is_some() {
eprintln!("SCCACHE_GCS_OAUTH_URL has been deprecated");
eprintln!("if you intend to use vm metadata for auth, please set correct service account instead");
}

let credential_url = env::var("SCCACHE_GCS_CREDENTIALS_URL").ok();
let credential_url = string_from_env_var("SCCACHE_GCS_CREDENTIALS_URL");

let cred_path = env::var("SCCACHE_GCS_KEY_PATH").ok();
let service_account = env::var("SCCACHE_GCS_SERVICE_ACCOUNT").ok();
let cred_path = string_from_env_var("SCCACHE_GCS_KEY_PATH");
let service_account = string_from_env_var("SCCACHE_GCS_SERVICE_ACCOUNT");

let rw_mode = match env::var("SCCACHE_GCS_RW_MODE").as_ref().map(String::as_str) {
Ok("READ_ONLY") => CacheModeConfig::ReadOnly,
Ok("READ_WRITE") => CacheModeConfig::ReadWrite,
let rw_mode = match string_from_env_var("SCCACHE_GCS_RW_MODE").as_deref() {
Some("READ_ONLY") => CacheModeConfig::ReadOnly,
Some("READ_WRITE") => CacheModeConfig::ReadWrite,
// TODO: unsure if these should warn during the configuration loading
// or at the time when they're actually used to connect to GCS
Ok(_) => {
Some(_) => {
warn!("Invalid SCCACHE_GCS_RW_MODE -- defaulting to READ_ONLY.");
CacheModeConfig::ReadOnly
}
Expand All @@ -886,7 +888,7 @@ fn config_from_env() -> Result<EnvConfig> {
});

// ======= GHA =======
let gha = if let Ok(version) = env::var("SCCACHE_GHA_VERSION") {
let gha = if let Some(version) = string_from_env_var("SCCACHE_GHA_VERSION") {
// If SCCACHE_GHA_VERSION has been set, we don't need to check
// SCCACHE_GHA_ENABLED's value anymore.
Some(GHACacheConfig {
Expand All @@ -905,9 +907,9 @@ fn config_from_env() -> Result<EnvConfig> {
};

// ======= Azure =======
let azure = if let (Ok(connection_string), Ok(container)) = (
env::var("SCCACHE_AZURE_CONNECTION_STRING"),
env::var("SCCACHE_AZURE_BLOB_CONTAINER"),
let azure = if let (Some(connection_string), Some(container)) = (
string_from_env_var("SCCACHE_AZURE_CONNECTION_STRING"),
string_from_env_var("SCCACHE_AZURE_BLOB_CONTAINER"),
) {
let key_prefix = key_prefix_from_env_var("SCCACHE_AZURE_KEY_PREFIX");
Some(AzureCacheConfig {
Expand All @@ -920,11 +922,11 @@ fn config_from_env() -> Result<EnvConfig> {
};

// ======= WebDAV =======
let webdav = if let Ok(endpoint) = env::var("SCCACHE_WEBDAV_ENDPOINT") {
let webdav = if let Some(endpoint) = string_from_env_var("SCCACHE_WEBDAV_ENDPOINT") {
let key_prefix = key_prefix_from_env_var("SCCACHE_WEBDAV_KEY_PREFIX");
let username = env::var("SCCACHE_WEBDAV_USERNAME").ok();
let password = env::var("SCCACHE_WEBDAV_PASSWORD").ok();
let token = env::var("SCCACHE_WEBDAV_TOKEN").ok();
let username = string_from_env_var("SCCACHE_WEBDAV_USERNAME");
let password = string_from_env_var("SCCACHE_WEBDAV_PASSWORD");
let token = string_from_env_var("SCCACHE_WEBDAV_TOKEN");

Some(WebdavCacheConfig {
endpoint,
Expand All @@ -938,8 +940,8 @@ fn config_from_env() -> Result<EnvConfig> {
};

// ======= OSS =======
let oss = if let Ok(bucket) = env::var("SCCACHE_OSS_BUCKET") {
let endpoint = env::var("SCCACHE_OSS_ENDPOINT").ok();
let oss = if let Some(bucket) = string_from_env_var("SCCACHE_OSS_BUCKET") {
let endpoint = string_from_env_var("SCCACHE_OSS_ENDPOINT");
let key_prefix = key_prefix_from_env_var("SCCACHE_OSS_KEY_PREFIX");

let no_credentials = bool_from_env_var("SCCACHE_OSS_NO_CREDENTIALS")?.unwrap_or(false);
Expand All @@ -965,8 +967,8 @@ fn config_from_env() -> Result<EnvConfig> {
}

// ======= COS =======
let cos = if let Ok(bucket) = env::var("SCCACHE_COS_BUCKET") {
let endpoint = env::var("SCCACHE_COS_ENDPOINT").ok();
let cos = if let Some(bucket) = string_from_env_var("SCCACHE_COS_BUCKET") {
let endpoint = string_from_env_var("SCCACHE_COS_ENDPOINT");
let key_prefix = key_prefix_from_env_var("SCCACHE_COS_KEY_PREFIX");

Some(COSCacheConfig {
Expand All @@ -980,9 +982,7 @@ fn config_from_env() -> Result<EnvConfig> {

// ======= Local =======
let disk_dir = env::var_os("SCCACHE_DIR").map(PathBuf::from);
let disk_sz = env::var("SCCACHE_CACHE_SIZE")
.ok()
.and_then(|v| parse_size(&v));
let disk_sz = string_from_env_var("SCCACHE_CACHE_SIZE").and_then(|v| parse_size(&v));

let mut preprocessor_mode_config = PreprocessorCacheModeConfig::activated();
let preprocessor_mode_overridden = if let Some(value) = bool_from_env_var("SCCACHE_DIRECT")? {
Expand All @@ -992,18 +992,16 @@ fn config_from_env() -> Result<EnvConfig> {
false
};

let (disk_rw_mode, disk_rw_mode_overridden) = match env::var("SCCACHE_LOCAL_RW_MODE")
.as_ref()
.map(String::as_str)
{
Ok("READ_ONLY") => (CacheModeConfig::ReadOnly, true),
Ok("READ_WRITE") => (CacheModeConfig::ReadWrite, true),
Ok(_) => {
warn!("Invalid SCCACHE_LOCAL_RW_MODE -- defaulting to READ_WRITE.");
(CacheModeConfig::ReadWrite, false)
}
_ => (CacheModeConfig::ReadWrite, false),
};
let (disk_rw_mode, disk_rw_mode_overridden) =
match string_from_env_var("SCCACHE_LOCAL_RW_MODE").as_deref() {
Some("READ_ONLY") => (CacheModeConfig::ReadOnly, true),
Some("READ_WRITE") => (CacheModeConfig::ReadWrite, true),
Some(_) => {
warn!("Invalid SCCACHE_LOCAL_RW_MODE -- defaulting to READ_WRITE.");
(CacheModeConfig::ReadWrite, false)
}
_ => (CacheModeConfig::ReadWrite, false),
};

let any_overridden = disk_dir.is_some()
|| disk_sz.is_some()
Expand Down Expand Up @@ -1421,6 +1419,32 @@ fn test_parse_size() {
assert_eq!(Some(1024 * TEN_GIGS), parse_size("10T"));
}

#[test]
fn test_string_from_env_var() {
let var_name = "TEST_SCCACHE_VAR";
for value in [None, Some(""), Some("foo")] {
match value {
None => unsafe {
std::env::remove_var(var_name);
},
Some(value) => unsafe {
std::env::set_var(var_name, value);
},
}
let result = string_from_env_var(var_name);
unsafe {
std::env::remove_var(var_name);
}

let expected = match value {
None | Some("") => None,
Some(value) => Some(value.to_string()),
};

assert_eq!(result, expected);
}
}

#[test]
fn config_overrides() {
let env_conf = EnvConfig {
Expand Down
Loading