Skip to content

aws: support multiple subnets to handle capacity issues #92

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
emilyalbini wants to merge 1 commit into
base: main
Choose a base branch
from
Open

aws: support multiple subnets to handle capacity issues #92

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 84 additions & 60 deletions factory/aws/src/aws.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ use std::time::{Duration, UNIX_EPOCH};
use std::{collections::HashMap, time::SystemTime};

use anyhow::{anyhow, bail, Result};
use aws_sdk_ec2::error::ProvideErrorMetadata as _;
use aws_sdk_ec2::types::{
BlockDeviceMapping, EbsBlockDevice, Filter,
InstanceNetworkInterfaceSpecification, InstanceType, ResourceType, Tag,
Expand Down Expand Up @@ -134,66 +135,89 @@ async fn create_instance(
let script = base64::engine::general_purpose::STANDARD.encode(&script);

info!(log, "creating an instance (worker {})...", id);
let res = ec2
.run_instances()
.image_id(&target.ami)
.instance_type(InstanceType::from_str(&target.instance_type)?)
.key_name(&config.aws.key)
.min_count(1)
.max_count(1)
.tag_specifications(
TagSpecification::builder()
.resource_type(ResourceType::Instance)
.tags(
Tag::builder().key("Name").value(format!("w-{id}")).build(),
)
.tags(
Tag::builder()
.key(&config.aws.tag)
.value("1".to_string())
.build(),
)
.tags(
Tag::builder()
.key(config.aws.tagkey_worker())
.value(&id)
.build(),
)
.tags(
Tag::builder()
.key(config.aws.tagkey_lease())
.value(lease_id)
.build(),
)
.build(),
)
.block_device_mappings(
BlockDeviceMapping::builder()
.device_name("/dev/sda1")
.ebs(
EbsBlockDevice::builder()
.volume_size(target.root_size_gb)
.build(),
)
.build(),
)
.network_interfaces(
InstanceNetworkInterfaceSpecification::builder()
.subnet_id(&config.aws.subnet)
.device_index(0)
.associate_public_ip_address(false)
.groups(&config.aws.security_group)
.build(),
)
.user_data(script)
.send()
.await?;

let mut instances = res
.instances()
.into_iter()
.map(|i| Instance::from((i, config.aws.tag.as_str())))
.collect::<Vec<_>>();
let mut instances = Vec::new();
/*
* We occasionally noticed AWS running out of the instance type we use in
* the Availability Zone our subnet is created in. Falling back to other
* subnets allows us to mitigate the issue.
*/
for subnet in config.aws.subnet.as_slice() {
let res = ec2
.run_instances()
.image_id(&target.ami)
.instance_type(InstanceType::from_str(&target.instance_type)?)
.key_name(&config.aws.key)
.min_count(1)
.max_count(1)
.tag_specifications(
TagSpecification::builder()
.resource_type(ResourceType::Instance)
.tags(
Tag::builder()
.key("Name")
.value(format!("w-{id}"))
.build(),
)
.tags(
Tag::builder()
.key(&config.aws.tag)
.value("1".to_string())
.build(),
)
.tags(
Tag::builder()
.key(config.aws.tagkey_worker())
.value(&id)
.build(),
)
.tags(
Tag::builder()
.key(config.aws.tagkey_lease())
.value(lease_id)
.build(),
)
.build(),
)
.block_device_mappings(
BlockDeviceMapping::builder()
.device_name("/dev/sda1")
.ebs(
EbsBlockDevice::builder()
.volume_size(target.root_size_gb)
.build(),
)
.build(),
)
.network_interfaces(
InstanceNetworkInterfaceSpecification::builder()
.subnet_id(subnet)
.device_index(0)
.associate_public_ip_address(false)
.groups(&config.aws.security_group)
.build(),
)
.user_data(&script)
.send()
.await;
match res {
Ok(res) => {
instances = res
.instances()
.into_iter()
.map(|i| Instance::from((i, config.aws.tag.as_str())))
.collect();
break;
}
Err(e) if e.code() == Some("InsufficientInstanceCapacity") => {
warn!(
log,
"AWS ran out of instance type {:?} in subnet {subnet:?}",
target.instance_type
);
}
Err(e) => return Err(e.into()),
}
}

if instances.len() != 1 {
bail!("wanted one instance, got {instances:?}");
Expand Down
20 changes: 19 additions & 1 deletion factory/aws/src/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ pub(crate) struct ConfigFileAws {
pub profile: Option<String>,
pub region: String,
pub vpc: String,
pub subnet: String,
pub subnet: ConfigFileAwsSubnets,
pub tag: String,
pub key: String,
pub security_group: String,
Expand All @@ -64,3 +64,21 @@ impl ConfigFileAws {
format!("{}-lease_id", self.tag)
}
}

#[derive(Deserialize, Debug)]
#[serde(untagged)]
pub enum ConfigFileAwsSubnets {
Single(String),
Multiple(Vec<String>),
}

impl ConfigFileAwsSubnets {
pub fn as_slice(&self) -> &[String] {
match self {
ConfigFileAwsSubnets::Single(subnet) => {
std::slice::from_ref(subnet)
}
ConfigFileAwsSubnets::Multiple(subnets) => subnets,
}
}
}