Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -490,7 +490,7 @@ batcher_start: ./crates/batcher/.env user_fund_payment_service
@echo "Starting Batcher..."
@cargo run --manifest-path ./crates/batcher/Cargo.toml --release -- --config ./config-files/config-batcher.yaml --env-file ./crates/batcher/.env

batcher_start_local: user_fund_payment_service ## Start the Batcher locally. It runs LocalStack as S3 service.
batcher_start_local: user_fund_payment_service ## Start the Batcher locally. It runs S3Mock as S3 service.
@echo "Starting Batcher..."
@$(MAKE) storage_start &
@cargo run --manifest-path ./crates/batcher/Cargo.toml --release -- --config ./config-files/config-batcher.yaml --env-file ./crates/batcher/.env.dev
Expand All @@ -500,7 +500,7 @@ batcher_start_local_no_fund:
@$(MAKE) storage_start &
@cargo run --manifest-path ./crates/batcher/Cargo.toml --release -- --config ./config-files/config-batcher.yaml --env-file ./crates/batcher/.env.dev

batcher_start_ethereum_package: user_fund_payment_service ## Start the Batcher with Ethereum package config. It runs LocalStack as S3 service.
batcher_start_ethereum_package: user_fund_payment_service ## Start the Batcher with Ethereum package config. It runs S3Mock as S3 service.
@echo "Starting Batcher..."
@$(MAKE) storage_start &
@cargo run --manifest-path ./crates/batcher/Cargo.toml --release -- --config ./config-files/config-batcher-ethereum-package.yaml --env-file ./crates/batcher/.env.dev
Expand Down
8 changes: 4 additions & 4 deletions crates/batcher/.env.docker
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,16 @@ AWS_SECRET_ACCESS_KEY=test
AWS_REGION=us-east-2
AWS_ACCESS_KEY_ID=test
AWS_BUCKET_NAME=aligned.storage
UPLOAD_ENDPOINT=http://localstack:4566
DOWNLOAD_ENDPOINT=http://localstack:4566/aligned.storage
UPLOAD_ENDPOINT=http://s3mock:9090
DOWNLOAD_ENDPOINT=http://s3mock:9090/aligned.storage

# Secondary S3 bucket configuration
AWS_SECRET_ACCESS_KEY_SECONDARY=test2
AWS_REGION_SECONDARY=us-west-1
AWS_ACCESS_KEY_ID_SECONDARY=test2
AWS_BUCKET_NAME_SECONDARY=aligned.storage
UPLOAD_ENDPOINT_SECONDARY=http://localstack2:4567
DOWNLOAD_ENDPOINT_SECONDARY=http://localstack2:4567/aligned.storage
UPLOAD_ENDPOINT_SECONDARY=http://s3mock2:9090
DOWNLOAD_ENDPOINT_SECONDARY=http://s3mock2:9090/aligned.storage

RUST_LOG=info
RUST_BACKTRACE=1
2 changes: 1 addition & 1 deletion crates/batcher/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ impl Batcher {
pub async fn new(config_file: String) -> Self {
dotenv().ok();

// https://docs.aws.amazon.com/sdk-for-rust/latest/dg/localstack.html
// S3-compatible endpoint (S3Mock for local dev, real S3 in production)
// Primary S3 configuration
let s3_config_primary = s3::S3Config {
access_key_id: env::var("AWS_ACCESS_KEY_ID").ok(),
Expand Down
16 changes: 8 additions & 8 deletions docker-compose.yaml
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
services:

localstack:
s3mock:
extends:
file: storage-docker-compose.yaml
service: localstack
service: s3mock
profiles:
- base

localstack2:
s3mock2:
extends:
file: storage-docker-compose.yaml
service: localstack2
service: s3mock2
profiles:
- base

Expand Down Expand Up @@ -115,14 +115,14 @@ services:
AWS_REGION: us-east-2
AWS_ACCESS_KEY_ID: test
AWS_BUCKET_NAME: aligned.storage
UPLOAD_ENDPOINT: http://localstack:4566
DOWNLOAD_ENDPOINT: http://localstack:4566/aligned.storage
UPLOAD_ENDPOINT: http://s3mock:9090
DOWNLOAD_ENDPOINT: http://s3mock:9090/aligned.storage
AWS_SECRET_ACCESS_KEY_SECONDARY: test2
AWS_REGION_SECONDARY: us-west-1
AWS_ACCESS_KEY_ID_SECONDARY: test2
AWS_BUCKET_NAME_SECONDARY: aligned.storage
UPLOAD_ENDPOINT_SECONDARY: http://localstack2:4567
DOWNLOAD_ENDPOINT_SECONDARY: http://localstack2:4567/aligned.storage.secondary
UPLOAD_ENDPOINT_SECONDARY: http://s3mock2:9090
DOWNLOAD_ENDPOINT_SECONDARY: http://s3mock2:9090/aligned.storage.secondary
RUST_LOG: info
RUST_BACKTRACE: 1
build:
Expand Down
3 changes: 3 additions & 0 deletions docker/batcher.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ FROM lukemathwalker/cargo-chef:latest-rust-1 AS chef

FROM chef AS planner

COPY crates/sdk/Cargo.toml /aligned_layer/crates/sdk/Cargo.toml
COPY crates/sdk/src/lib.rs /aligned_layer/crates/sdk/src/lib.rs

COPY crates/batcher/Cargo.toml /aligned_layer/crates/batcher/Cargo.toml
COPY crates/batcher/src/main.rs /aligned_layer/crates/batcher/src/main.rs
WORKDIR /aligned_layer/crates/batcher/
Expand Down
2 changes: 1 addition & 1 deletion docs/3_guides/6_setup_aligned.md
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ To start the [Batcher](../2_architecture/components/1_batcher.md) locally:
make batcher_start_local
```

This starts a [localstack](https://www.localstack.cloud/) to act as a replacement for S3.
This starts an [S3Mock](https://github.com/adobe/S3Mock) container to act as a replacement for S3.

If you want to use the batcher under a real `S3` connection you'll need to specify the environment variables under `crates/batcher/.env` and then run:

Expand Down
7 changes: 0 additions & 7 deletions scripts/init-s3-main.sh

This file was deleted.

7 changes: 0 additions & 7 deletions scripts/init-s3-secondary.sh

This file was deleted.

32 changes: 0 additions & 32 deletions scripts/init-s3.py

This file was deleted.

38 changes: 12 additions & 26 deletions storage-docker-compose.yaml
Original file line number Diff line number Diff line change
@@ -1,32 +1,18 @@
services:
localstack:
container_name: "${LOCALSTACK_DOCKER_NAME:-localstack-main}"
image: localstack/localstack:s3-latest
s3mock:
container_name: "${S3MOCK_DOCKER_NAME:-s3mock-main}"
image: adobe/s3mock:4.11.0
ports:
- "127.0.0.1:4566:4566" # LocalStack Gateway
- "127.0.0.1:4566:9090"
environment:
- DEBUG=${DEBUG:-0}
- DEFAULT_REGION=us-east-2
- AWS_ACCESS_KEY_ID=test
- AWS_SECRET_ACCESS_KEY=test
volumes:
- "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
- "/var/run/docker.sock:/var/run/docker.sock"
- "./scripts/init-s3.py:/etc/localstack/init/ready.d/init-s3.py" # bucket initialization script
- "./scripts/init-s3-main.sh:/etc/localstack/init/ready.d/init-s3-main.sh" # bucket initialization script
- COM_ADOBE_TESTING_S3MOCK_STORE_INITIAL_BUCKETS=aligned.storage
- COM_ADOBE_TESTING_S3MOCK_STORE_REGION=us-east-2

localstack2:
container_name: "${LOCALSTACK2_DOCKER_NAME:-localstack-secondary}"
image: localstack/localstack:s3-latest
s3mock2:
container_name: "${S3MOCK2_DOCKER_NAME:-s3mock-secondary}"
image: adobe/s3mock:4.11.0
ports:
- "127.0.0.1:4567:4566" # LocalStack Gateway
- "127.0.0.1:4567:9090"
environment:
- DEBUG=${DEBUG:-0}
- DEFAULT_REGION=us-west-1
- AWS_ACCESS_KEY_ID=test2
- AWS_SECRET_ACCESS_KEY=test2
volumes:
- "${LOCALSTACK2_VOLUME_DIR:-./volume2}:/var/lib/localstack"
- "/var/run/docker.sock:/var/run/docker.sock"
- "./scripts/init-s3.py:/etc/localstack/init/ready.d/init-s3.py" # bucket initialization script
- "./scripts/init-s3-secondary.sh:/etc/localstack/init/ready.d/init-s3-secondary.sh" # bucket initialization script
- COM_ADOBE_TESTING_S3MOCK_STORE_INITIAL_BUCKETS=aligned.storage
- COM_ADOBE_TESTING_S3MOCK_STORE_REGION=us-west-1
Loading