diff --git a/.gitignore b/.gitignore index 93c0e7ee..778ca057 100644 --- a/.gitignore +++ b/.gitignore @@ -1,15 +1,27 @@ -build/ +# mkosi/build artifacts + +*.qcow2 build.*/ -mkosi/ +build/ env.json -mkosi.packages/ -mkosi.cache/ mkosi.builddir/ -*.qcow2 -.claudesync/ -.claudeignore -tmp/ -.temp +mkosi.cache/ +mkosi.packages/ +mkosi/ NvVars + +# temporary files + +.temp +tmp/ + +# IDEs/agents/whatnot + +.claudeignore +.claudesync/ .vscode + +# Special files for dev + .bypass-lima +mkosi.profiles/devtools/authorized_keys diff --git a/Makefile b/Makefile index b7394b2a..a7fe081b 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ SHELL := /usr/bin/env bash WRAPPER := scripts/env_wrapper.sh FILE ?= build/latest.efi +SERIAL_CONSOLE ?= false ##@ Help @@ -35,13 +36,17 @@ all: build setup: ## Install dependencies (Linux only) @scripts/setup_deps.sh +preflight: + @$(WRAPPER) echo "Ready to build" + # Build module build: setup ## Build the specified module $(WRAPPER) mkosi --force --image-id $(IMAGE) --include=images/$(IMAGE).conf # Build module with devtools profile +build-dev: SERIAL_CONSOLE_PROFILE := $(if $(filter true,$(SERIAL_CONSOLE)),serial-console,) build-dev: setup ## Build module with development tools - $(WRAPPER) mkosi --force --image-id $(IMAGE)-dev --profile=devtools --include=images/$(IMAGE).conf + $(WRAPPER) mkosi --force --image-id $(IMAGE)-dev --profile=devtools,$(SERIAL_CONSOLE_PROFILE) --include=images/$(IMAGE).conf ##@ Utilities diff --git a/README.md b/README.md index c1513c58..e9f46adb 100644 --- a/README.md +++ b/README.md @@ -47,10 +47,37 @@ make build IMAGE=l2-builder # Build with development tools make build-dev IMAGE=flashbox-l1 +# Build dev image with serial console + password auth enabled +make build-dev IMAGE=flashbox-l1 SERIAL_CONSOLE=true + # View all available targets make help ``` +#### Dev image SSH access + +Dev images (`build-dev`) use SSH authorized keys for root login instead of a password. Before building, place your public key at: + +``` +mkosi.profiles/devtools/authorized_keys +``` + +This file is gitignored, so you can add a personal public key. For example: + +```bash +cp ~/.ssh/id_ed25519.pub mkosi.profiles/devtools/authorized_keys +``` + +Once the image is running, SSH in as root: + +```bash +ssh -p 2222 root@localhost +``` + +#### Serial console + +Pass `SERIAL_CONSOLE=true` to enable the serial console service and password authentication. This also sets a fixed root password (`dqSPjo4p`) for SSH login, so only use this for local development/debugging. + ### Measuring TDX Boot Process **Export TDX measurements** for the built image: @@ -104,7 +131,7 @@ sudo usermod -aG kvm $USER ``` > [!NOTE] -> +> > Depending on your Linux distro, these commands may require changing the > supplied OVMF paths or installing your distro's OVMF package. diff --git a/images/l2-op-rbuilder-bproxy.conf b/images/l2-op-rbuilder-bproxy.conf new file mode 100644 index 00000000..2fb40c71 --- /dev/null +++ b/images/l2-op-rbuilder-bproxy.conf @@ -0,0 +1,11 @@ +[Config] +Profiles=gcp + +[Distribution] +Snapshot=20260301T083349Z + +[Include] +Include=shared/mkosi.conf +Include=modules/l2/_common/mkosi.conf +Include=modules/l2/_gcp/mkosi.conf +Include=modules/l2/op-rbuilder-bproxy/mkosi.conf diff --git a/images/l2-op-rbuilder.conf b/images/l2-op-rbuilder.conf new file mode 100644 index 00000000..6af1a1c3 --- /dev/null +++ b/images/l2-op-rbuilder.conf @@ -0,0 +1,11 @@ +[Config] +Profiles=gcp + +[Distribution] +Snapshot=20260301T083349Z + +[Include] +Include=shared/mkosi.conf +Include=modules/l2/_common/mkosi.conf +Include=modules/l2/_gcp/mkosi.conf +Include=modules/l2/op-rbuilder/mkosi.conf diff --git a/images/l2-simulator.conf b/images/l2-simulator.conf new file mode 100644 index 00000000..080fd1df --- /dev/null +++ b/images/l2-simulator.conf @@ -0,0 +1,11 @@ +[Config] +Profiles=gcp + +[Distribution] +Snapshot=20260301T083349Z + +[Include] +Include=shared/mkosi.conf +Include=modules/l2/_common/mkosi.conf +Include=modules/l2/_gcp/mkosi.conf +Include=modules/l2/simulator/mkosi.conf diff --git a/mkosi.profiles/devtools/mkosi.conf b/mkosi.profiles/devtools/mkosi.conf index c7eb8d59..ca2d38e4 100644 --- a/mkosi.profiles/devtools/mkosi.conf +++ b/mkosi.profiles/devtools/mkosi.conf @@ -1,6 +1,5 @@ [Content] -ExtraTrees=mkosi.extra - custom +ExtraTrees=custom PostInstallationScripts=custom.postinst.d/*.sh Packages=adjtimex @@ -18,6 +17,7 @@ Packages=adjtimex screen socat strace + sudo tcpdump tcpflow vim diff --git a/mkosi.profiles/devtools/mkosi.postinst b/mkosi.profiles/devtools/mkosi.postinst index cc0db73b..ad523e6d 100755 --- a/mkosi.profiles/devtools/mkosi.postinst +++ b/mkosi.profiles/devtools/mkosi.postinst @@ -1,21 +1,22 @@ #!/bin/bash -set -euxo pipefail -# Deterministically set root password -PASSWORD="dqSPjo4p" -HASH=$(mkosi-chroot openssl passwd -6 -salt salt "$PASSWORD") -mkosi-chroot passwd -u root -mkosi-chroot usermod -p "$HASH" root +set -euxo pipefail # Remove git files in custom/ folder mkosi-chroot rm /.gitignore /.gitkeep || true +if [ -f "$SRCDIR/mkosi.profiles/devtools/authorized_keys" ]; then + mkdir -p "$BUILDROOT/root/.ssh" + cp "$SRCDIR/mkosi.profiles/devtools/authorized_keys" "$BUILDROOT/root/.ssh/authorized_keys" + chmod 600 "$BUILDROOT/root/.ssh/authorized_keys" + chmod 700 "$BUILDROOT/root/.ssh" +fi + if [ -f "$BUILDROOT/etc/default/dropbear" ]; then # Remove -s, -w, -g flags from dropbear args sed -i '/^DROPBEAR_EXTRA_ARGS=/s/-[swg] \?//g' "$BUILDROOT/etc/default/dropbear" else echo "PermitRootLogin yes" >> "$BUILDROOT/etc/ssh/sshd_config" - echo "PasswordAuthentication yes" >> "$BUILDROOT/etc/ssh/sshd_config" - mkosi-chroot systemctl unmask ssh.service ssh.socket - mkosi-chroot systemctl add-wants minimal.target ssh.service + mkosi-chroot systemctl unmask ssh.service ssh.socket systemd-user-sessions.service + mkosi-chroot systemctl add-wants minimal.target ssh.service systemd-user-sessions.service fi diff --git a/mkosi.profiles/serial-console/mkosi.conf b/mkosi.profiles/serial-console/mkosi.conf new file mode 100644 index 00000000..e0779a78 --- /dev/null +++ b/mkosi.profiles/serial-console/mkosi.conf @@ -0,0 +1,2 @@ +[Content] +ExtraTrees=serial-console.service:/usr/lib/systemd/system/serial-console.service diff --git a/mkosi.profiles/serial-console/mkosi.postinst b/mkosi.profiles/serial-console/mkosi.postinst new file mode 100755 index 00000000..ac935593 --- /dev/null +++ b/mkosi.profiles/serial-console/mkosi.postinst @@ -0,0 +1,16 @@ +#!/bin/bash + +set -euxo pipefail + +# Enable console service +mkosi-chroot systemctl unmask serial-console.service +mkosi-chroot systemctl add-wants minimal.target serial-console.service + +# Deterministically set root password +PASSWORD="dqSPjo4p" +HASH=$(mkosi-chroot openssl passwd -6 -salt salt "$PASSWORD") +mkosi-chroot passwd -u root +mkosi-chroot usermod -p "$HASH" root + +# Allow login by password +echo "PasswordAuthentication yes" >> "$BUILDROOT/etc/ssh/sshd_config" diff --git a/mkosi.profiles/devtools/mkosi.extra/etc/systemd/system/serial-console.service b/mkosi.profiles/serial-console/serial-console.service similarity index 100% rename from mkosi.profiles/devtools/mkosi.extra/etc/systemd/system/serial-console.service rename to mkosi.profiles/serial-console/serial-console.service diff --git a/modules/l2/_common/kernel/config.d/30-xfs b/modules/l2/_common/kernel/config.d/30-xfs new file mode 100644 index 00000000..32f7f6d8 --- /dev/null +++ b/modules/l2/_common/kernel/config.d/30-xfs @@ -0,0 +1 @@ +CONFIG_XFS_FS=y diff --git a/modules/l2/_common/mkosi.build b/modules/l2/_common/mkosi.build new file mode 100755 index 00000000..762b970f --- /dev/null +++ b/modules/l2/_common/mkosi.build @@ -0,0 +1,62 @@ +#!/bin/bash + +set -euxo pipefail + +ENV_YAML="$SRCDIR/modules/l2/_common/mkosi.extra/etc/flashbots/l2.yaml" + +VAULT_REF=$(mkosi-chroot yq -r '.vault.git_reference' < "$ENV_YAML") +GOMPLATE_REF=$(mkosi-chroot yq -r '.gomplate.git_reference' < "$ENV_YAML") +GCP_OPS_AGENT_REF=$(mkosi-chroot yq -r '.gcp_ops_agent.git_reference' < "$ENV_YAML") + +source scripts/make_git_package.sh + +# build gomplate + +make_git_package \ + "gomplate" \ + "${GOMPLATE_REF}" \ + "https://github.com/hairyhenderson/gomplate" \ + 'go build -trimpath -ldflags "-s -w -X github.com/hairyhenderson/gomplate/v4/version=${GOMPLATE_REF} -buildid=" -o ./build/gomplate ./cmd/gomplate' \ + "build/gomplate:/usr/bin/gomplate" +chmod +x $DESTDIR/usr/bin/gomplate + +# build vault + +make_git_package \ + "vault" \ + "${VAULT_REF}" \ + "https://github.com/hashicorp/vault.git" \ + 'go build -tags minimal -trimpath -ldflags "-s -w -X github.com/hashicorp/vault/version.Version=${VAULT_REF} -buildid=" -o ./bin/vault .' \ + "bin/vault:/usr/bin/vault" +chmod +x $DESTDIR/usr/bin/vault + +# build gcp ops agent + +cd "$BUILDROOT" +IMPORT_PATH="github.com/GoogleCloudPlatform/ops-agent" +BUILD_CMD=" + # Main gcs agent binaries + mkdir -p out/libexec + LDFLAGS='-s -w -buildid=' + go build -buildvcs=false -trimpath -ldflags \"\$LDFLAGS \\ + -X $IMPORT_PATH/internal/version.BuildDistro=debian13 \\ + -X $IMPORT_PATH/internal/version.Version=$GCP_OPS_AGENT_REF\" \\ + -o out/libexec/google_cloud_ops_agent_engine \\ + $IMPORT_PATH/cmd/google_cloud_ops_agent_engine + + go build -buildvcs=false -trimpath -ldflags \"\$LDFLAGS\" \\ + -o out/libexec/google_cloud_ops_agent_wrapper \\ + $IMPORT_PATH/cmd/agent_wrapper +" +make_git_package \ + "google-cloud-ops-agent" \ + "$GCP_OPS_AGENT_REF" \ + "https://github.com/GoogleCloudPlatform/ops-agent" \ + "$BUILD_CMD" \ + "out/libexec:/opt/google-cloud-ops-agent/libexec" \ + "systemd/google-cloud-ops-agent-fluent-bit.service:/usr/lib/systemd/system/google-cloud-ops-agent-fluent-bit.service" \ + "systemd/google-cloud-ops-agent.service:/usr/lib/systemd/system/google-cloud-ops-agent.service" +chmod +x $DESTDIR/opt/google-cloud-ops-agent/libexec/* +mkdir -p "$DESTDIR/opt/google-cloud-ops-agent/subagents" +ln -s /opt/fluent-bit "$DESTDIR/opt/google-cloud-ops-agent/subagents/fluent-bit" +sed -i 's|@PREFIX@|/opt/google-cloud-ops-agent|g; s|@SYSCONFDIR@|/etc|g' "$DESTDIR/usr/lib/systemd/system/google-cloud-ops-agent"*.service diff --git a/modules/l2/_common/mkosi.conf b/modules/l2/_common/mkosi.conf new file mode 100644 index 00000000..027a4076 --- /dev/null +++ b/modules/l2/_common/mkosi.conf @@ -0,0 +1,20 @@ +[Build] +Environment=KERNEL_CONFIG_SNIPPETS_L2_COMMON=modules/l2/_common/kernel/config.d +WithNetwork=true + +[Content] +BuildScripts=modules/l2/_common/mkosi.build +ExtraTrees=modules/l2/_common/mkosi.extra +PostInstallationScripts=modules/l2/_common/mkosi.postinst.chroot +SyncScripts=modules/l2/_common//mkosi.sync + +Packages=fluent-bit + prometheus-node-exporter + prometheus-process-exporter + usrmerge + xfsprogs + +BuildPackages=golang + libssl-dev + unzip + yq diff --git a/modules/l2/_common/mkosi.extra/etc/default/prometheus-node-exporter b/modules/l2/_common/mkosi.extra/etc/default/prometheus-node-exporter new file mode 100644 index 00000000..93dea254 --- /dev/null +++ b/modules/l2/_common/mkosi.extra/etc/default/prometheus-node-exporter @@ -0,0 +1,7 @@ +# Set the command-line arguments to pass to the server. +ARGS="\ +--collector.systemd \ +--collector.systemd.unit-include=\".*(prometheus-node-exporter|prometheus-process-exporter|vault-agent).*\" \ +--log.format=json \ +--web.listen-address=0.0.0.0:9100 \ +" diff --git a/modules/l2/_common/mkosi.extra/etc/default/prometheus-process-exporter b/modules/l2/_common/mkosi.extra/etc/default/prometheus-process-exporter new file mode 100644 index 00000000..71b8d97c --- /dev/null +++ b/modules/l2/_common/mkosi.extra/etc/default/prometheus-process-exporter @@ -0,0 +1,6 @@ +# Set the command-line arguments to pass to the server. +ARGS="\ +-config.path=/etc/prometheus-process-exporter/config.yaml \ +-threads=false \ +-web.listen-address=0.0.0.0:9256 \ +" diff --git a/modules/l2/_common/mkosi.extra/etc/flashbots/l2.yaml b/modules/l2/_common/mkosi.extra/etc/flashbots/l2.yaml new file mode 100644 index 00000000..bcc1a81a --- /dev/null +++ b/modules/l2/_common/mkosi.extra/etc/flashbots/l2.yaml @@ -0,0 +1,8 @@ +gcp_ops_agent: + git_reference: 2.57.0 + +gomplate: + git_reference: v4.3.0 + +vault: + git_reference: v1.20.1 diff --git a/modules/l2/_common/mkosi.extra/etc/google-cloud-ops-agent/config.yaml b/modules/l2/_common/mkosi.extra/etc/google-cloud-ops-agent/config.yaml new file mode 100644 index 00000000..2c740cfc --- /dev/null +++ b/modules/l2/_common/mkosi.extra/etc/google-cloud-ops-agent/config.yaml @@ -0,0 +1,18 @@ +logging: + receivers: + syslog: + type: files + include_paths: + - /var/log/messages + - /var/log/syslog + processors: + parse_json: + type: parse_json + field: message + time_key: "@timestamp" + time_format: "%Y-%m-%dT%H:%M:%S.%L%z" + service: + pipelines: + default_pipeline: + receivers: [syslog] + processors: [parse_json] diff --git a/modules/l2/_common/mkosi.extra/etc/logrotate.d/rsyslog b/modules/l2/_common/mkosi.extra/etc/logrotate.d/rsyslog new file mode 100644 index 00000000..16d7e1ee --- /dev/null +++ b/modules/l2/_common/mkosi.extra/etc/logrotate.d/rsyslog @@ -0,0 +1,13 @@ +/var/log/syslog { + hourly + rotate 2 + notifempty + create + nocompress + missingok + size 128M + sharedscripts + postrotate + /usr/lib/rsyslog/rsyslog-rotate + endscript +} diff --git a/modules/l2/_common/mkosi.extra/etc/prometheus-process-exporter/config.yaml b/modules/l2/_common/mkosi.extra/etc/prometheus-process-exporter/config.yaml new file mode 100644 index 00000000..2941f049 --- /dev/null +++ b/modules/l2/_common/mkosi.extra/etc/prometheus-process-exporter/config.yaml @@ -0,0 +1,4 @@ +process_names: + - name: vault-agent + cmdline: + - '^\/([-.0-9a-zA-Z]+\/)*vault[-.0-9a-zA-Z]* ' diff --git a/modules/l2/_common/mkosi.extra/etc/rsyslog.d/01-json-template.conf b/modules/l2/_common/mkosi.extra/etc/rsyslog.d/01-json-template.conf new file mode 100644 index 00000000..b6f485e5 --- /dev/null +++ b/modules/l2/_common/mkosi.extra/etc/rsyslog.d/01-json-template.conf @@ -0,0 +1,51 @@ +# +# - this template formats event's metadata as JSON while leaving the +# event's message as-is (expects it's already JSON) +# - the position.from addresses this known behavior +# https://www.rsyslog.com/log-normalization-and-the-leading-space/ +# +template(name="json-event-json-msg" + type="list") { + constant(value="{") + constant(value="\"@timestamp\":\"") property(name="timereported" dateFormat="rfc3339") + constant(value="\",\"sysloghost\":\"") property(name="hostname") + constant(value="\",\"procid\":\"") property(name="procid") + constant(value="\",\"facility\":\"") property(name="syslogfacility-text") + constant(value="\",\"severity\":\"") property(name="syslogseverity-text") + constant(value="\",\"source\":\"") property(name="programname") + constant(value="\",\"message\":") property(name="msg" position.from="2") + constant(value="}\n") +} + +# +# - this template formats event's metadata and message as JSON (expects +# message is string) +# - the position.from addresses this known behavior +# https://www.rsyslog.com/log-normalization-and-the-leading-space/ +# +template(name="json-event-string-msg" + type="list") { + constant(value="{") + constant(value="\"@timestamp\":\"") property(name="timereported" dateFormat="rfc3339") + constant(value="\",\"sysloghost\":\"") property(name="hostname") + constant(value="\",\"procid\":\"") property(name="procid") + constant(value="\",\"facility\":\"") property(name="syslogfacility-text") + constant(value="\",\"severity\":\"") property(name="syslogseverity-text") + constant(value="\",\"source\":\"") property(name="programname") + constant(value="\",\"message\":\"") property(name="msg" format="json" position.from="2") + constant(value="\"}\n") +} + +# +# - this conditional action determines which template to use based on +# the beginning of the event's message value +# - assumes that if a message starts with {, then that message is JSON +# - this follows the pattern found in /etc/rsyslog.d/50-default.conf +# +if ($msg startswith " {") then { + *.*;auth,authpriv.none -/var/log/syslog;json-event-json-msg + & stop +} else { + *.*;auth,authpriv.none -/var/log/syslog;json-event-string-msg + & stop +} diff --git a/modules/l2/_common/mkosi.extra/etc/systemd/system/automount-data.service b/modules/l2/_common/mkosi.extra/etc/systemd/system/automount-data.service new file mode 100644 index 00000000..bf2887fe --- /dev/null +++ b/modules/l2/_common/mkosi.extra/etc/systemd/system/automount-data.service @@ -0,0 +1,16 @@ +[Unit] +Description=Automatically mount data volume + +[Service] +Type=oneshot +SyslogIdentifier=automount-data +User=root +Group=root + +RemainAfterExit=yes + +EnvironmentFile=-/etc/sysconfig/automount-data.env +ExecStart=/usr/bin/automount-data.sh + +[Install] +WantedBy=default.target diff --git a/modules/l2/_common/mkosi.extra/etc/systemd/system/logrotate.timer.d/override.conf b/modules/l2/_common/mkosi.extra/etc/systemd/system/logrotate.timer.d/override.conf new file mode 100644 index 00000000..5d1d4b7b --- /dev/null +++ b/modules/l2/_common/mkosi.extra/etc/systemd/system/logrotate.timer.d/override.conf @@ -0,0 +1,3 @@ +[Timer] +OnCalendar=*-*-* *:0/5:0 +RandomizedDelaySec=15 diff --git a/modules/l2/_common/mkosi.extra/etc/systemd/system/ptlb-routes-nanny.service b/modules/l2/_common/mkosi.extra/etc/systemd/system/ptlb-routes-nanny.service new file mode 100644 index 00000000..cab73ee4 --- /dev/null +++ b/modules/l2/_common/mkosi.extra/etc/systemd/system/ptlb-routes-nanny.service @@ -0,0 +1,15 @@ +[Unit] +Description=Ensure presence of local routes required for passthrough load-balancer +After=network-online.target +Wants=network-online.target +StartLimitIntervalSec=2 +StartLimitBurst=2 + +[Service] +User=root +Group=root +Type=oneshot +ExecStart=/usr/bin/ptlb-routes-nanny.sh + +[Install] +WantedBy=multi-user.target diff --git a/modules/l2/_common/mkosi.extra/etc/systemd/system/ptlb-routes-nanny.timer b/modules/l2/_common/mkosi.extra/etc/systemd/system/ptlb-routes-nanny.timer new file mode 100644 index 00000000..fe1cb6e1 --- /dev/null +++ b/modules/l2/_common/mkosi.extra/etc/systemd/system/ptlb-routes-nanny.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Ensure presence of local routes required for passthrough load-balancer + +[Timer] +OnCalendar=*-*-* *:*:5 +Unit=ptlb-routes-nanny.service + +[Install] +WantedBy=multi-user.target diff --git a/modules/l2/_common/mkosi.extra/etc/systemd/system/systemd-networkd.service.d/override.conf b/modules/l2/_common/mkosi.extra/etc/systemd/system/systemd-networkd.service.d/override.conf new file mode 100644 index 00000000..937d2fcd --- /dev/null +++ b/modules/l2/_common/mkosi.extra/etc/systemd/system/systemd-networkd.service.d/override.conf @@ -0,0 +1,3 @@ +[Service] +Environment=SYSTEMD_LOG_LEVEL=debug +Environment=SYSTEMD_LOG_LOCATION=1 diff --git a/modules/l2/_common/mkosi.extra/etc/systemd/system/vault-agent.service b/modules/l2/_common/mkosi.extra/etc/systemd/system/vault-agent.service new file mode 100644 index 00000000..96f27849 --- /dev/null +++ b/modules/l2/_common/mkosi.extra/etc/systemd/system/vault-agent.service @@ -0,0 +1,33 @@ +[Unit] +Description=HashiCorp Vault Agent +After=network.target network-setup.service +Requires=automount-data.service +Wants=network-setup.service + +[Service] +Type=simple +SyslogIdentifier=vault-agent +User=root +Group=root + +KillMode=process +KillSignal=SIGINT +NoNewPrivileges=yes +Restart=on-failure +RestartSec=5s +TimeoutStopSec=30 + +ExecStartPre=/usr/bin/gomplate \ + --left-delim "[[" \ + --right-delim "]]" \ + --input-dir "/etc/vault-agent/gomplate" \ + --output-dir "/etc/vault-agent" \ + --include "*.hcl,*.ctmpl" + +ExecStart=/usr/bin/vault agent \ + -config /etc/vault-agent \ + -log-level info \ + -log-format json + +[Install] +WantedBy=default.target diff --git a/modules/l2/_common/mkosi.extra/etc/vault-agent/gomplate/authorized_keys.ctmpl b/modules/l2/_common/mkosi.extra/etc/vault-agent/gomplate/authorized_keys.ctmpl new file mode 100644 index 00000000..faebfad0 --- /dev/null +++ b/modules/l2/_common/mkosi.extra/etc/vault-agent/gomplate/authorized_keys.ctmpl @@ -0,0 +1,11 @@ +# +# rendered by vault-agent/gomplate +# +[[- range ( ( gcp.Meta "attributes/ssh-keys" ) | strings.Split "\n" ) ]] +[[- if strings.HasPrefix "debian:" . ]] +[[ strings.TrimPrefix "debian:" . ]] +[[- end ]] +[[- if strings.HasPrefix "ubuntu:" . ]] +[[ strings.TrimPrefix "ubuntu:" . ]] +[[- end ]] +[[- end ]] diff --git a/modules/l2/_common/mkosi.extra/etc/vault-agent/gomplate/authorized_keys.hcl b/modules/l2/_common/mkosi.extra/etc/vault-agent/gomplate/authorized_keys.hcl new file mode 100644 index 00000000..708e0748 --- /dev/null +++ b/modules/l2/_common/mkosi.extra/etc/vault-agent/gomplate/authorized_keys.hcl @@ -0,0 +1,13 @@ +[[- if ( file.Exists "/home/debian/.ssh/authorized_keys" ) ]] +template { + left_delimiter = "((" + right_delimiter = "))" + + source = "/etc/vault-agent/authorized_keys.ctmpl" + destination = "/home/debian/.ssh/authorized_keys" + + user = "debian" + group = "debian" + perms = "0600" +} +[[- end ]] diff --git a/modules/l2/_common/mkosi.extra/etc/vault-agent/gomplate/config.hcl b/modules/l2/_common/mkosi.extra/etc/vault-agent/gomplate/config.hcl new file mode 100644 index 00000000..06b89b32 --- /dev/null +++ b/modules/l2/_common/mkosi.extra/etc/vault-agent/gomplate/config.hcl @@ -0,0 +1,20 @@ +pid_file = "/var/run/vault-agent.pid" + +vault { + address = "[[ gcp.Meta "attributes/vault_addr" ]]" + + retry { + num_retries = 5 + } +} + +auto_auth { + method "gcp" { + mount_path = "[[ gcp.Meta "attributes/vault_auth_mount_gcp" ]]" + + config = { + type = "gce" + role = "[[ gcp.Meta "name" ]]" + } + } +} diff --git a/modules/l2/_common/mkosi.extra/usr/bin/automount-data.sh b/modules/l2/_common/mkosi.extra/usr/bin/automount-data.sh new file mode 100755 index 00000000..0db826a1 --- /dev/null +++ b/modules/l2/_common/mkosi.extra/usr/bin/automount-data.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +set -eu + +udevadm trigger --subsystem-match=block +udevadm settle --timeout=30 + +if [ -e /dev/disk/by-id/google-data ]; then + device=$( realpath /dev/disk/by-id/google-data ) + if ! grep -qs "${device}" /proc/mounts; then + eval $( blkid --output export ${device} ) + if [ -z "${TYPE:-}" ]; then + mkfs.ext4 -m 0 ${device} + eval $( blkid --output export ${device} ) + fi + AUTOMOUNT_PATH_DATA=${AUTOMOUNT_PATH_DATA:-/var/opt/data} + echo "UUID=${UUID} ${AUTOMOUNT_PATH_DATA} ${TYPE} defaults 0 0" >> /etc/fstab + mkdir -p "${AUTOMOUNT_PATH_DATA}" + chmod 0777 "${AUTOMOUNT_PATH_DATA}" + systemctl daemon-reload + mount --all + else + echo "Device ${device} is already mounted, skipping..." + fi +else + echo "/dev/disk/by-id/google-data doesn't exist, skipping..." +fi diff --git a/modules/l2/_common/mkosi.extra/usr/bin/ptlb-routes-nanny.sh b/modules/l2/_common/mkosi.extra/usr/bin/ptlb-routes-nanny.sh new file mode 100755 index 00000000..cdd30336 --- /dev/null +++ b/modules/l2/_common/mkosi.extra/usr/bin/ptlb-routes-nanny.sh @@ -0,0 +1,57 @@ +#!/bin/sh + +set -eu + +is_ip4() { + echo "$1" | awk -F. ' + $1 <= 255 && $2 <= 255 && $3 <= 255 && $4 <= 255 && + $1 >= 0 && $2 >= 0 && $3 >= 0 && $4 >= 0 && + $1 != "" && $2 != "" && $3 != "" && $4 != "" && + NF == 4 + { exit 0 } + { exit 1 } + ' +} + +for line in "$( + ip -br link show | grep -v lo +)"; do + interface="${line%% *}" + + for idx in $( + curl \ + --header "metadata-flavor: Google" \ + --max-time 1 \ + --show-error \ + --silent \ + http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/forwarded-ips/ + ); do + ip=$( + curl \ + --header "metadata-flavor: Google" \ + --max-time 1 \ + --show-error \ + --silent \ + http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/forwarded-ips/${idx} + ) + + if is_ip4 "${ip}" > /dev/null; then + route="local ${ip} dev ${interface} proto 66 scope host" + + if ! ip route show table local | grep -q "${route}"; then + echo "---" + echo "$ ip route show table local" + ip route show table local + echo "---" + echo "Route is missing, adding..." + echo "---" + echo "$ ip route add ${route}" + ip route add ${route} + echo "---" + echo "$ ip route show table local" + ip route show table local + echo "---" + fi + fi + done +done diff --git a/modules/l2/_common/mkosi.postinst.chroot b/modules/l2/_common/mkosi.postinst.chroot new file mode 100755 index 00000000..2c0d8e90 --- /dev/null +++ b/modules/l2/_common/mkosi.postinst.chroot @@ -0,0 +1,35 @@ +#!/bin/bash + +set -euxo pipefail + +# Enable systemd services + +systemctl add-wants minimal.target \ + automount-data.service \ + google-cloud-ops-agent-fluent-bit.service \ + google-cloud-ops-agent.service \ + prometheus-node-exporter.service \ + prometheus-process-exporter.service \ + ptlb-routes-nanny.timer \ + vault-agent.service + +# Remove automatically generated vault cert + +rm -rf "$BUILDROOT/opt/vault/tls" + +# Fix permissions + +mkdir -p /vault/secrets +chmod 0770 /vault/secrets + +chmod 0750 /etc/vault-agent +chmod 0750 /etc/vault-agent/gomplate +chmod 0640 /etc/vault-agent/gomplate/* + +# Create /etc/sysconfig secrets injection + +mkdir -p /etc/sysconfig + +# Limit root filesystem size to 4GB + +sed -i '1a mount -o remount,size=4G /' /init diff --git a/modules/l2/_common/mkosi.sync b/modules/l2/_common/mkosi.sync new file mode 100755 index 00000000..0cb90316 --- /dev/null +++ b/modules/l2/_common/mkosi.sync @@ -0,0 +1,5 @@ +#!/bin/bash + +if [ ! -f "$SRCDIR/mkosi.packages/fluent-bit.deb" ]; then + curl -sSfL https://packages.fluentbit.io/debian/bookworm/fluent-bit_3.1.6_amd64.deb -o "$SRCDIR/mkosi.packages/fluent-bit.deb" +fi diff --git a/modules/l2/_gcp/mkosi.conf b/modules/l2/_gcp/mkosi.conf new file mode 100644 index 00000000..cc85c9ac --- /dev/null +++ b/modules/l2/_gcp/mkosi.conf @@ -0,0 +1,6 @@ +[Match] +Profiles=gcp + +[Content] +ExtraTrees=modules/l2/_gcp/mkosi.extra +PostInstallationScripts=modules/l2/_gcp/mkosi.postinst.chroot diff --git a/modules/l2/_gcp/mkosi.extra/etc/systemd/resolved.conf b/modules/l2/_gcp/mkosi.extra/etc/systemd/resolved.conf new file mode 100644 index 00000000..3bae126c --- /dev/null +++ b/modules/l2/_gcp/mkosi.extra/etc/systemd/resolved.conf @@ -0,0 +1,2 @@ +[Resolve] +DNS=169.254.169.254 diff --git a/modules/l2/_gcp/mkosi.extra/etc/systemd/system/set-hostname.service b/modules/l2/_gcp/mkosi.extra/etc/systemd/system/set-hostname.service new file mode 100644 index 00000000..263867fa --- /dev/null +++ b/modules/l2/_gcp/mkosi.extra/etc/systemd/system/set-hostname.service @@ -0,0 +1,14 @@ +[Unit] +Description=Set hostname +ConditionFirstBoot=yes +After=network.target network-setup.service +Wants=network-setup.service + +[Service] +User=root +Group=root +Type=oneshot +ExecStart=/usr/bin/set-hostname-gcp + +[Install] +WantedBy=default.target diff --git a/mkosi.profiles/gcp/mkosi.extra/usr/bin/set-hostname-gcp b/modules/l2/_gcp/mkosi.extra/usr/bin/set-hostname-gcp similarity index 94% rename from mkosi.profiles/gcp/mkosi.extra/usr/bin/set-hostname-gcp rename to modules/l2/_gcp/mkosi.extra/usr/bin/set-hostname-gcp index 526cce44..779d2430 100755 --- a/mkosi.profiles/gcp/mkosi.extra/usr/bin/set-hostname-gcp +++ b/modules/l2/_gcp/mkosi.extra/usr/bin/set-hostname-gcp @@ -1,5 +1,6 @@ #!/bin/bash -set -e + +set -euxo pipefail hostname=$(curl --header "Metadata-Flavor: Google" --fail --silent --show-error \ --retry 100 --retry-delay 1 --retry-all-errors \ diff --git a/modules/l2/_gcp/mkosi.postinst.chroot b/modules/l2/_gcp/mkosi.postinst.chroot new file mode 100755 index 00000000..aea5f967 --- /dev/null +++ b/modules/l2/_gcp/mkosi.postinst.chroot @@ -0,0 +1,7 @@ +#!/bin/bash + +set -euxo pipefail + +# Enable systemd services + +systemctl add-wants minimal.target set-hostname.service diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.build b/modules/l2/op-rbuilder-bproxy/mkosi.build new file mode 100755 index 00000000..629e0c8c --- /dev/null +++ b/modules/l2/op-rbuilder-bproxy/mkosi.build @@ -0,0 +1,63 @@ +#!/bin/bash + +set -euxo pipefail + +ENV_YAML="$SRCDIR/modules/l2/op-rbuilder/mkosi.extra/etc/flashbots/op-rbuilder.yaml" + +BPROXY_REF=$(mkosi-chroot yq -r .bproxy.git_reference < "$ENV_YAML") +NODE_HEALTHCHECKER_REF=$(mkosi-chroot yq -r .node_healthchecker.git_reference < "$ENV_YAML") +OP_RBUILDER_FEATURES=$(mkosi-chroot yq -r '.op_rbuilder.features // "default"' < "$ENV_YAML") +OP_RBUILDER_REF=$(mkosi-chroot yq -r .op_rbuilder.git_reference < "$ENV_YAML") +TDX_QUOTE_PROVIDER_REF=$(mkosi-chroot yq -r .tdx_quote_provider.git_reference < "$ENV_YAML") + +source scripts/make_git_package.sh +source scripts/build_rust_package.sh + +export CARGO_HOME="/cargo" +export RUSTUP_HOME="/rustup" +export PATH="$CARGO_HOME/bin:$PATH" +RUST_VERSION=$(mkosi-chroot yq -r '.rust.version' < "$ENV_YAML") +mkosi-chroot rustup toolchain install $RUST_VERSION +mkosi-chroot rustup default $RUST_VERSION + +# build op-rbuilder + +build_rust_package \ + "op-rbuilder" \ + "${OP_RBUILDER_REF}" \ + "https://github.com/flashbots/op-rbuilder.git" \ + "" "${OP_RBUILDER_FEATURES}" "-g" +chmod +x $DESTDIR/usr/bin/op-rbuilder + +# build tdx-quote-provider + +build_rust_package \ + "tdx-quote-provider" \ + "${TDX_QUOTE_PROVIDER_REF}" \ + "https://github.com/flashbots/op-rbuilder.git" \ + "" "" "-g" +chmod +x $DESTDIR/usr/bin/tdx-quote-provider + +# build bproxy + +make_git_package \ + "bproxy" \ + "${BPROXY_REF}" \ + "https://github.com/flashbots/bproxy.git" \ + 'make build' \ + "bin/bproxy:/usr/bin/bproxy" +chmod +x $DESTDIR/usr/bin/bproxy + +# build node-healthchecker + +make_git_package \ + "node-healthchecker" \ + "${NODE_HEALTHCHECKER_REF}" \ + "https://github.com/flashbots/node-healthchecker.git" \ + 'go build -trimpath -ldflags "-s -w -X main.version=${NODE_HEALTHCHECKER_REF} -buildid=" -o ./bin/node-healthchecker github.com/flashbots/node-healthchecker/cmd' \ + "bin/node-healthchecker:/usr/bin/node-healthchecker" +chmod +x $DESTDIR/usr/bin/node-healthchecker + +# generate version + +cp $BUILDDIR/op-rbuilder.git $BUILDDIR/workload.git diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.conf b/modules/l2/op-rbuilder-bproxy/mkosi.conf new file mode 100644 index 00000000..6ff09886 --- /dev/null +++ b/modules/l2/op-rbuilder-bproxy/mkosi.conf @@ -0,0 +1,17 @@ +[Build] +WithNetwork=true + +[Content] +BuildScripts=modules/l2/op-rbuilder/mkosi.build +ExtraTrees=modules/l2/op-rbuilder/mkosi.extra +PostInstallationScripts=modules/l2/op-rbuilder/mkosi.postinst.chroot + +Packages=libtss2-dev + sudo + unzip + +BuildPackages=golang + libssl-dev + rustup + unzip + yq diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/default/prometheus-node-exporter b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/default/prometheus-node-exporter new file mode 100644 index 00000000..6d083815 --- /dev/null +++ b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/default/prometheus-node-exporter @@ -0,0 +1,7 @@ +# Set the command-line arguments to pass to the server. +ARGS="\ +--collector.systemd \ +--collector.systemd.unit-include=\".*(bproxy|node-healthchecker|op-rbuilder|prometheus-node-exporter|prometheus-process-exporter|vault-agent).*\" \ +--log.format=json \ +--web.listen-address=0.0.0.0:9100 \ +" diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/flashbots/op-rbuilder.yaml b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/flashbots/op-rbuilder.yaml new file mode 100644 index 00000000..dcfbe317 --- /dev/null +++ b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/flashbots/op-rbuilder.yaml @@ -0,0 +1,10 @@ +bproxy: + git_reference: v0.0.92-hf.3 +node_healthchecker: + git_reference: v0.1.11 +op_rbuilder: + git_reference: op-rbuilder/v0.4.0 +rust: + version: 1.94.0 +tdx_quote_provider: + git_reference: tdx-quote-provider/v0.1.0 diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/prometheus-process-exporter/config.yaml b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/prometheus-process-exporter/config.yaml new file mode 100644 index 00000000..5ccb59e8 --- /dev/null +++ b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/prometheus-process-exporter/config.yaml @@ -0,0 +1,13 @@ +process_names: + - name: bproxy + cmdline: + - '^\/([-.0-9a-zA-Z]+\/)*bproxy[-.0-9a-zA-Z]* ' + - name: node-healthchecker + cmdline: + - '^\/([-.0-9a-zA-Z]+\/)*node-healthchecker[-.0-9a-zA-Z]* ' + - name: op-rbuilder + cmdline: + - '^\/([-.0-9a-zA-Z]+\/)*op-rbuilder[-.0-9a-zA-Z]* ' + - name: vault-agent + cmdline: + - '^\/([-.0-9a-zA-Z]+\/)*vault[-.0-9a-zA-Z]* ' diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/sysconfig/automount-data.env b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/sysconfig/automount-data.env new file mode 100644 index 00000000..2963c609 --- /dev/null +++ b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/sysconfig/automount-data.env @@ -0,0 +1 @@ +AUTOMOUNT_PATH_DATA=/var/opt/optimism diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/systemd/system/node-healthchecker.service b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/systemd/system/node-healthchecker.service new file mode 100644 index 00000000..35ef6122 --- /dev/null +++ b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/systemd/system/node-healthchecker.service @@ -0,0 +1,25 @@ +[Unit] +Description=Blockchain node healthchecker +After=network.target +Wants=network.target + +[Service] +Type=simple +SyslogIdentifier=node-healthchecker +User=op-rbuilder +Group=optimism + +Restart=always +RestartSec=5 +TimeoutStopSec=60 + +ExecStart=/usr/bin/node-healthchecker serve \ + --healthcheck-block-age-threshold 10s \ + --healthcheck-timeout 500ms \ + --healthcheck-reth-base-url http://127.0.0.1:18645 \ + --healthcheck-unconditional-fail-duration 1m \ + --http-status-warning 200 \ + --server-listen-address 0.0.0.0:8080 + +[Install] +WantedBy=default.target diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/systemd/system/tdx-quote-provider.service b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/systemd/system/tdx-quote-provider.service new file mode 100644 index 00000000..e9aa03fd --- /dev/null +++ b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/systemd/system/tdx-quote-provider.service @@ -0,0 +1,26 @@ +[Install] +WantedBy=default.target + +[Unit] +Description=TDX quote provider +After=network.target +Wants=network.target + +[Service] +Type=simple +SyslogIdentifier=tdx-quote-provider +User=root +Group=root + +Restart=always +RestartSec=5 +TimeoutStopSec=60 + +ExecStart=/usr/bin/tdx-quote-provider \ + --service-host 127.0.0.1 \ + --service-port 5757 \ + --metrics \ + --metrics-host 0.0.0.0 \ + --metrics-port 9009 \ + --log-level trace \ + --log-format json diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/bproxy-tls.crt.hcl b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/bproxy-tls.crt.hcl new file mode 100644 index 00000000..31d440fb --- /dev/null +++ b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/bproxy-tls.crt.hcl @@ -0,0 +1,29 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + destination = "/etc/bproxy/tls.crt" + + user = "op-rbuilder" + group = "optimism" + perms = "0640" + + exec { + timeout = "60s" + + command = ["/bin/sh", "-c", + <<-EOT + chown -R op-rbuilder:optimism /etc/bproxy + systemctl restart bproxy + EOT + ] + } + + contents = <<-EOT + ((- $tls_crt := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_tls[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data.tls_crt -)) + + ((- if $tls_crt -)) + (( $tls_crt )) + ((- end -)) + EOT +} diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/bproxy-tls.key.hcl b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/bproxy-tls.key.hcl new file mode 100644 index 00000000..a7a766c1 --- /dev/null +++ b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/bproxy-tls.key.hcl @@ -0,0 +1,29 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + destination = "/etc/bproxy/tls.key" + + user = "op-rbuilder" + group = "optimism" + perms = "0640" + + exec { + timeout = "60s" + + command = ["/bin/sh", "-c", + <<-EOT + chown -R op-rbuilder:optimism /etc/bproxy + systemctl restart bproxy + EOT + ] + } + + contents = <<-EOT + ((- $tls_key := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_tls[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data.tls_key -)) + + ((- if $tls_key -)) + (( $tls_key )) + ((- end -)) + EOT +} diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/bproxy.service.hcl b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/bproxy.service.hcl new file mode 100644 index 00000000..0a406054 --- /dev/null +++ b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/bproxy.service.hcl @@ -0,0 +1,23 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + source = "/etc/vault-agent/bproxy.service.ctmpl" + destination = "/etc/systemd/system/bproxy.service" + + user = "root" + group = "root" + perms = "0644" + + exec { + timeout = "60s" + + command = ["/bin/sh", "-c", + <<-EOT + systemctl daemon-reload + systemctl add-wants minimal.target bproxy.service + systemctl restart bproxy.service + EOT + ] + } +} diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/discovery-secret.hcl b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/discovery-secret.hcl new file mode 100644 index 00000000..396f56c1 --- /dev/null +++ b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/discovery-secret.hcl @@ -0,0 +1,29 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + destination = "/var/opt/optimism/rbuilder/discovery-secret" + + user = "op-rbuilder" + group = "optimism" + perms = "0600" + + exec { + timeout = "60s" + + command = ["/bin/sh", "-c", + <<-EOT + # discovery-secret + chown op-rbuilder:optimism /var/opt/optimism/rbuilder + chmod 0750 /var/opt/optimism/rbuilder + systemctl restart op-rbuilder + EOT + ] + } + + contents = <<-EOT + ((- $node := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/[[ gcp.Meta "name" ]]" ).Data.data -)) + + ((- $node.el_nodekey -)) + EOT +} diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/genesis.json.hcl b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/genesis.json.hcl new file mode 100644 index 00000000..f7920fcd --- /dev/null +++ b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/genesis.json.hcl @@ -0,0 +1,31 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + destination = "/var/opt/optimism/rbuilder/genesis.json.tar.gz.base64" + + user = "op-rbuilder" + group = "optimism" + perms = "0640" + + exec { + timeout = "60s" + + command = ["/bin/sh", "-c", + <<-EOT + # genesis.json + chown op-rbuilder:optimism /var/opt/optimism/rbuilder + chmod 0750 /var/opt/optimism/rbuilder + systemctl restart op-rbuilder + EOT + ] + } + + contents = <<-EOT + ((- $service := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_common[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data -)) + + ((- if $service.genesis_json -)) + ((- $service.genesis_json -)) + ((- end -)) + EOT +} diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/jwtsecret.hcl b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/jwtsecret.hcl new file mode 100644 index 00000000..52f90602 --- /dev/null +++ b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/jwtsecret.hcl @@ -0,0 +1,16 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + destination = "/var/opt/optimism/jwtsecret" + + user = "root" + group = "optimism" + perms = "0440" + + contents = <<-EOT + ((- with secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/[[ gcp.Meta "name" ]]" -)) + (( .Data.data.jwt_secret )) + ((- end -)) + EOT +} diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.env.hcl b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.env.hcl new file mode 100644 index 00000000..5118676a --- /dev/null +++ b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.env.hcl @@ -0,0 +1,48 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + destination = "/etc/sysconfig/op-rbuilder.env" + + user = "root" + group = "optimism" + perms = "0640" + + exec { + timeout = "60s" + + command = ["/bin/sh", "-c", + <<-EOT + # op-rbuilder.env + systemctl restart op-rbuilder + EOT + ] + } + + contents = <<-EOT + ((- printf "# %s\n\n" "op-rbuilder" -)) + + ((- $node := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/[[ gcp.Meta "name" ]]" ).Data.data -)) + ((- $service := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_common[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data -)) + + ((- if $node.builder_secret_key -)) + BUILDER_SECRET_KEY=(( $node.builder_secret_key ))(( "\n" )) + ((- end -)) + + ((- if $node.coinbase_secret_key -)) + COINBASE_SECRET_KEY=(( $node.coinbase_secret_key ))(( "\n" )) + ((- end -)) + + ((- if $service.otel_exporter_otlp_endpoint -)) + OTEL_EXPORTER_OTLP_ENDPOINT=(( $service.otel_exporter_otlp_endpoint ))(( "\n" )) + ((- end -)) + + ((- if $node.otel_exporter_otlp_headers -)) + OTEL_EXPORTER_OTLP_HEADERS=(( $node.otel_exporter_otlp_headers ))(( "\n" )) + ((- end -)) + + ((- if $service.otel_service_name -)) + OTEL_SERVICE_NAME=(( $service.otel_service_name ))(( "\n" )) + ((- end -)) + EOT +} diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.ctmpl b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.ctmpl new file mode 100644 index 00000000..27c006d8 --- /dev/null +++ b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.ctmpl @@ -0,0 +1,120 @@ +# op-rbuilder + +[Install] +WantedBy=default.target + +[Unit] +Description=op-rbuilder +After=network-online.target +Before=bproxy.service node-healthchecker.service +Requires=automount-data.service vault-agent.service +Wants=network-online.target + +[Service] +Type=simple +SyslogIdentifier=op-rbuilder +User=op-rbuilder +Group=optimism + +EnvironmentFile=-/etc/sysconfig/op-rbuilder.env +KillMode=control-group +KillSignal=SIGINT +OOMScoreAdjust=-500 +Restart=always +RestartSec=10s +TimeoutStopSec=120 + +Environment=HOME=/home/op-rbuilder +Environment=RUST_BACKTRACE=full + +ExecStartPre=+/usr/bin/init-op-rbuilder.sh + +((- $service := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_common[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data )) + +ExecStart=/usr/bin/op-rbuilder node \ + --authrpc.addr '0.0.0.0' \ + --authrpc.jwtsecret '/var/opt/optimism/jwtsecret' \ + --authrpc.port '18651' \ + ((- if $service.el_bootnodes ))(( range $idx, $enode := $service.el_bootnodes )) + --bootnodes '(( printf "%s" $enode ))' \ + ((- end ))(( end )) + ((- if $service.feat_revert_protection ))(( if $service.feat_revert_protection | parseBool )) + --builder.enable-revert-protection \ + ((- end ))(( end )) + --builder.extra-block-deadline-secs '30' \ + --builder.extradata 'Illuminate Dmocratize Dstribute' \ + --builder.log-pool-transactions \ + ((- if $service.genesis_json )) + --chain '/var/opt/optimism/rbuilder/genesis.json' \ + ((- else ))(( if $service.network_name )) + --chain '(( $service.network_name ))' \ + ((- end ))(( end )) + --color 'never' \ + --datadir '/var/opt/optimism/rbuilder' \ + --discovery.port '9200' \ + --enable-discv5-discovery \ + ((- if $service.feat_flashblocks ))(( if $service.feat_flashblocks | parseBool )) + --flashblocks.addr '0.0.0.0' \ + ((- if $service.feat_flashblocks_block_time )) + --flashblocks.block-time '(( $service.feat_flashblocks_block_time ))' \ + ((- end )) + --flashblocks.enabled \ + --flashblocks.port '11111' \ + ((- if $service.feat_flashblocks_number_contract_address )) + --flashblocks.number-contract-address '(( $service.feat_flashblocks_number_contract_address ))' \ + --flashblocks.number-contract-use-permit \ + ((- end )) + ((- end ))(( end )) + ((- if $service.feat_flashtestations ))(( if $service.feat_flashtestations | parseBool )) + ((- if $service.feat_flashtestations_builder_policy_address )) + --flashtestations.builder-policy-address '(( $service.feat_flashtestations_builder_policy_address ))' \ + ((- end )) + ((- if $service.feat_flashtestations_block_proofs ))(( if $service.feat_flashtestations_block_proofs | parseBool )) + --flashtestations.enable-block-proofs \ + ((- end ))(( end )) + --flashtestations.enabled \ + --flashtestations.quote-provider http://127.0.0.1:5757/attest \ + ((- if $service.feat_flashtestations_registry_address )) + --flashtestations.registry-address '(( $service.feat_flashtestations_registry_address ))' \ + ((- end )) + ((- if $service.feat_flashtestations_rpc_url )) + --flashtestations.rpc-url '(( $service.feat_flashtestations_rpc_url ))' \ + ((- end )) + ((- end ))(( end )) + --http \ + --http.addr '0.0.0.0' \ + --http.api 'admin,debug,eth,net,trace,txpool' \ + --http.corsdomain '*' \ + --http.port '18645' \ + --log.file.filter 'off' \ + --log.stdout.format 'json' \ + --metrics '0.0.0.0:9001' \ + --port '40404' \ + --rollup.disable-tx-pool-gossip \ + --rpc-max-connections '5000' \ + --rpc-max-logs-per-response '200000' \ + --rpc-max-request-size '150' \ + --rpc-max-response-size '1150' \ + --rpc-max-subscriptions-per-connection '10240' \ + --rpc-max-tracing-requests '250' \ + ((- if $service.feat_telemetry_sampling_ratio ))(( if $service.feat_telemetry_sampling_ratio )) + --telemetry.sampling-ratio '(( $service.feat_telemetry_sampling_ratio ))' \ + ((- end ))(( end )) + ((- if $service.el_static_peers ))(( range $idx, $enode := $service.el_static_peers )) + --trusted-peers '(( printf "%s" $enode ))' \ + ((- end ))(( end )) + --txpool.max-new-pending-txs-notifications '100000' \ + --txpool.max-new-txns '100000' \ + --txpool.max-pending-txns '100000' \ + --ws \ + --ws.addr '0.0.0.0' \ + --ws.api 'admin,debug,eth,net,trace,txpool' \ + --ws.origins '*' \ + --ws.port '8646' \ + ((- if $service.custom_flags ))(( range $idx, $flag := $service.custom_flags )) + (( printf "%s" $flag )) \ + ((- end ))(( end )) + +ExecStop=+/usr/bin/sh -c "kill -1 $( pgrep node-health ) | true" +ExecStop=+/usr/bin/sleep 15 +ExecStop=+/usr/bin/sh -c "PID=$( pgrep op-rbuilder ); if [ 0${PID} -gt 0 ]; then kill -2 ${PID}; while kill -0 ${PID} 2>/dev/null; do sleep 1; done; fi" diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.hcl b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.hcl new file mode 100644 index 00000000..9696e71f --- /dev/null +++ b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.hcl @@ -0,0 +1,33 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + source = "/etc/vault-agent/op-rbuilder.service.ctmpl" + destination = "/etc/systemd/system/op-rbuilder.service" + + user = "root" + group = "root" + perms = "0644" + + exec { + timeout = "60s" + + command = ["/bin/sh", "-c", + <<-EOT + # op-rbuilder + + systemctl daemon-reload + systemctl add-wants minimal.target op-rbuilder.service + + # patterns longer than 15 chars result in 0 matches + PID=$( pgrep node-health ); if [ 0${PID} -gt 0 ]; then kill -1 ${PID} || true; fi + sleep 5 + + PID=$( pgrep rproxy ); if [ 0${PID} -gt 0 ]; then kill -1 ${PID} || true; fi + + systemctl restart op-rbuilder.service + systemctl restart node-healthchecker.service + EOT + ] + } +} diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.extra/usr/bin/init-op-rbuilder.sh b/modules/l2/op-rbuilder-bproxy/mkosi.extra/usr/bin/init-op-rbuilder.sh new file mode 100755 index 00000000..79307b98 --- /dev/null +++ b/modules/l2/op-rbuilder-bproxy/mkosi.extra/usr/bin/init-op-rbuilder.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +set -eu + +if [ ! -d /home/op-rbuilder ]; then + mkdir -p /home/op-rbuilder + chown -R op-rbuilder:optimism /home/op-rbuilder + chmod 0750 /home/op-rbuilder +fi + +if [ ! -d /var/opt/optimism/rbuilder ]; then + if [ -d /var/opt/optimism/unichain-builder ]; then + mv /var/opt/optimism/unichain-builder /var/opt/optimism/rbuilder + elif [ -d /var/opt/optimism/simulator ]; then + mv /var/opt/optimism/simulator /var/opt/optimism/rbuilder + else + mkdir -p /var/opt/optimism/rbuilder + fi + + chown -R op-rbuilder:optimism /var/opt/optimism/rbuilder + chmod 0750 /var/opt/optimism/rbuilder +fi + +if [ -f /var/opt/optimism/rbuilder/genesis.json.tar.gz.base64 ]; then + if [ -s /var/opt/optimism/rbuilder/genesis.json.tar.gz.base64 ]; then + if [ ! -f /var/opt/optimism/rbuilder/genesis.json ]; then + cat /var/opt/optimism/rbuilder/genesis.json.tar.gz.base64 | base64 -d | tar -xz -C /var/opt/optimism/rbuilder + chown op-rbuilder:optimism /var/opt/optimism/rbuilder/genesis.json + chmod 0640 /var/opt/optimism/rbuilder/genesis.json + fi + fi +fi + +if [ -f /var/opt/optimism/rbuilder/genesis.json ]; then + if [ ! -f /var/opt/optimism/rbuilder/db/database.version ]; then + sudo -u op-rbuilder /usr/bin/op-rbuilder init \ + --chain /var/opt/optimism/rbuilder/genesis.json \ + --color never \ + --datadir /var/opt/optimism/rbuilder \ + --log.stdout.format json + chown -R op-rbuilder:optimism /var/opt/optimism/rbuilder + fi +fi diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.postinst.chroot b/modules/l2/op-rbuilder-bproxy/mkosi.postinst.chroot new file mode 100755 index 00000000..13d9ec96 --- /dev/null +++ b/modules/l2/op-rbuilder-bproxy/mkosi.postinst.chroot @@ -0,0 +1,14 @@ +#!/bin/bash + +set -euxo pipefail + +# Enable systemd services + +systemctl add-wants minimal.target \ + node-healthchecker.service \ + tdx-quote-provider.service + +# Create users and groups + +groupadd -g 2000 optimism || true +useradd -u 2002 -g optimism -m -s /bin/bash op-rbuilder || true diff --git a/modules/l2/op-rbuilder/mkosi.build b/modules/l2/op-rbuilder/mkosi.build new file mode 100755 index 00000000..9f12363e --- /dev/null +++ b/modules/l2/op-rbuilder/mkosi.build @@ -0,0 +1,64 @@ +#!/bin/bash + +set -euxo pipefail + +ENV_YAML="$SRCDIR/modules/l2/op-rbuilder/mkosi.extra/etc/flashbots/op-rbuilder.yaml" + +NODE_HEALTHCHECKER_REF=$(mkosi-chroot yq -r '.node_healthchecker.git_reference' < "$ENV_YAML") +OP_RBUILDER_FEATURES=$(mkosi-chroot yq -r '.op_rbuilder.features // "default"' < "$ENV_YAML") +OP_RBUILDER_REF=$(mkosi-chroot yq -r '.op_rbuilder.git_reference' < "$ENV_YAML") +RPROXY_FEATURES=$(mkosi-chroot yq -r '.rproxy.features // ["default"] | sort | join(",")' < "$ENV_YAML") +RPROXY_REF=$(mkosi-chroot yq -r '.rproxy.git_reference' < "$ENV_YAML") +TDX_QUOTE_PROVIDER_REF=$(mkosi-chroot yq -r '.tdx_quote_provider.git_reference' < "$ENV_YAML") + +source scripts/make_git_package.sh +source scripts/build_rust_package.sh + +export CARGO_HOME="/cargo" +export RUSTUP_HOME="/rustup" +export PATH="$CARGO_HOME/bin:$PATH" +RUST_VERSION=$(mkosi-chroot yq -r '.rust.version' < "$ENV_YAML") +mkosi-chroot rustup toolchain install $RUST_VERSION +mkosi-chroot rustup default $RUST_VERSION + +# build op-rbuilder + +build_rust_package \ + "op-rbuilder" \ + "${OP_RBUILDER_REF}" \ + "https://github.com/flashbots/op-rbuilder.git" \ + "" "${OP_RBUILDER_FEATURES}" "-g" +chmod +x $DESTDIR/usr/bin/op-rbuilder + +# build tdx-quote-provider + +build_rust_package \ + "tdx-quote-provider" \ + "${TDX_QUOTE_PROVIDER_REF}" \ + "https://github.com/flashbots/op-rbuilder.git" \ + "" "" "-g" +chmod +x $DESTDIR/usr/bin/tdx-quote-provider + +# build rproxy + +make_git_package \ + "rproxy" \ + "${RPROXY_REF}" \ + "https://github.com/flashbots/rproxy.git" \ + "FEATURES=${RPROXY_FEATURES} TARGET=x86_64-unknown-linux-gnu ./build.sh" \ + "target/x86_64-unknown-linux-gnu/release/rproxy:/usr/bin/rproxy" +chmod +x $DESTDIR/usr/bin/rproxy + +# build node-healthchecker + +make_git_package \ + "node-healthchecker" \ + "${NODE_HEALTHCHECKER_REF}" \ + "https://github.com/flashbots/node-healthchecker.git" \ + 'go build -trimpath -ldflags "-s -w -X main.version=${NODE_HEALTHCHECKER_REF} -buildid=" -o ./bin/node-healthchecker github.com/flashbots/node-healthchecker/cmd' \ + "bin/node-healthchecker:/usr/bin/node-healthchecker" +chmod +x $DESTDIR/usr/bin/node-healthchecker + +# generate version + +cp $BUILDDIR/op-rbuilder.git $BUILDDIR/workload.git diff --git a/modules/l2/op-rbuilder/mkosi.conf b/modules/l2/op-rbuilder/mkosi.conf new file mode 100644 index 00000000..6ff09886 --- /dev/null +++ b/modules/l2/op-rbuilder/mkosi.conf @@ -0,0 +1,17 @@ +[Build] +WithNetwork=true + +[Content] +BuildScripts=modules/l2/op-rbuilder/mkosi.build +ExtraTrees=modules/l2/op-rbuilder/mkosi.extra +PostInstallationScripts=modules/l2/op-rbuilder/mkosi.postinst.chroot + +Packages=libtss2-dev + sudo + unzip + +BuildPackages=golang + libssl-dev + rustup + unzip + yq diff --git a/modules/l2/op-rbuilder/mkosi.extra/etc/default/prometheus-node-exporter b/modules/l2/op-rbuilder/mkosi.extra/etc/default/prometheus-node-exporter new file mode 100644 index 00000000..5f6e8585 --- /dev/null +++ b/modules/l2/op-rbuilder/mkosi.extra/etc/default/prometheus-node-exporter @@ -0,0 +1,7 @@ +# Set the command-line arguments to pass to the server. +ARGS="\ +--collector.systemd \ +--collector.systemd.unit-include=\".*(node-healthchecker|op-rbuilder|prometheus-node-exporter|prometheus-process-exporter|rproxy|vault-agent).*\" \ +--log.format=json \ +--web.listen-address=0.0.0.0:9100 \ +" diff --git a/modules/l2/op-rbuilder/mkosi.extra/etc/flashbots/op-rbuilder.yaml b/modules/l2/op-rbuilder/mkosi.extra/etc/flashbots/op-rbuilder.yaml new file mode 100644 index 00000000..79c62c6b --- /dev/null +++ b/modules/l2/op-rbuilder/mkosi.extra/etc/flashbots/op-rbuilder.yaml @@ -0,0 +1,10 @@ +node_healthchecker: + git_reference: v0.1.11 +op_rbuilder: + git_reference: op-rbuilder/v0.4.0 +rproxy: + git_reference: v0.0.11 +rust: + version: 1.94.0 +tdx_quote_provider: + git_reference: tdx-quote-provider/v0.1.0 diff --git a/modules/l2/op-rbuilder/mkosi.extra/etc/prometheus-process-exporter/config.yaml b/modules/l2/op-rbuilder/mkosi.extra/etc/prometheus-process-exporter/config.yaml new file mode 100644 index 00000000..4e58d4d1 --- /dev/null +++ b/modules/l2/op-rbuilder/mkosi.extra/etc/prometheus-process-exporter/config.yaml @@ -0,0 +1,13 @@ +process_names: + - name: node-healthchecker + cmdline: + - '^\/([-.0-9a-zA-Z]+\/)*node-healthchecker[-.0-9a-zA-Z]* ' + - name: op-rbuilder + cmdline: + - '^\/([-.0-9a-zA-Z]+\/)*op-rbuilder[-.0-9a-zA-Z]* ' + - name: rproxy + cmdline: + - '^\/([-.0-9a-zA-Z]+\/)*rproxy[-.0-9a-zA-Z]* ' + - name: vault-agent + cmdline: + - '^\/([-.0-9a-zA-Z]+\/)*vault[-.0-9a-zA-Z]* ' diff --git a/modules/l2/op-rbuilder/mkosi.extra/etc/sysconfig/automount-data.env b/modules/l2/op-rbuilder/mkosi.extra/etc/sysconfig/automount-data.env new file mode 100644 index 00000000..2963c609 --- /dev/null +++ b/modules/l2/op-rbuilder/mkosi.extra/etc/sysconfig/automount-data.env @@ -0,0 +1 @@ +AUTOMOUNT_PATH_DATA=/var/opt/optimism diff --git a/modules/l2/op-rbuilder/mkosi.extra/etc/systemd/system/node-healthchecker.service b/modules/l2/op-rbuilder/mkosi.extra/etc/systemd/system/node-healthchecker.service new file mode 100644 index 00000000..35ef6122 --- /dev/null +++ b/modules/l2/op-rbuilder/mkosi.extra/etc/systemd/system/node-healthchecker.service @@ -0,0 +1,25 @@ +[Unit] +Description=Blockchain node healthchecker +After=network.target +Wants=network.target + +[Service] +Type=simple +SyslogIdentifier=node-healthchecker +User=op-rbuilder +Group=optimism + +Restart=always +RestartSec=5 +TimeoutStopSec=60 + +ExecStart=/usr/bin/node-healthchecker serve \ + --healthcheck-block-age-threshold 10s \ + --healthcheck-timeout 500ms \ + --healthcheck-reth-base-url http://127.0.0.1:18645 \ + --healthcheck-unconditional-fail-duration 1m \ + --http-status-warning 200 \ + --server-listen-address 0.0.0.0:8080 + +[Install] +WantedBy=default.target diff --git a/modules/l2/op-rbuilder/mkosi.extra/etc/systemd/system/tdx-quote-provider.service b/modules/l2/op-rbuilder/mkosi.extra/etc/systemd/system/tdx-quote-provider.service new file mode 100644 index 00000000..e9aa03fd --- /dev/null +++ b/modules/l2/op-rbuilder/mkosi.extra/etc/systemd/system/tdx-quote-provider.service @@ -0,0 +1,26 @@ +[Install] +WantedBy=default.target + +[Unit] +Description=TDX quote provider +After=network.target +Wants=network.target + +[Service] +Type=simple +SyslogIdentifier=tdx-quote-provider +User=root +Group=root + +Restart=always +RestartSec=5 +TimeoutStopSec=60 + +ExecStart=/usr/bin/tdx-quote-provider \ + --service-host 127.0.0.1 \ + --service-port 5757 \ + --metrics \ + --metrics-host 0.0.0.0 \ + --metrics-port 9009 \ + --log-level trace \ + --log-format json diff --git a/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/discovery-secret.hcl b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/discovery-secret.hcl new file mode 100644 index 00000000..396f56c1 --- /dev/null +++ b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/discovery-secret.hcl @@ -0,0 +1,29 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + destination = "/var/opt/optimism/rbuilder/discovery-secret" + + user = "op-rbuilder" + group = "optimism" + perms = "0600" + + exec { + timeout = "60s" + + command = ["/bin/sh", "-c", + <<-EOT + # discovery-secret + chown op-rbuilder:optimism /var/opt/optimism/rbuilder + chmod 0750 /var/opt/optimism/rbuilder + systemctl restart op-rbuilder + EOT + ] + } + + contents = <<-EOT + ((- $node := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/[[ gcp.Meta "name" ]]" ).Data.data -)) + + ((- $node.el_nodekey -)) + EOT +} diff --git a/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/genesis.json.hcl b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/genesis.json.hcl new file mode 100644 index 00000000..f7920fcd --- /dev/null +++ b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/genesis.json.hcl @@ -0,0 +1,31 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + destination = "/var/opt/optimism/rbuilder/genesis.json.tar.gz.base64" + + user = "op-rbuilder" + group = "optimism" + perms = "0640" + + exec { + timeout = "60s" + + command = ["/bin/sh", "-c", + <<-EOT + # genesis.json + chown op-rbuilder:optimism /var/opt/optimism/rbuilder + chmod 0750 /var/opt/optimism/rbuilder + systemctl restart op-rbuilder + EOT + ] + } + + contents = <<-EOT + ((- $service := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_common[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data -)) + + ((- if $service.genesis_json -)) + ((- $service.genesis_json -)) + ((- end -)) + EOT +} diff --git a/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/jwtsecret.hcl b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/jwtsecret.hcl new file mode 100644 index 00000000..52f90602 --- /dev/null +++ b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/jwtsecret.hcl @@ -0,0 +1,16 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + destination = "/var/opt/optimism/jwtsecret" + + user = "root" + group = "optimism" + perms = "0440" + + contents = <<-EOT + ((- with secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/[[ gcp.Meta "name" ]]" -)) + (( .Data.data.jwt_secret )) + ((- end -)) + EOT +} diff --git a/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.env.hcl b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.env.hcl new file mode 100644 index 00000000..5118676a --- /dev/null +++ b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.env.hcl @@ -0,0 +1,48 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + destination = "/etc/sysconfig/op-rbuilder.env" + + user = "root" + group = "optimism" + perms = "0640" + + exec { + timeout = "60s" + + command = ["/bin/sh", "-c", + <<-EOT + # op-rbuilder.env + systemctl restart op-rbuilder + EOT + ] + } + + contents = <<-EOT + ((- printf "# %s\n\n" "op-rbuilder" -)) + + ((- $node := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/[[ gcp.Meta "name" ]]" ).Data.data -)) + ((- $service := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_common[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data -)) + + ((- if $node.builder_secret_key -)) + BUILDER_SECRET_KEY=(( $node.builder_secret_key ))(( "\n" )) + ((- end -)) + + ((- if $node.coinbase_secret_key -)) + COINBASE_SECRET_KEY=(( $node.coinbase_secret_key ))(( "\n" )) + ((- end -)) + + ((- if $service.otel_exporter_otlp_endpoint -)) + OTEL_EXPORTER_OTLP_ENDPOINT=(( $service.otel_exporter_otlp_endpoint ))(( "\n" )) + ((- end -)) + + ((- if $node.otel_exporter_otlp_headers -)) + OTEL_EXPORTER_OTLP_HEADERS=(( $node.otel_exporter_otlp_headers ))(( "\n" )) + ((- end -)) + + ((- if $service.otel_service_name -)) + OTEL_SERVICE_NAME=(( $service.otel_service_name ))(( "\n" )) + ((- end -)) + EOT +} diff --git a/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.ctmpl b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.ctmpl new file mode 100644 index 00000000..41c25948 --- /dev/null +++ b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.ctmpl @@ -0,0 +1,120 @@ +# op-rbuilder + +[Install] +WantedBy=default.target + +[Unit] +Description=op-rbuilder +After=network-online.target +Before=rproxy.service node-healthchecker.service +Requires=automount-data.service vault-agent.service +Wants=network-online.target + +[Service] +Type=simple +SyslogIdentifier=op-rbuilder +User=op-rbuilder +Group=optimism + +EnvironmentFile=-/etc/sysconfig/op-rbuilder.env +KillMode=control-group +KillSignal=SIGINT +OOMScoreAdjust=-500 +Restart=always +RestartSec=10s +TimeoutStopSec=120 + +Environment=HOME=/home/op-rbuilder +Environment=RUST_BACKTRACE=full + +ExecStartPre=+/usr/bin/init-op-rbuilder.sh + +((- $service := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_common[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data )) + +ExecStart=/usr/bin/op-rbuilder node \ + --authrpc.addr '0.0.0.0' \ + --authrpc.jwtsecret '/var/opt/optimism/jwtsecret' \ + --authrpc.port '18651' \ + ((- if $service.el_bootnodes ))(( range $idx, $enode := $service.el_bootnodes )) + --bootnodes '(( printf "%s" $enode ))' \ + ((- end ))(( end )) + ((- if $service.feat_revert_protection ))(( if $service.feat_revert_protection | parseBool )) + --builder.enable-revert-protection \ + ((- end ))(( end )) + --builder.extra-block-deadline-secs '30' \ + --builder.extradata 'Illuminate Dmocratize Dstribute' \ + --builder.log-pool-transactions \ + ((- if $service.genesis_json )) + --chain '/var/opt/optimism/rbuilder/genesis.json' \ + ((- else ))(( if $service.network_name )) + --chain '(( $service.network_name ))' \ + ((- end ))(( end )) + --color 'never' \ + --datadir '/var/opt/optimism/rbuilder' \ + --discovery.port '9200' \ + --enable-discv5-discovery \ + ((- if $service.feat_flashblocks ))(( if $service.feat_flashblocks | parseBool )) + --flashblocks.addr '0.0.0.0' \ + ((- if $service.feat_flashblocks_block_time )) + --flashblocks.block-time '(( $service.feat_flashblocks_block_time ))' \ + ((- end )) + --flashblocks.enabled \ + --flashblocks.port '11111' \ + ((- if $service.feat_flashblocks_number_contract_address )) + --flashblocks.number-contract-address '(( $service.feat_flashblocks_number_contract_address ))' \ + --flashblocks.number-contract-use-permit \ + ((- end )) + ((- end ))(( end )) + ((- if $service.feat_flashtestations ))(( if $service.feat_flashtestations | parseBool )) + ((- if $service.feat_flashtestations_builder_policy_address )) + --flashtestations.builder-policy-address '(( $service.feat_flashtestations_builder_policy_address ))' \ + ((- end )) + ((- if $service.feat_flashtestations_block_proofs ))(( if $service.feat_flashtestations_block_proofs | parseBool )) + --flashtestations.enable-block-proofs \ + ((- end ))(( end )) + --flashtestations.enabled \ + --flashtestations.quote-provider http://127.0.0.1:5757/attest \ + ((- if $service.feat_flashtestations_registry_address )) + --flashtestations.registry-address '(( $service.feat_flashtestations_registry_address ))' \ + ((- end )) + ((- if $service.feat_flashtestations_rpc_url )) + --flashtestations.rpc-url '(( $service.feat_flashtestations_rpc_url ))' \ + ((- end )) + ((- end ))(( end )) + --http \ + --http.addr '0.0.0.0' \ + --http.api 'admin,debug,eth,net,trace,txpool' \ + --http.corsdomain '*' \ + --http.port '18645' \ + --log.file.filter 'off' \ + --log.stdout.format 'json' \ + --metrics '0.0.0.0:9001' \ + --port '40404' \ + --rollup.disable-tx-pool-gossip \ + --rpc-max-connections '5000' \ + --rpc-max-logs-per-response '200000' \ + --rpc-max-request-size '150' \ + --rpc-max-response-size '1150' \ + --rpc-max-subscriptions-per-connection '10240' \ + --rpc-max-tracing-requests '250' \ + ((- if $service.feat_telemetry_sampling_ratio ))(( if $service.feat_telemetry_sampling_ratio )) + --telemetry.sampling-ratio '(( $service.feat_telemetry_sampling_ratio ))' \ + ((- end ))(( end )) + ((- if $service.el_static_peers ))(( range $idx, $enode := $service.el_static_peers )) + --trusted-peers '(( printf "%s" $enode ))' \ + ((- end ))(( end )) + --txpool.max-new-pending-txs-notifications '100000' \ + --txpool.max-new-txns '100000' \ + --txpool.max-pending-txns '100000' \ + --ws \ + --ws.addr '0.0.0.0' \ + --ws.api 'admin,debug,eth,net,trace,txpool' \ + --ws.origins '*' \ + --ws.port '8646' \ + ((- if $service.custom_flags ))(( range $idx, $flag := $service.custom_flags )) + (( printf "%s" $flag )) \ + ((- end ))(( end )) + +ExecStop=+/usr/bin/sh -c "kill -1 $( pgrep node-health ) | true" +ExecStop=+/usr/bin/sleep 15 +ExecStop=+/usr/bin/sh -c "PID=$( pgrep op-rbuilder ); if [ 0${PID} -gt 0 ]; then kill -2 ${PID}; while kill -0 ${PID} 2>/dev/null; do sleep 1; done; fi" diff --git a/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.hcl b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.hcl new file mode 100644 index 00000000..9696e71f --- /dev/null +++ b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.hcl @@ -0,0 +1,33 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + source = "/etc/vault-agent/op-rbuilder.service.ctmpl" + destination = "/etc/systemd/system/op-rbuilder.service" + + user = "root" + group = "root" + perms = "0644" + + exec { + timeout = "60s" + + command = ["/bin/sh", "-c", + <<-EOT + # op-rbuilder + + systemctl daemon-reload + systemctl add-wants minimal.target op-rbuilder.service + + # patterns longer than 15 chars result in 0 matches + PID=$( pgrep node-health ); if [ 0${PID} -gt 0 ]; then kill -1 ${PID} || true; fi + sleep 5 + + PID=$( pgrep rproxy ); if [ 0${PID} -gt 0 ]; then kill -1 ${PID} || true; fi + + systemctl restart op-rbuilder.service + systemctl restart node-healthchecker.service + EOT + ] + } +} diff --git a/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/rproxy-tls.crt.hcl b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/rproxy-tls.crt.hcl new file mode 100644 index 00000000..57810187 --- /dev/null +++ b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/rproxy-tls.crt.hcl @@ -0,0 +1,28 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + destination = "/etc/rproxy/tls.crt" + + user = "op-rbuilder" + group = "optimism" + perms = "0640" + + exec { + timeout = "60s" + + command = ["/bin/sh", "-c", + <<-EOT + systemctl restart rproxy + EOT + ] + } + + contents = <<-EOT + ((- $tls_crt := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_tls[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data.tls_crt -)) + + ((- if $tls_crt -)) + (( $tls_crt )) + ((- end -)) + EOT +} diff --git a/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/rproxy-tls.key.hcl b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/rproxy-tls.key.hcl new file mode 100644 index 00000000..581f9a70 --- /dev/null +++ b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/rproxy-tls.key.hcl @@ -0,0 +1,28 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + destination = "/etc/rproxy/tls.key" + + user = "op-rbuilder" + group = "optimism" + perms = "0640" + + exec { + timeout = "60s" + + command = ["/bin/sh", "-c", + <<-EOT + systemctl restart rproxy + EOT + ] + } + + contents = <<-EOT + ((- $tls_key := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_tls[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data.tls_key -)) + + ((- if $tls_key -)) + (( $tls_key )) + ((- end -)) + EOT +} diff --git a/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/rproxy.service.ctmpl b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/rproxy.service.ctmpl new file mode 100644 index 00000000..941ece2c --- /dev/null +++ b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/rproxy.service.ctmpl @@ -0,0 +1,77 @@ +[Install] +WantedBy=default.target + +[Unit] +Description=L2 builder proxy +After=network.target +Wants=network.target + +[Service] +Type=simple +SyslogIdentifier=rproxy +User=op-rbuilder +Group=optimism + +Restart=always +RestartSec=5 +TimeoutStopSec=60 + +ExecStartPre=+/usr/bin/mkdir -p /etc/rproxy +ExecStartPre=+/usr/bin/chown -R op-rbuilder:optimism /etc/rproxy + +((- $service := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_common[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data )) +((- $tls_crt := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_tls[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data.tls_crt )) +((- $tls_key := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_tls[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data.tls_key )) + +ExecStart=/usr/bin/rproxy \ + --authrpc-backend 'http://127.0.0.1:18651' \ + --authrpc-enabled \ + --authrpc-listen-address '0.0.0.0:8651' \ + --authrpc-log-proxied-requests \ + --authrpc-log-proxied-responses \ + --authrpc-log-sanitise \ + --authrpc-max-request-size-mb '150' \ + --authrpc-max-response-size-mb '1150' \ + ((- if $service.authrpc_peers )) + ((- range $i, $e := $service.authrpc_peers )) + --authrpc-mirroring-peer '(( printf "%s" $e ))' \ + ((- end )) + --authrpc-remove-backend-from-mirroring-peers \ + ((- end )) + --circuit-breaker-reset-continuously \ + --circuit-breaker-url 'http://127.0.0.1:8080' \ + --flashblocks-backend 'ws://127.0.0.1:11111' \ + --flashblocks-enabled \ + --flashblocks-listen-address '0.0.0.0:1111' \ + --flashblocks-log-backend-messages \ + --flashblocks-log-client-messages \ + --flashblocks-log-sanitise \ + --metrics-listen-address '0.0.0.0:6786' \ + --rpc-backend 'http://127.0.0.1:18645' \ + --rpc-enabled \ + --rpc-listen-address '0.0.0.0:8645' \ + --rpc-log-proxied-requests \ + --rpc-log-proxied-responses \ + --rpc-log-sanitise \ + --rpc-max-request-size-mb '150' \ + --rpc-max-response-size-mb '1150' \ + ((- if $service.rpc_peers )) + --rpc-mirror-errored-requests \ + ((- range $idx, $enode := $service.rpc_peers )) + --rpc-mirroring-peer '(( printf "%s" $enode ))' \ + ((- end )) + --rpc-remove-backend-from-mirroring-peers \ + ((- end )) + ((- if $tls_crt )) + --tls-certificate '/etc/rproxy/tls.crt' \ + ((- end )) + ((- if $tls_key )) + --tls-key '/etc/rproxy/tls.key' \ + ((- end )) + ((- if $service.rproxy_custom_flags ))(( range $idx, $flag := $service.rproxy_custom_flags )) + (( printf "%s" $flag )) \ + ((- end ))(( end )) + +ExecStop=+/usr/bin/sh -c "kill -1 $( pgrep node-health ) | true" +ExecStop=+/usr/bin/sleep 15 +ExecStop=+/usr/bin/sh -c "PID=$( pgrep rproxy ); if [ 0${PID} -gt 0 ]; then kill -2 ${PID}; while kill -0 ${PID} 2>/dev/null; do sleep 1; done; fi" diff --git a/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/rproxy.service.hcl b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/rproxy.service.hcl new file mode 100644 index 00000000..0e1a979a --- /dev/null +++ b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/rproxy.service.hcl @@ -0,0 +1,23 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + source = "/etc/vault-agent/rproxy.service.ctmpl" + destination = "/etc/systemd/system/rproxy.service" + + user = "root" + group = "root" + perms = "0644" + + exec { + timeout = "60s" + + command = ["/bin/sh", "-c", + <<-EOT + systemctl daemon-reload + systemctl add-wants minimal.target rproxy.service + systemctl restart rproxy.service + EOT + ] + } +} diff --git a/modules/l2/op-rbuilder/mkosi.extra/usr/bin/init-op-rbuilder.sh b/modules/l2/op-rbuilder/mkosi.extra/usr/bin/init-op-rbuilder.sh new file mode 100755 index 00000000..79307b98 --- /dev/null +++ b/modules/l2/op-rbuilder/mkosi.extra/usr/bin/init-op-rbuilder.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +set -eu + +if [ ! -d /home/op-rbuilder ]; then + mkdir -p /home/op-rbuilder + chown -R op-rbuilder:optimism /home/op-rbuilder + chmod 0750 /home/op-rbuilder +fi + +if [ ! -d /var/opt/optimism/rbuilder ]; then + if [ -d /var/opt/optimism/unichain-builder ]; then + mv /var/opt/optimism/unichain-builder /var/opt/optimism/rbuilder + elif [ -d /var/opt/optimism/simulator ]; then + mv /var/opt/optimism/simulator /var/opt/optimism/rbuilder + else + mkdir -p /var/opt/optimism/rbuilder + fi + + chown -R op-rbuilder:optimism /var/opt/optimism/rbuilder + chmod 0750 /var/opt/optimism/rbuilder +fi + +if [ -f /var/opt/optimism/rbuilder/genesis.json.tar.gz.base64 ]; then + if [ -s /var/opt/optimism/rbuilder/genesis.json.tar.gz.base64 ]; then + if [ ! -f /var/opt/optimism/rbuilder/genesis.json ]; then + cat /var/opt/optimism/rbuilder/genesis.json.tar.gz.base64 | base64 -d | tar -xz -C /var/opt/optimism/rbuilder + chown op-rbuilder:optimism /var/opt/optimism/rbuilder/genesis.json + chmod 0640 /var/opt/optimism/rbuilder/genesis.json + fi + fi +fi + +if [ -f /var/opt/optimism/rbuilder/genesis.json ]; then + if [ ! -f /var/opt/optimism/rbuilder/db/database.version ]; then + sudo -u op-rbuilder /usr/bin/op-rbuilder init \ + --chain /var/opt/optimism/rbuilder/genesis.json \ + --color never \ + --datadir /var/opt/optimism/rbuilder \ + --log.stdout.format json + chown -R op-rbuilder:optimism /var/opt/optimism/rbuilder + fi +fi diff --git a/modules/l2/op-rbuilder/mkosi.postinst.chroot b/modules/l2/op-rbuilder/mkosi.postinst.chroot new file mode 100755 index 00000000..13d9ec96 --- /dev/null +++ b/modules/l2/op-rbuilder/mkosi.postinst.chroot @@ -0,0 +1,14 @@ +#!/bin/bash + +set -euxo pipefail + +# Enable systemd services + +systemctl add-wants minimal.target \ + node-healthchecker.service \ + tdx-quote-provider.service + +# Create users and groups + +groupadd -g 2000 optimism || true +useradd -u 2002 -g optimism -m -s /bin/bash op-rbuilder || true diff --git a/modules/l2/simulator/mkosi.build b/modules/l2/simulator/mkosi.build new file mode 100755 index 00000000..4f700923 --- /dev/null +++ b/modules/l2/simulator/mkosi.build @@ -0,0 +1,56 @@ +#!/bin/bash + +set -euxo pipefail + +ENV_YAML="$SRCDIR/modules/l2/simulator/mkosi.extra/etc/flashbots/simulator.yaml" + +NODE_HEALTHCHECKER_REF=$(mkosi-chroot yq -r '.node_healthchecker.git_reference' < "$ENV_YAML") +RPROXY_FEATURES=$(mkosi-chroot yq -r '.rproxy.features // ["default"] | sort | join(",")' < "$ENV_YAML") +RPROXY_REF=$(mkosi-chroot yq -r '.rproxy.git_reference' < "$ENV_YAML") +SIMULATOR_FEATURES=$(mkosi-chroot yq -r '.simulator.features // ""' < "$ENV_YAML") +SIMULATOR_REF=$(mkosi-chroot yq -r '.simulator.git_reference' < "$ENV_YAML") +TDX_QUOTE_PROVIDER_REF=$(mkosi-chroot yq -r '.tdx_quote_provider.git_reference' < "$ENV_YAML") + +source scripts/make_git_package.sh +source scripts/build_rust_package.sh + +export CARGO_HOME="/cargo" +export RUSTUP_HOME="/rustup" +export PATH="$CARGO_HOME/bin:$PATH" +RUST_VERSION=$(mkosi-chroot yq -r '.rust.version' < "$ENV_YAML") +mkosi-chroot rustup toolchain install $RUST_VERSION +mkosi-chroot rustup default $RUST_VERSION + +# build simulator + +build_rust_package \ + "signal-boost" \ + "${SIMULATOR_REF}" \ + "https://github.com/flashbots/signal-boost.git" \ + "" "${SIMULATOR_FEATURES}" "-g" +mv $DESTDIR/usr/bin/signal-boost $DESTDIR/usr/bin/simulator +chmod +x $DESTDIR/usr/bin/simulator + +# build rproxy + +make_git_package \ + "rproxy" \ + "${RPROXY_REF}" \ + "https://github.com/flashbots/rproxy.git" \ + "FEATURES=${RPROXY_FEATURES} TARGET=x86_64-unknown-linux-gnu ./build.sh" \ + "target/x86_64-unknown-linux-gnu/release/rproxy:/usr/bin/rproxy" +chmod +x $DESTDIR/usr/bin/rproxy + +# build node-healthchecker + +make_git_package \ + "node-healthchecker" \ + "${NODE_HEALTHCHECKER_REF}" \ + "https://github.com/flashbots/node-healthchecker.git" \ + 'go build -trimpath -ldflags "-s -w -X main.version=${NODE_HEALTHCHECKER_REF} -buildid=" -o ./bin/node-healthchecker github.com/flashbots/node-healthchecker/cmd' \ + "bin/node-healthchecker:/usr/bin/node-healthchecker" +chmod +x $DESTDIR/usr/bin/node-healthchecker + +# generate version + +cp $BUILDDIR/signal-boost.git $BUILDDIR/workload.git diff --git a/modules/l2/simulator/mkosi.conf b/modules/l2/simulator/mkosi.conf new file mode 100644 index 00000000..53e2c2aa --- /dev/null +++ b/modules/l2/simulator/mkosi.conf @@ -0,0 +1,16 @@ +[Build] +WithNetwork=true + +[Content] +BuildScripts=modules/l2/simulator/mkosi.build +ExtraTrees=modules/l2/simulator/mkosi.extra +PostInstallationScripts=modules/l2/simulator/mkosi.postinst.chroot + +Packages=sudo + unzip + +BuildPackages=golang + libssl-dev + rustup + unzip + yq diff --git a/modules/l2/simulator/mkosi.extra/etc/default/prometheus-node-exporter b/modules/l2/simulator/mkosi.extra/etc/default/prometheus-node-exporter new file mode 100644 index 00000000..7e6bdaee --- /dev/null +++ b/modules/l2/simulator/mkosi.extra/etc/default/prometheus-node-exporter @@ -0,0 +1,7 @@ +# Set the command-line arguments to pass to the server. +ARGS="\ +--collector.systemd \ +--collector.systemd.unit-include=\".*(node-healthchecker|prometheus-node-exporter|prometheus-process-exporter|rproxy|simulator|vault-agent).*\" \ +--log.format=json \ +--web.listen-address=0.0.0.0:9100 \ +" diff --git a/modules/l2/simulator/mkosi.extra/etc/flashbots/simulator.yaml b/modules/l2/simulator/mkosi.extra/etc/flashbots/simulator.yaml new file mode 100644 index 00000000..6b93125c --- /dev/null +++ b/modules/l2/simulator/mkosi.extra/etc/flashbots/simulator.yaml @@ -0,0 +1,8 @@ +node_healthchecker: + git_reference: v0.1.11 +simulator: + git_reference: trunk +rproxy: + git_reference: v0.0.11 +rust: + version: 1.93.0 diff --git a/modules/l2/simulator/mkosi.extra/etc/prometheus-process-exporter/config.yaml b/modules/l2/simulator/mkosi.extra/etc/prometheus-process-exporter/config.yaml new file mode 100644 index 00000000..6aa4cab3 --- /dev/null +++ b/modules/l2/simulator/mkosi.extra/etc/prometheus-process-exporter/config.yaml @@ -0,0 +1,13 @@ +process_names: + - name: node-healthchecker + cmdline: + - '^\/([-.0-9a-zA-Z]+\/)*node-healthchecker[-.0-9a-zA-Z]* ' + - name: rproxy + cmdline: + - '^\/([-.0-9a-zA-Z]+\/)*rproxy[-.0-9a-zA-Z]* ' + - name: simulator + cmdline: + - '^\/([-.0-9a-zA-Z]+\/)*simulator[-.0-9a-zA-Z]* ' + - name: vault-agent + cmdline: + - '^\/([-.0-9a-zA-Z]+\/)*vault[-.0-9a-zA-Z]* ' diff --git a/modules/l2/simulator/mkosi.extra/etc/sysconfig/automount-data.env b/modules/l2/simulator/mkosi.extra/etc/sysconfig/automount-data.env new file mode 100644 index 00000000..2963c609 --- /dev/null +++ b/modules/l2/simulator/mkosi.extra/etc/sysconfig/automount-data.env @@ -0,0 +1 @@ +AUTOMOUNT_PATH_DATA=/var/opt/optimism diff --git a/modules/l2/simulator/mkosi.extra/etc/systemd/system/node-healthchecker.service b/modules/l2/simulator/mkosi.extra/etc/systemd/system/node-healthchecker.service new file mode 100644 index 00000000..5375c821 --- /dev/null +++ b/modules/l2/simulator/mkosi.extra/etc/systemd/system/node-healthchecker.service @@ -0,0 +1,25 @@ +[Unit] +Description=Blockchain node healthchecker +After=network.target +Wants=network.target + +[Service] +Type=simple +SyslogIdentifier=node-healthchecker +User=simulator +Group=optimism + +Restart=always +RestartSec=5 +TimeoutStopSec=60 + +ExecStart=/usr/bin/node-healthchecker serve \ + --healthcheck-block-age-threshold 10s \ + --healthcheck-timeout 500ms \ + --healthcheck-reth-base-url http://127.0.0.1:18645 \ + --healthcheck-unconditional-fail-duration 1m \ + --http-status-warning 200 \ + --server-listen-address 0.0.0.0:8080 + +[Install] +WantedBy=default.target diff --git a/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/discovery-secret.hcl b/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/discovery-secret.hcl new file mode 100644 index 00000000..e6150345 --- /dev/null +++ b/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/discovery-secret.hcl @@ -0,0 +1,29 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + destination = "/var/opt/optimism/simulator/discovery-secret" + + user = "simulator" + group = "optimism" + perms = "0600" + + exec { + timeout = "60s" + + command = ["/bin/sh", "-c", + <<-EOT + # discovery-secret + chown simulator:optimism /var/opt/optimism/simulator + chmod 0750 /var/opt/optimism/simulator + systemctl restart simulator + EOT + ] + } + + contents = <<-EOT + ((- $node := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/[[ gcp.Meta "name" ]]" ).Data.data -)) + + ((- $node.el_nodekey -)) + EOT +} diff --git a/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/genesis.json.hcl b/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/genesis.json.hcl new file mode 100644 index 00000000..6d6ab77b --- /dev/null +++ b/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/genesis.json.hcl @@ -0,0 +1,31 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + destination = "/var/opt/optimism/simulator/genesis.json.tar.gz.base64" + + user = "simulator" + group = "optimism" + perms = "0640" + + exec { + timeout = "60s" + + command = ["/bin/sh", "-c", + <<-EOT + # genesis.json + chown simulator:optimism /var/opt/optimism/simulator + chmod 0750 /var/opt/optimism/simulator + systemctl restart simulator + EOT + ] + } + + contents = <<-EOT + ((- $service := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_common[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data -)) + + ((- if $service.genesis_json -)) + ((- $service.genesis_json -)) + ((- end -)) + EOT +} diff --git a/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/jwtsecret.hcl b/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/jwtsecret.hcl new file mode 100644 index 00000000..52f90602 --- /dev/null +++ b/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/jwtsecret.hcl @@ -0,0 +1,16 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + destination = "/var/opt/optimism/jwtsecret" + + user = "root" + group = "optimism" + perms = "0440" + + contents = <<-EOT + ((- with secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/[[ gcp.Meta "name" ]]" -)) + (( .Data.data.jwt_secret )) + ((- end -)) + EOT +} diff --git a/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/rproxy-tls.crt.hcl b/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/rproxy-tls.crt.hcl new file mode 100644 index 00000000..55660258 --- /dev/null +++ b/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/rproxy-tls.crt.hcl @@ -0,0 +1,28 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + destination = "/etc/rproxy/tls.crt" + + user = "simulator" + group = "optimism" + perms = "0640" + + exec { + timeout = "60s" + + command = ["/bin/sh", "-c", + <<-EOT + systemctl restart rproxy + EOT + ] + } + + contents = <<-EOT + ((- $tls_crt := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_tls[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data.tls_crt -)) + + ((- if $tls_crt -)) + (( $tls_crt )) + ((- end -)) + EOT +} diff --git a/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/rproxy-tls.key.hcl b/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/rproxy-tls.key.hcl new file mode 100644 index 00000000..ef243150 --- /dev/null +++ b/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/rproxy-tls.key.hcl @@ -0,0 +1,28 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + destination = "/etc/rproxy/tls.key" + + user = "simulator" + group = "optimism" + perms = "0640" + + exec { + timeout = "60s" + + command = ["/bin/sh", "-c", + <<-EOT + systemctl restart rproxy + EOT + ] + } + + contents = <<-EOT + ((- $tls_key := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_tls[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data.tls_key -)) + + ((- if $tls_key -)) + (( $tls_key )) + ((- end -)) + EOT +} diff --git a/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/rproxy.service.ctmpl b/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/rproxy.service.ctmpl new file mode 100644 index 00000000..0195a673 --- /dev/null +++ b/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/rproxy.service.ctmpl @@ -0,0 +1,69 @@ +[Install] +WantedBy=default.target + +[Unit] +Description=L2 builder proxy +After=network.target +Wants=network.target + +[Service] +Type=simple +SyslogIdentifier=rproxy +User=simulator +Group=optimism + +Restart=always +RestartSec=5 +TimeoutStopSec=60 + +ExecStartPre=+/usr/bin/mkdir -p /etc/rproxy +ExecStartPre=+/usr/bin/chown -R simulator:optimism /etc/rproxy + +((- $service := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_common[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data )) +((- $tls_crt := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_tls[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data.tls_crt )) +((- $tls_key := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_tls[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data.tls_key )) + +ExecStart=/usr/bin/rproxy \ + --authrpc-backend 'http://127.0.0.1:18651' \ + --authrpc-enabled \ + --authrpc-listen-address '0.0.0.0:8651' \ + --authrpc-log-proxied-requests \ + --authrpc-log-proxied-responses \ + --authrpc-log-sanitise \ + --authrpc-max-request-size-mb '150' \ + --authrpc-max-response-size-mb '1150' \ + ((- if $service.authrpc_peers )) + ((- range $i, $e := $service.authrpc_peers )) + --authrpc-mirroring-peer '(( printf "%s" $e ))' \ + ((- end )) + --authrpc-remove-backend-from-mirroring-peers \ + ((- end )) + --metrics-listen-address '0.0.0.0:6786' \ + --rpc-backend 'http://127.0.0.1:18645' \ + --rpc-enabled \ + --rpc-listen-address '0.0.0.0:8645' \ + --rpc-log-proxied-requests \ + --rpc-log-proxied-responses \ + --rpc-log-sanitise \ + --rpc-max-request-size-mb '150' \ + --rpc-max-response-size-mb '1150' \ + ((- if $service.rpc_peers )) + --rpc-mirror-errored-requests \ + ((- range $idx, $enode := $service.rpc_peers )) + --rpc-mirroring-peer '(( printf "%s" $enode ))' \ + ((- end )) + --rpc-remove-backend-from-mirroring-peers \ + ((- end )) + ((- if $tls_crt )) + --tls-certificate '/etc/rproxy/tls.crt' \ + ((- end )) + ((- if $tls_key )) + --tls-key '/etc/rproxy/tls.key' \ + ((- end )) + ((- if $service.rproxy_custom_flags ))(( range $idx, $flag := $service.rproxy_custom_flags )) + (( printf "%s" $flag )) \ + ((- end ))(( end )) + +ExecStop=+/usr/bin/sh -c "kill -1 $( pgrep node-health ) | true" +ExecStop=+/usr/bin/sleep 15 +ExecStop=+/usr/bin/sh -c "PID=$( pgrep rproxy ); if [ 0${PID} -gt 0 ]; then kill -2 ${PID}; while kill -0 ${PID} 2>/dev/null; do sleep 1; done; fi" diff --git a/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/rproxy.service.hcl b/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/rproxy.service.hcl new file mode 100644 index 00000000..0e1a979a --- /dev/null +++ b/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/rproxy.service.hcl @@ -0,0 +1,23 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + source = "/etc/vault-agent/rproxy.service.ctmpl" + destination = "/etc/systemd/system/rproxy.service" + + user = "root" + group = "root" + perms = "0644" + + exec { + timeout = "60s" + + command = ["/bin/sh", "-c", + <<-EOT + systemctl daemon-reload + systemctl add-wants minimal.target rproxy.service + systemctl restart rproxy.service + EOT + ] + } +} diff --git a/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/simulator.env.hcl b/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/simulator.env.hcl new file mode 100644 index 00000000..7a55e4d8 --- /dev/null +++ b/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/simulator.env.hcl @@ -0,0 +1,31 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + destination = "/etc/sysconfig/simulator.env" + + user = "root" + group = "optimism" + perms = "0640" + + exec { + timeout = "60s" + + command = ["/bin/sh", "-c", + <<-EOT + # simulator.env + systemctl restart simulator + EOT + ] + } + + contents = <<-EOT + ((- printf "# %s\n\n" "simulator" -)) + + ((- $node := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/[[ gcp.Meta "name" ]]" ).Data.data -)) + + ((- if $node.clickhouse_password -)) + CLICKHOUSE_PASSWORD=(( $node.clickhouse_password ))(( "\n" )) + ((- end -)) + EOT +} diff --git a/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/simulator.service.ctmpl b/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/simulator.service.ctmpl new file mode 100644 index 00000000..fb4b0f1e --- /dev/null +++ b/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/simulator.service.ctmpl @@ -0,0 +1,114 @@ +# simulator + +[Install] +WantedBy=default.target + +[Unit] +Description=simulator +After=network-online.target +Before=rproxy.service node-healthchecker.service +Requires=automount-data.service vault-agent.service +Wants=network-online.target + +[Service] +Type=simple +SyslogIdentifier=simulator +User=simulator +Group=optimism + +EnvironmentFile=-/etc/sysconfig/simulator.env +KillMode=control-group +KillSignal=SIGINT +OOMScoreAdjust=-500 +Restart=always +RestartSec=10s +TimeoutStopSec=600 + +Environment=HOME=/home/simulator +Environment=RUST_BACKTRACE=full + +ExecStartPre=+/usr/bin/init-simulator.sh + +((- $node := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/[[ gcp.Meta "name" ]]" ).Data.data )) +((- $service := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_common[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data )) + +ExecStart=/usr/bin/simulator node \ + --authrpc.addr '0.0.0.0' \ + --authrpc.jwtsecret '/var/opt/optimism/jwtsecret' \ + --authrpc.port '18651' \ + --backrun.rpc.addr '0.0.0.0' \ + --backrun.rpc.port '8601' \ + ((- if $service.el_bootnodes )) + --bootnodes '(( range $i, $e := $service.el_bootnodes ))(( if $i )),(( end ))(( printf "%s" $e ))(( end ))' \ + ((- end )) + ((- if $service.builder_rpc_url )) + --builder.rpc-url '(( $service.builder_rpc_url ))' \ + ((- end )) + ((- if $service.genesis_json )) + --chain '/var/opt/optimism/simulator/genesis.json' \ + ((- else ))(( if $service.network_name )) + --chain '(( $service.network_name | trimSuffix "-mainnet" ))' \ + ((- end ))(( end )) + ((- if $service.feat_clickhouse ))(( if $service.feat_clickhouse | parseBool )) + ((- if $service.feat_clickhouse_database )) + --clickhouse.database '(( $service.feat_clickhouse_database ))' \ + ((- end )) + ((- if $service.feat_clickhouse_url )) + --clickhouse.url '(( $service.feat_clickhouse_url ))' \ + ((- end )) + ((- if $service.feat_clickhouse_user )) + --clickhouse.user '(( $service.feat_clickhouse_user ))' \ + ((- end )) + ((- end ))(( end )) + --color 'never' \ + --datadir '/var/opt/optimism/simulator' \ + --discovery.port '9200' \ + --enable-discv5-discovery \ + ((- if $node.flashblocks_url )) + --flashblocks.ws (( $node.flashblocks_url )) \ + ((- else if $service.flashblocks_url )) + --flashblocks.ws (( $service.flashblocks_url )) \ + ((- end )) + --http \ + --http.addr '0.0.0.0' \ + --http.api 'admin,debug,eth,net,trace,txpool' \ + --http.corsdomain '*' \ + --http.port '18645' \ + --log.stdout.format 'json' \ + --log-pool-transactions \ + --metrics '0.0.0.0:9001' \ + --port '40404' \ + --rollup.disable-tx-pool-gossip \ + --rpc-max-connections '5000' \ + --rpc-max-logs-per-response '200000' \ + --rpc-max-request-size '150' \ + --rpc-max-response-size '1150' \ + --rpc-max-subscriptions-per-connection '10240' \ + --rpc-max-tracing-requests '250' \ + --simulation-gas-limit '25000000' \ + --simulation-nonce-limit '16' \ + --simulation-workers '10' \ + --state-diff.rpc.addr '0.0.0.0' \ + --state-diff.rpc.port '8600' \ + ((- if $service.el_static_peers ))(( range $idx, $enode := $service.el_static_peers )) + --trusted-peers '(( printf "%s" $enode ))' \ + ((- end ))(( end )) + --txpool.lifetime '30' \ + --txpool.max-account-slots '128' \ + --txpool.max-new-pending-txs-notifications '100000' \ + --txpool.max-new-txns '100000' \ + --txpool.max-pending-txns '100000' \ + --txpool.max-tx-gas '25000000' \ + --txpool.nolocals \ + --ws \ + --ws.addr '0.0.0.0' \ + --ws.api 'admin,debug,eth,net,trace,txpool' \ + --ws.origins '*' \ + --ws.port '8646' \ + ((- if $service.custom_flags ))(( range $i, $v := $service.custom_flags )) + (( printf "%s" $v )) \ + ((- end ))(( end )) + +ExecStop=+/usr/bin/sh -c "kill -1 $( pgrep node-health ) | true" +ExecStop=+/usr/bin/sleep 15 +ExecStop=+/usr/bin/sh -c "PID=$( pgrep simulator ); if [ 0${PID} -gt 0 ]; then kill -2 ${PID}; while kill -0 ${PID} 2>/dev/null; do sleep 1; done; fi" diff --git a/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/simulator.service.hcl b/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/simulator.service.hcl new file mode 100644 index 00000000..aef5549e --- /dev/null +++ b/modules/l2/simulator/mkosi.extra/etc/vault-agent/gomplate/simulator.service.hcl @@ -0,0 +1,33 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + source = "/etc/vault-agent/simulator.service.ctmpl" + destination = "/etc/systemd/system/simulator.service" + + user = "root" + group = "root" + perms = "0644" + + exec { + timeout = "60s" + + command = ["/bin/sh", "-c", + <<-EOT + # simulator + + systemctl daemon-reload + systemctl add-wants minimal.target simulator.service + + # patterns longer than 15 chars result in 0 matches + PID=$( pgrep node-health ); if [ 0${PID} -gt 0 ]; then kill -1 ${PID} || true; fi + sleep 5 + + PID=$( pgrep rproxy ); if [ 0${PID} -gt 0 ]; then kill -1 ${PID} || true; fi + + systemctl restart simulator.service + systemctl restart node-healthchecker.service + EOT + ] + } +} diff --git a/modules/l2/simulator/mkosi.extra/usr/bin/init-simulator.sh b/modules/l2/simulator/mkosi.extra/usr/bin/init-simulator.sh new file mode 100755 index 00000000..db5e6d13 --- /dev/null +++ b/modules/l2/simulator/mkosi.extra/usr/bin/init-simulator.sh @@ -0,0 +1,41 @@ +#!/bin/sh + +set -eu + +if [ ! -d /home/simulator ]; then + mkdir -p /home/simulator + chown -R simulator:optimism /home/simulator + chmod 0750 /home/simulator +fi + +if [ ! -d /var/opt/optimism/simulator ]; then + if [ -d /var/opt/optimism/rbuilder ]; then + mv /var/opt/optimism/rbuilder /var/opt/optimism/simulator + else + mkdir -p /var/opt/optimism/simulator + fi + + chown -R simulator:optimism /var/opt/optimism/simulator + chmod 0750 /var/opt/optimism/simulator +fi + +if [ -f /var/opt/optimism/simulator/genesis.json.tar.gz.base64 ]; then + if [ -s /var/opt/optimism/simulator/genesis.json.tar.gz.base64 ]; then + if [ ! -f /var/opt/optimism/simulator/genesis.json ]; then + cat /var/opt/optimism/simulator/genesis.json.tar.gz.base64 | base64 -d | tar -xz -C /var/opt/optimism/simulator + chown simulator:optimism /var/opt/optimism/simulator/genesis.json + chmod 0640 /var/opt/optimism/simulator/genesis.json + fi + fi +fi + +if [ -f /var/opt/optimism/simulator/genesis.json ]; then + if [ ! -f /var/opt/optimism/simulator/db/database.version ]; then + sudo -u simulator /usr/bin/simulator init \ + --chain /var/opt/optimism/simulator/genesis.json \ + --color never \ + --datadir /var/opt/optimism/simulator \ + --log.stdout.format json + chown -R simulator:optimism /var/opt/optimism/simulator + fi +fi diff --git a/modules/l2/simulator/mkosi.postinst.chroot b/modules/l2/simulator/mkosi.postinst.chroot new file mode 100755 index 00000000..80ed0deb --- /dev/null +++ b/modules/l2/simulator/mkosi.postinst.chroot @@ -0,0 +1,13 @@ +#!/bin/bash + +set -euxo pipefail + +# Enable systemd services + +systemctl add-wants minimal.target \ + node-healthchecker.service + +# Create users and groups + +groupadd -g 2000 optimism || true +useradd -u 2003 -g optimism -m -s /bin/bash simulator || true diff --git a/scripts/build_rust_package.sh b/scripts/build_rust_package.sh index f1636d9f..3943c02a 100755 --- a/scripts/build_rust_package.sh +++ b/scripts/build_rust_package.sh @@ -19,18 +19,11 @@ build_rust_package() { return fi - # If binary is cached, skip compilation - local safe_version="${version//\//_}" - local cached_binary="$BUILDDIR/${package}-${safe_version}" - if [ -f "$cached_binary" ]; then - echo "Using cached binary for $package version $version" - cp "$cached_binary" "$dest_path" - return - fi - # Clone the repository local build_dir="$BUILDROOT/build/$package" mkdir -p "$build_dir" + set +x # don't leak github token into logs + echo "Cloning ${git_url}" if [ -f "$BUILDDIR/.ghtoken" ]; then git_url="${git_url/#https:\/\/github.com/https:\/\/x-access-token:$(cat "$BUILDDIR/.ghtoken")@github.com}" fi @@ -39,6 +32,29 @@ build_rust_package() { git clone "$git_url" "$build_dir" && git -C "$build_dir" checkout "$version" ) + set -x + + # Get the git reference + local git_describe=$( git -C "$build_dir" describe --always --long --tags ) + printf "${git_describe#$package/}" > "$BUILDDIR/$package.git" + + # If binary is cached, skip compilation + if [ -n "${extra_features}" ]; then + local cached_binary="$BUILDDIR/${package}-${git_describe#${package}/}-${extra_features}/${package}" + else + local cached_binary="$BUILDDIR/${package}-${git_describe#${package}/}/${package}" + fi + local cached_binary="${cached_binary//,/-}" + if [ -f "$cached_binary" ]; then + echo "Using cached binary for $package version $version" + if [ -n "${extra_features}" ]; then + echo "| \`$package\` | \`$version\` (\`$git_describe\`, features: ${extra_features}) | reused from cache | \`$( du -sh $cached_binary | cut -f1 )\` | |" ">> $BUILDDIR/manifest.md" + else + echo "| \`$package\` | \`$version\` (\`$git_describe\`) | reused from cache | \`$( du -sh $cached_binary | cut -f1 )\` | |" >> "$BUILDDIR/manifest.md" + fi + cp "$cached_binary" "$dest_path" + return + fi # Define Rust flags for reproducibility local rustflags=( @@ -48,7 +64,14 @@ build_rust_package() { "-L /usr/lib/x86_64-linux-gnu" ) + # add extra flags from .cargo/config.toml + if [[ -f $build_dir/.cargo/config.toml ]] && local cargoflags=$( mkosi-chroot dasel -f /build/$package/.cargo/config.toml -r toml 'build.rustflags' --pretty=false -w json ); then + local cargoflags=$( echo $cargoflags | jq -r '. | join(" ")' ) + local rustflags+=( "$cargoflags" ) + fi + # Build inside mkosi chroot + local ts=$( date +%s ) mkosi-chroot bash -c " unset DESTDIR export RUSTFLAGS='${rustflags[*]} ${extra_rustflags}' \ @@ -62,8 +85,13 @@ build_rust_package() { cargo fetch cargo build --release --frozen ${extra_features:+--features $extra_features} --package $cargo_package " + local seconds=$(( $( date +%s ) - ts )) + local duration=$( printf "%dm%ds" $(( seconds / 60 )) $(( seconds % 60 )) ) # Cache and install the built binary + mkdir -p "$( dirname $cached_binary )" install -m 755 "$build_dir/target/release/$package" "$cached_binary" install -m 755 "$cached_binary" "$dest_path" + + echo "| \`$package\` | \`$version\` (\`$git_describe\`, ${extra_features}) | built | \`$( du -sh $cached_binary | cut -f1 )\` | \`$duration\` |" >> "$BUILDDIR/manifest.md" } diff --git a/scripts/make_git_package.sh b/scripts/make_git_package.sh index f3323ad4..360a2e9e 100644 --- a/scripts/make_git_package.sh +++ b/scripts/make_git_package.sh @@ -8,40 +8,52 @@ make_git_package() { local git_url="$3" local build_cmd="$4" # All remaining arguments are artifact mappings in src:dest format - + mkdir -p "$DESTDIR/usr/bin" - local safe_version="${version//\//_}" - local cache_dir="$BUILDDIR/${package}-${safe_version}" + + # Clone the repository + local build_dir="$BUILDROOT/build/$package" + set +x # don't leak github token into logs + echo "Cloning ${git_url}" + if [ -f "$BUILDDIR/.ghtoken" ]; then + git_url="${git_url/#https:\/\/github.com/https:\/\/x-access-token:$(cat "$BUILDDIR/.ghtoken")@github.com}" + fi + git clone --depth 1 --branch "$version" "$git_url" "$build_dir" || ( + echo "Could not clone branch/tag, attempting to checkout the commit by sha" + git clone "$git_url" "$build_dir" && + git -C "$build_dir" checkout "$version" + ) + set -x + + # Get the git reference + local git_describe=$( git -C "$build_dir" describe --always --long --tags ) + printf "${git_describe#$package/}" > "$BUILDDIR/$package.git" + + local cache_dir="$BUILDDIR/${package}-${git_describe#${package}/}" # Use cached artifacts if available if [ -n "$cache_dir" ] && [ -d "$cache_dir" ] && [ "$(ls -A "$cache_dir" 2>/dev/null)" ]; then echo "Using cached artifacts for $package version $version" + echo "| \`$package\` | \`$version\` (\`$git_describe\`) | reused from cache | \`$( du -sh $cache_dir | cut -f1 )\` | |" >> "$BUILDDIR/manifest.md" for artifact_map in "${@:5}"; do local src="${artifact_map%%:*}" local dest="${artifact_map#*:}" mkdir -p "$(dirname "$DESTDIR$dest")" - local cached_name="$(echo "$src" | tr '/' '_')" - if [ -d "$cache_dir/$cached_name" ]; then + if [ -d "$cache_dir/$src" ]; then mkdir -p "$DESTDIR$dest" - cp -r "$cache_dir/$cached_name"/* "$DESTDIR$dest/" + cp -r "$cache_dir/$src"/* "$DESTDIR$dest/" else - cp "$cache_dir/$cached_name" "$DESTDIR$dest" + cp "$cache_dir/$src" "$DESTDIR$dest" fi done return 0 fi - + # Build from source - local build_dir="$BUILDROOT/build/$package" - if [ -f "$BUILDDIR/.ghtoken" ]; then - git_url="${git_url/#https:\/\/github.com/https:\/\/x-access-token:$(cat "$BUILDDIR/.ghtoken")@github.com}" - fi - git clone --depth 1 --branch "$version" "$git_url" "$build_dir" || ( - echo "Could not clone branch/tag, attempting to checkout the commit by sha" - git clone "$git_url" "$build_dir" && - git -C "$build_dir" checkout "$version" - ) + local ts=$( date +%s ) mkosi-chroot bash -c "unset DESTDIR && cd '/build/$package' && $build_cmd" + local seconds=$(( $( date +%s ) - ts )) + local duration=$( printf "%dm%ds" $(( seconds / 60 )) $(( seconds % 60 )) ) # Copy artifacts to image and cache for artifact_map in "${@:5}"; do @@ -56,15 +68,17 @@ make_git_package() { mkdir -p "$(dirname "$DESTDIR$dest")" cp "$build_dir/$src" "$DESTDIR$dest" fi - + # Cache artifact mkdir -p "$cache_dir" - local cached_name="$(echo "$src" | tr '/' '_')" if [ -d "$build_dir/$src" ]; then - mkdir -p "$cache_dir/$cached_name" - cp -r "$build_dir/$src"/* "$cache_dir/$cached_name/" + mkdir -p "$cache_dir/$src" + cp -r "$build_dir/$src"/* "$cache_dir/$src/" else - cp "$build_dir/$src" "$cache_dir/$cached_name" + mkdir -p "$( dirname $cache_dir/$src )" + cp "$build_dir/$src" "$cache_dir/$src" fi done -} \ No newline at end of file + + echo "| \`$package\` | \`$version\` (\`$git_describe\`) | built | \`$( du -sh $cache_dir | cut -f1 )\` | \`$duration\` |" >> "$BUILDDIR/manifest.md" +} diff --git a/shared/mkosi.build.d/10-kernel.sh b/shared/mkosi.build.d/10-kernel.sh index 264eafff..9ff7a575 100755 --- a/shared/mkosi.build.d/10-kernel.sh +++ b/shared/mkosi.build.d/10-kernel.sh @@ -51,10 +51,18 @@ cache_hash=$( cache_dir="$BUILDDIR/kernel-${KERNEL_VERSION}-${cache_hash}" cached_deb="$cache_dir/kernel.deb" +cat < $BUILDDIR/manifest.md +| component | version | built / cached | size | duration | +| ---------- | -------- | --------------- | ----- | --------- | +EOF + # Use cached kernel .deb if available if [[ -f "$cached_deb" ]] && [[ -s "$cached_deb" ]]; then echo "Using cached kernel .deb: $cached_deb" + echo "| \`kernel\` | \`${KERNEL_VERSION}\` (config hash \`${cache_hash}\`) | reused from cache | \`$( du -sh $cached_deb | cut -f1 )\` | |" >> $BUILDDIR/manifest.md else + ts=$( date +%s ) + echo "Building kernel from source..." # Build directory layout (chroot-relative paths, then host paths derived from BUILDROOT) @@ -139,6 +147,9 @@ else echo "Kernel version: ${kernel_version_string}" echo "Built .deb: $(basename "${built_deb}")" + seconds=$(( $( date +%s ) - ts )) + duration=$( printf "%dm%ds" $(( seconds / 60 )) $(( seconds % 60 )) ) + # Cache the .deb mkdir -p "${cache_dir}" cp "${built_deb}" "${cached_deb}" @@ -147,6 +158,8 @@ else echo "Cached kernel to: ${cache_dir}" rm -rf "${kernel_build_dir}" + + echo "| \`kernel\` | \`${KERNEL_VERSION}\` (config hash \`${cache_hash}\`) | built | \`$( du -sh $cached_deb | cut -f1 )\` | \`$duration\` |" >> $BUILDDIR/manifest.md fi # Copy to PACKAGEDIR for mkosi VolatilePackages installation diff --git a/shared/mkosi.extra/etc/systemd/network/10-ethernet.network b/shared/mkosi.extra/etc/systemd/network/10-ethernet.network index de6c37da..be08783e 100644 --- a/shared/mkosi.extra/etc/systemd/network/10-ethernet.network +++ b/shared/mkosi.extra/etc/systemd/network/10-ethernet.network @@ -5,6 +5,7 @@ Name=eth* en* DHCP=yes DNSDefaultRoute=no DNSOverTLS=no +Domains=~internal [DHCPv4] UseDNS=yes