Skip to content

[ML] Automate version bump in CI pipeline #3018

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
edsavage wants to merge 3 commits into
base: main
Choose a base branch
from
+309 −74
Open

[ML] Automate version bump in CI pipeline #3018

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
22e0945
[ML] Automate version bump in CI pipeline
edsavage Apr 7, 2026
79904fc
[ML] Add DRY_RUN mode to version bump script
Apr 8, 2026
8226e07
[ML] Add minor version workflow to version bump automation
edsavage Apr 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
206 changes: 132 additions & 74 deletions .buildkite/job-version-bump.json.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,100 +10,158 @@
#
# This script generates JSON for the ml-cpp version bump pipeline.
# It is intended to be triggered by the centralized release-eng pipeline.
# It can be integrated into existing or new workflows and includes a plugin
# that polls artifact URLs until the expected version is available.
#
# Supports two workflows via the WORKFLOW env var:
# patch (default) — bump version on BRANCH, wait for 2 artifact sets
# minor — create minor branch + bump BRANCH, wait for 3 artifact sets


import contextlib
import json
import os


WOLFI_IMAGE = "docker.elastic.co/release-eng/wolfi-build-essential-release-eng:latest"
STAGING_URL = "https://artifacts-staging.elastic.co/ml-cpp/latest"
SNAPSHOT_URL = "https://storage.googleapis.com/elastic-artifacts-snapshot/ml-cpp/latest"


def json_watcher_plugin(url, expected_value):
return {
"elastic/json-watcher#v1.0.0": {
"url": url,
"field": ".version",
"expected_value": expected_value,
"polling_interval": "30",
}
}


def dra_step(label, key, depends_on, plugins):
return {
"label": label,
"key": key,
"depends_on": depends_on,
"agents": {
"image": WOLFI_IMAGE,
"cpu": "250m",
"memory": "512Mi",
"ephemeralStorage": "1Gi",
},
"command": [
'echo "Waiting for DRA artifacts..."',
],
"timeout_in_minutes": 240,
"retry": {
"automatic": [{"exit_status": "*", "limit": 2}],
"manual": {"permit_on_passed": True},
},
"plugins": plugins,
}


def main():
pipeline = {}
# TODO: replace the block step with version bump logic
workflow = os.environ.get("WORKFLOW", "patch")

pipeline_steps = [
{
"block": "Ready to fetch for DRA artifacts?",
"prompt": (
"Unblock when your team is ready to proceed.\n\n"
"Trigger parameters:\n"
"- NEW_VERSION: ${NEW_VERSION}\n"
"- BRANCH: ${BRANCH}\n"
"- WORKFLOW: ${WORKFLOW}\n"
),
"key": "block-get-dra-artifacts",
"blocked_state": "running",
},
{
"label": "Fetch DRA Artifacts",
"key": "fetch-dra-artifacts",
"depends_on": "block-get-dra-artifacts",
"label": "Bump version to ${NEW_VERSION}",
"key": "bump-version",
"agents": {
"image": "docker.elastic.co/release-eng/wolfi-build-essential-release-eng:latest",
"image": WOLFI_IMAGE,
"cpu": "250m",
"memory": "512Mi",
"ephemeralStorage": "1Gi",
},
"command": [
'echo "Starting DRA artifacts retrieval..."',
],
"timeout_in_minutes": 240,
"retry": {
"automatic": [
{
"exit_status": "*",
"limit": 2,
}
],
"manual": {"permit_on_passed": True},
},
"plugins": [
{
"elastic/json-watcher#v1.0.0": {
"url": "https://artifacts-staging.elastic.co/ml-cpp/latest/${BRANCH}.json",
"field": ".version",
"expected_value": "${NEW_VERSION}",
"polling_interval": "30",
}
},
{
"elastic/json-watcher#v1.0.0": {
"url": "https://storage.googleapis.com/elastic-artifacts-snapshot/ml-cpp/latest/${BRANCH}.json",
"field": ".version",
"expected_value": "${NEW_VERSION}-SNAPSHOT",
"polling_interval": "30",
}
},
"dev-tools/bump_version.sh",
],
},
]

pipeline["steps"] = pipeline_steps
pipeline["notify"] = [
{
"slack": {"channels": ["#machine-learn-build"]},
"if": (
"(build.branch == 'main' || "
"build.branch =~ /^[0-9]+\\.[0-9x]+$/) && "
"(build.state == 'passed' || build.state == 'failed')"
),
},
{
"slack": {
"channels": ["#machine-learn-build"],
"message": (
"🚦 Pipeline waiting for approval 🚦\n"
"Repo: `${REPO}`\n\n"
"Ready to fetch DRA artifacts - please unblock when ready.\n"
"New version: `${NEW_VERSION}`\n"
"Branch: `${BRANCH}`\n"
"Workflow: `${WORKFLOW}`\n"
"${BUILDKITE_BUILD_URL}\n"
if workflow == "minor":
# Minor workflow: artifact checks for both the upstream branch and the
# new minor branch, running in parallel after the bump step.
#
# Derive the minor branch from NEW_VERSION: if NEW_VERSION=9.5.0
# then the previous minor (the new branch) is 9.4 with version 9.4.0.
new_version = os.environ.get("NEW_VERSION", "0.0.0")
parts = new_version.split(".")
if len(parts) >= 2:
major, minor_num = parts[0], int(parts[1])
minor_branch = f"{major}.{minor_num - 1}"
minor_version = f"{major}.{minor_num - 1}.0"
else:
minor_branch = "unknown"
minor_version = "unknown"

pipeline_steps.append(
dra_step(
label=f"Fetch DRA Artifacts (${{BRANCH}})",
key="fetch-dra-upstream",
depends_on="bump-version",
plugins=[
json_watcher_plugin(
f"{STAGING_URL}/${{BRANCH}}.json",
"${NEW_VERSION}",
),
json_watcher_plugin(
f"{SNAPSHOT_URL}/${{BRANCH}}.json",
"${NEW_VERSION}-SNAPSHOT",
),
],
)
)

pipeline_steps.append(
dra_step(
label=f"Fetch DRA Artifacts ({minor_branch})",
key="fetch-dra-minor",
depends_on="bump-version",
plugins=[
json_watcher_plugin(
f"{STAGING_URL}/{minor_branch}.json",
minor_version,
),
json_watcher_plugin(
f"{SNAPSHOT_URL}/{minor_branch}.json",
f"{minor_version}-SNAPSHOT",
),
],
)
)
else:
# Patch workflow: staging + snapshot for BRANCH
pipeline_steps.append(
dra_step(
label="Fetch DRA Artifacts",
key="fetch-dra-artifacts",
depends_on="bump-version",
plugins=[
json_watcher_plugin(
f"{STAGING_URL}/${{BRANCH}}.json",
"${NEW_VERSION}",
),
json_watcher_plugin(
f"{SNAPSHOT_URL}/${{BRANCH}}.json",
"${NEW_VERSION}-SNAPSHOT",
),
],
)
)

pipeline = {
"steps": pipeline_steps,
"notify": [
{
"slack": {"channels": ["#machine-learn-build"]},
"if": (
"(build.branch == 'main' || "
"build.branch =~ /^[0-9]+\\.[0-9x]+$/) && "
"(build.state == 'passed' || build.state == 'failed')"
),
},
"if": 'build.state == "blocked"',
},
]
],
}

print(json.dumps(pipeline, indent=2))

Expand Down
Loading