validate_task_images.yml

# This workflow only validates task images without pushing them to registry
# it acts as a sanity checker without verifying correctness of tasks' contents
name: Validate all task images

# * Always run on "main"
# * Always run on PRs
on:
  push:
    branches: [ main ]
  pull_request:

# If triggered by a PR, it will be in the same group. However, each commit on main will be in its own unique group
concurrency:
  group: ${{ github.workflow }}-${{ (github.head_ref && github.ref) || github.run_id }}
  cancel-in-progress: true

jobs:
  check-task-structure:

    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4

    - name: Install poetry via pipx
      run: pipx install poetry

    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.12'
        cache: 'poetry'

    - name: Install Python dependencies using Poetry
      run: poetry install

    - name: Check task structure
      run: |
        bash .github/validate_task_structure.sh

  test-evaluator-basic:
    strategy:
      matrix:
        task-type: [ sde, pm, other ]

    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4

    - name: Set up Docker Buildx
      id: buildx
      uses: docker/setup-buildx-action@v3

    - name: Build base image
      run: |
        cd workspaces/base_image
        make build

    - name: Run evaluator in each task container
      run: |
        bash .github/validate_evaluators.sh ${{ matrix.task-type }}