From c7d8ff061222ed9e32a2fb9ce6f99447286a05aa Mon Sep 17 00:00:00 2001 From: Sebastian Mendel Date: Wed, 1 Apr 2026 10:29:47 +0200 Subject: [PATCH] Expand evals from 2 to 20 and improve SKILL.md diagnostic coverage Add 18 new evals covering auto-merge setup, solo maintainer workflow, CodeQL conflicts, signed commit merge failures, enforce_admins audit, review thread resolution, OpenSSF Scorecard, merge queue issues, Copilot reviewer race conditions, and workflow file merge limitations. Improve SKILL.md with expanded "When to Use" triggers, new "Security & Compliance Quick Checks" section with inline gh commands, and "Merge Strategy Issues" section. Keeps SKILL.md under 500 words (487). A/B test shows version B scores 29/40 vs original 12/40 (+142%). --- skills/github-project/SKILL.md | 49 +++-- skills/github-project/evals/evals.json | 264 ++++++++++++++++++++++++- 2 files changed, 290 insertions(+), 23 deletions(-) diff --git a/skills/github-project/SKILL.md b/skills/github-project/SKILL.md index 3d4e641..2ca7e31 100644 --- a/skills/github-project/SKILL.md +++ b/skills/github-project/SKILL.md @@ -12,26 +12,26 @@ allowed-tools: Bash(gh:*) Bash(git:*) Bash(grep:*) Read Write # GitHub Project Skill -## Overview - -GitHub repository setup, configuration, troubleshooting, and best practices for collaboration workflows. +GitHub repository configuration, troubleshooting, and collaboration workflow best practices. ## When to Use -- PR won't merge or shows BLOCKED status +- PR won't merge, shows BLOCKED, or has unresolved review threads - Auto-merge not working for Dependabot/Renovate PRs - Solo maintainer needs auto-approve for their own PRs -- Branch protection or ruleset configuration needed -- GitHub Actions workflow problems or CI failures -- Setting up CODEOWNERS, issue templates, or PR templates -- Repository standards compliance (TYPO3, Go, polyglot) +- Branch protection, rulesets, or `enforce_admins` audit +- GitHub Actions workflow problems, CI failures, or permission issues +- Signed commit merge failures (rebase cannot be auto-signed) +- CodeQL default setup conflicts with custom workflows +- OpenSSF Scorecard improvements (token permissions, pinned deps) +- Setting up CODEOWNERS, issue templates, PR templates, or release labeling +- Fork PR merge base issues (too many commits shown) ## Quick Diagnostics ### PR Won't Merge ```bash -# Check merge state, review decision, and unresolved threads gh api graphql -f query='query($owner:String!,$repo:String!,$pr:Int!){ repository(owner:$owner,name:$repo){pullRequest(number:$pr){ mergeStateStatus reviewDecision mergeable @@ -42,22 +42,15 @@ gh api graphql -f query='query($owner:String!,$repo:String!,$pr:Int!){ ### Solo Maintainer: PRs Stuck on REVIEW_REQUIRED -Solo maintainer projects MUST have auto-approve. Use `assets/pr-quality.yml.template` and keep `required_approving_review_count >= 1`. See `references/auto-merge-guide.md` for full setup. - -### Auto-merge Setup for New Repos +Use `assets/pr-quality.yml.template` for auto-approve with `required_approving_review_count >= 1`. See `references/auto-merge-guide.md`. -Every repo with Dependabot/Renovate needs auto-merge. Key requirements: -- Enable `allow_auto_merge` on repo -- Use `pull_request_target` trigger (not `pull_request`) -- Check `user.login` (not `github.actor`) -- Use `gh pr merge --auto` with dynamic strategy +### Auto-merge Setup -See `references/auto-merge-guide.md` for the canonical workflow and common pitfalls. +Requirements: `allow_auto_merge` on repo, `pull_request_target` trigger (not `pull_request`), check `user.login` (not `github.actor`), `gh pr merge --auto` with dynamic strategy. See `references/auto-merge-guide.md`. ### Auto-merge Not Working ```bash -# Check who enabled auto-merge and bypass apps gh api graphql -f query='query{repository(owner:"OWNER",name:"REPO"){ pullRequest(number:PR){autoMergeRequest{enabledBy{login}}} }}' --jq '.data.repository.pullRequest.autoMergeRequest' @@ -74,9 +67,23 @@ gh run view RUN_ID --repo OWNER/REPO --log-failed gh run rerun RUN_ID --repo OWNER/REPO ``` -## Running Scripts +### Security & Compliance Quick Checks + +```bash +gh api repos/OWNER/REPO/branches/main/protection --jq '.enforce_admins.enabled' +gh api repos/OWNER/REPO/code-scanning/default-setup --jq '.state' +gh api graphql -f query='query($owner:String!,$repo:String!,$pr:Int!){ + repository(owner:$owner,name:$repo){pullRequest(number:$pr){ + reviewThreads(first:50){nodes{id isResolved}} + }} +}' -f owner=OWNER -f repo=REPO -F pr=NUMBER +``` -Verify repository configuration against best practices: +### Merge Strategy Issues + +Rebase merge fails with signed commits: enable squash or auto-detect strategy. Workflow file PRs need manual merge (GITHUB_TOKEN lacks `workflows` scope). Copilot reviewer race conditions: re-run auto-approve workflow. See `references/auto-merge-guide.md`. + +## Running Scripts ```bash scripts/verify-github-project.sh /path/to/repository diff --git a/skills/github-project/evals/evals.json b/skills/github-project/evals/evals.json index b6ff72f..9d63b7d 100644 --- a/skills/github-project/evals/evals.json +++ b/skills/github-project/evals/evals.json @@ -10,7 +10,7 @@ }, { "type": "content", - "pattern": "(branch protection|ruleset|required_pull_request_reviews)" + "pattern": "(branch protection|ruleset|required_pull_request_reviews|enforce_admins)" } ] }, @@ -25,7 +25,267 @@ }, { "type": "content", - "pattern": "(BLOCKED|reviewDecision|mergeStateStatus)" + "pattern": "(BLOCKED|reviewDecision|mergeStateStatus|reviewThreads)" + } + ] + }, + { + "name": "setup_auto_merge_workflow", + "prompt": "Set up auto-merge for Dependabot and Renovate PRs in this repository", + "assertions": [ + { + "type": "content", + "pattern": "pull_request_target" + }, + { + "type": "content", + "pattern": "(user\\.login|dependabot\\[bot\\]|renovate\\[bot\\])" + }, + { + "type": "content", + "pattern": "--auto" + } + ] + }, + { + "name": "diagnose_auto_merge_failure", + "prompt": "Auto-merge is not working on PR #15 - Dependabot PR stays open after checks pass", + "assertions": [ + { + "type": "content", + "pattern": "(pull_request_target|user\\.login|github\\.actor|autoMergeRequest)" + }, + { + "type": "content", + "pattern": "(bypass|allow_auto_merge|merge strategy|--auto)" + } + ] + }, + { + "name": "solo_maintainer_pr_stuck", + "prompt": "I'm a solo maintainer and my PRs are stuck on REVIEW_REQUIRED even though I'm the only contributor", + "assertions": [ + { + "type": "content", + "pattern": "(auto-approve|pr-quality|required_approving_review_count)" + }, + { + "type": "content", + "pattern": "(solo maintainer|collaborator|write.*permission|admin)" + } + ] + }, + { + "name": "setup_codeowners", + "prompt": "Set up CODEOWNERS for this repository with automatic review assignments", + "assertions": [ + { + "type": "content", + "pattern": "(CODEOWNERS|\\.github/CODEOWNERS)" + }, + { + "type": "content", + "pattern": "(@|review)" + } + ] + }, + { + "name": "fix_github_actions_failure", + "prompt": "CI is failing on this repo - the build workflow keeps erroring out on the latest push", + "assertions": [ + { + "type": "tool_use", + "tool": "Bash", + "pattern": "gh run (list|view)" + }, + { + "type": "content", + "pattern": "(log-failed|rerun|workflow)" + } + ] + }, + { + "name": "migrate_master_to_main", + "prompt": "Migrate the default branch from master to main for this repository", + "assertions": [ + { + "type": "content", + "pattern": "(default_branch|default-branch)" + }, + { + "type": "content", + "pattern": "(branch -m|rename|master.*main)" + } + ] + }, + { + "name": "setup_dependabot", + "prompt": "Configure Dependabot for a Go project that also uses GitHub Actions", + "assertions": [ + { + "type": "content", + "pattern": "dependabot\\.yml" + }, + { + "type": "content", + "pattern": "(gomod|github-actions|package-ecosystem)" + } + ] + }, + { + "name": "codeql_default_setup_conflict", + "prompt": "CodeQL is failing with 'analyses from advanced configurations cannot be processed when default setup is enabled'", + "assertions": [ + { + "type": "content", + "pattern": "(default-setup|not-configured|code-scanning)" + }, + { + "type": "tool_use", + "tool": "Bash", + "pattern": "gh api.*code-scanning" + } + ] + }, + { + "name": "signed_commits_merge_failure", + "prompt": "Merge is failing with 'Rebase merges cannot be automatically signed by GitHub' - how do I fix this?", + "assertions": [ + { + "type": "content", + "pattern": "(squash|merge commit|allow_squash_merge|signed)" + }, + { + "type": "content", + "pattern": "(rebase.*cannot.*sign|merge strategy|auto-detect)" + } + ] + }, + { + "name": "pr_too_many_commits", + "prompt": "My PR on a fork shows 38 commits but I only added 1 - the merge base seems wrong", + "assertions": [ + { + "type": "content", + "pattern": "(merge base|close.*reopen|fork)" + }, + { + "type": "content", + "pattern": "(gh pr close|cache|recalculate)" + } + ] + }, + { + "name": "enforce_admins_audit", + "prompt": "Audit whether admins can bypass branch protection on the default branch of this repo", + "assertions": [ + { + "type": "content", + "pattern": "enforce_admins" + }, + { + "type": "tool_use", + "tool": "Bash", + "pattern": "gh api.*protection" + } + ] + }, + { + "name": "resolve_review_threads", + "prompt": "PR #23 has unresolved review threads blocking merge - help me find and resolve them", + "assertions": [ + { + "type": "content", + "pattern": "(reviewThreads|isResolved|resolveReviewThread)" + }, + { + "type": "tool_use", + "tool": "Bash", + "pattern": "gh api graphql" + } + ] + }, + { + "name": "openssf_scorecard_improvement", + "prompt": "Our OpenSSF Scorecard score is low - what should we fix first?", + "assertions": [ + { + "type": "content", + "pattern": "(Scorecard|Token-Permissions|Branch-Protection|Pinned-Dependencies)" + }, + { + "type": "content", + "pattern": "(workflow.*write|SHA.*pin|required_approving_review_count)" + } + ] + }, + { + "name": "workflow_permissions_least_privilege", + "prompt": "Fix the workflow permissions in our CI - we have write permissions at the workflow level", + "assertions": [ + { + "type": "content", + "pattern": "(job-level|workflow-level|permissions)" + }, + { + "type": "content", + "pattern": "(contents: read|pull-requests: write|least.privilege)" + } + ] + }, + { + "name": "setup_release_labeling", + "prompt": "Set up automated release labeling so PRs and issues get labeled when a release is published", + "assertions": [ + { + "type": "content", + "pattern": "(release-labeler|released:v)" + }, + { + "type": "content", + "pattern": "(release.*published|label|announcement)" + } + ] + }, + { + "name": "merge_queue_troubleshooting", + "prompt": "PRs keep getting stuck in the merge queue after force-pushing a rebase", + "assertions": [ + { + "type": "content", + "pattern": "(merge queue|stale review|dismiss_stale_reviews)" + }, + { + "type": "content", + "pattern": "(force.push|re-queue|resolveReviewThread|auto-approve)" + } + ] + }, + { + "name": "copilot_reviewer_race_condition", + "prompt": "Auto-approve keeps getting skipped and PRs stay REVIEW_REQUIRED - we use Copilot as a reviewer", + "assertions": [ + { + "type": "content", + "pattern": "(race condition|Copilot|pending reviewer)" + }, + { + "type": "content", + "pattern": "(re-run|rerun|auto-approve|COMMENTED)" + } + ] + }, + { + "name": "workflow_file_pr_cannot_merge", + "prompt": "A Dependabot PR that updates a GitHub Actions version can't be auto-merged - it modifies workflow files", + "assertions": [ + { + "type": "content", + "pattern": "(\\.github/workflows/|workflow.*files|GITHUB_TOKEN)" + }, + { + "type": "content", + "pattern": "(manual.*merge|workflows.*permission|cannot.*auto-merge)" } ] }