Tests: cover issue snapshots and add 8.5/next set coverage

.dev/check-ruleset-sets-parity.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(dirname "$SCRIPT_DIR")"
+
+tmp_expected=$(mktemp)
+tmp_actual=$(mktemp)
+trap 'rm -f "$tmp_expected" "$tmp_actual"' EXIT
+
+for ruleset in "$ROOT_DIR"/ruleset-*.xml; do
+	name=$(basename "$ruleset")
+	name=${name#ruleset-}
+	name=${name%.xml}
+	echo "$name"
+done | sort -u > "$tmp_expected"
+
+for set_dir in "$ROOT_DIR"/tests/Sets/*; do
+	if [ -d "$set_dir" ]; then
+		echo "$(basename "$set_dir")"
+	fi
+done | sort -u > "$tmp_actual"
+
+missing_sets=$(comm -23 "$tmp_expected" "$tmp_actual" || true)
+orphan_sets=$(comm -13 "$tmp_expected" "$tmp_actual" || true)
+
+if [ -n "$missing_sets" ] || [ -n "$orphan_sets" ]; then
+	echo "Ruleset/set parity check failed."
+
+	if [ -n "$missing_sets" ]; then
+		echo "Missing tests/Sets directories for rulesets:"
+		echo "$missing_sets"
+	fi
+
+	if [ -n "$orphan_sets" ]; then
+		echo "tests/Sets directories without matching ruleset-*.xml:"
+		echo "$orphan_sets"
+	fi
+
+	exit 1
+fi
+
+echo "Ruleset/set parity check passed."

.dev/ruleset-diff-report.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(dirname "$SCRIPT_DIR")"
+REPORT_DIR="$ROOT_DIR/.reports"
+REPORT_FILE="$REPORT_DIR/ruleset-diff.md"
+
+mkdir -p "$REPORT_DIR"
+
+tmp_dir=$(mktemp -d)
+trap 'rm -rf "$tmp_dir"' EXIT
+
+rulesets=("ruleset.xml" "ruleset-8.2.xml" "ruleset-8.3.xml" "ruleset-8.4.xml" "ruleset-8.5.xml" "ruleset-next.xml")
+
+extract_sniffs() {
+	local standard="$1"
+	vendor/bin/phpcs --standard="$standard" -e \
+		| awk '$1 ~ /^[A-Z][A-Za-z0-9]*\.[A-Za-z0-9]+\.[A-Za-z0-9]+$/ {print $1}'
+}
+
+for ruleset in "${rulesets[@]}"; do
+	extract_sniffs "$ruleset" | sort -u > "$tmp_dir/$ruleset.txt"
+done
+
+{
+	echo "# Ruleset Diff Report"
+	echo
+	echo "Generated: $(date -u +'%Y-%m-%d %H:%M:%S UTC')"
+	echo
+	echo "## Sniff Counts"
+	echo
+	echo "| Ruleset | Sniff count |"
+	echo "|---|---:|"
+	for ruleset in "${rulesets[@]}"; do
+		count=$(wc -l < "$tmp_dir/$ruleset.txt" | tr -d ' ')
+		echo "| \`$ruleset\` | $count |"
+	done
+	echo
+
+	echo "## Differences vs ruleset.xml"
+	echo
+	for ruleset in "${rulesets[@]}"; do
+		if [ "$ruleset" = "ruleset.xml" ]; then
+			continue
+		fi
+
+		added=$(comm -13 "$tmp_dir/ruleset.xml.txt" "$tmp_dir/$ruleset.txt" || true)
+		removed=$(comm -23 "$tmp_dir/ruleset.xml.txt" "$tmp_dir/$ruleset.txt" || true)
+
+		echo "### \`$ruleset\`"
+		echo
+		if [ -z "$added" ] && [ -z "$removed" ]; then
+			echo "No sniff-list differences compared to \`ruleset.xml\`."
+			echo
+			continue
+		fi
+
+		if [ -n "$added" ]; then
+			echo "Added sniffs:"
+			echo '```'
+			echo "$added"
+			echo '```'
+			echo
+		fi
+
+		if [ -n "$removed" ]; then
+			echo "Removed sniffs:"
+			echo '```'
+			echo "$removed"
+			echo '```'
+			echo
+		fi
+	done
+} > "$REPORT_FILE"
+
+echo "Generated $REPORT_FILE"

.dev/sniff-coverage-matrix.php

@@ -0,0 +1,165 @@
+#!/usr/bin/env php
+<?php declare(strict_types = 1);
+
+$root = dirname(__DIR__);
+$reportsDir = $root . '/.reports';
+$sniffsRoot = $root . '/tests/Sniffs';
+
+$phpcsOutput = [];
+$exitCode = 0;
+exec('vendor/bin/phpcs --standard=ruleset.xml -e 2>&1', $phpcsOutput, $exitCode);
+
+if ($exitCode !== 0) {
+	fwrite(STDERR, "Failed to list enabled sniffs.\n");
+	fwrite(STDERR, implode("\n", $phpcsOutput) . "\n");
+	exit($exitCode);
+}
+
+$enabledSniffs = [];
+foreach ($phpcsOutput as $line) {
+	if (preg_match('#^\s*([A-Z][A-Za-z0-9]*\.[A-Za-z0-9_]+\.[A-Za-z0-9_]+)\s*$#', $line, $matches) === 1) {
+		$enabledSniffs[$matches[1]] = true;
+	}
+}
+$enabledSniffs = array_keys($enabledSniffs);
+sort($enabledSniffs);
+
+$refDirectories = [];
+$iterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($sniffsRoot));
+foreach ($iterator as $file) {
+	if (!$file->isFile() || !str_ends_with($file->getFilename(), '.ruleset.xml')) {
+		continue;
+	}
+
+	$content = file_get_contents($file->getPathname());
+	if ($content === false) {
+		continue;
+	}
+
+	if (preg_match_all('#<rule\s+ref="([^"]+)"#', $content, $matches) !== false) {
+		foreach ($matches[1] as $ref) {
+			if (!str_contains($ref, '.')) {
+				continue;
+			}
+
+			$refDirectories[$ref] ??= [];
+			$refDirectories[$ref][$file->getPath()] = true;
+		}
+	}
+}
+
+foreach ($refDirectories as &$directories) {
+	$directories = array_keys($directories);
+	sort($directories);
+}
+unset($directories);
+
+if (!is_dir($reportsDir)) {
+	mkdir($reportsDir, 0777, true);
+}
+
+$csvPath = $reportsDir . '/sniff-coverage-matrix.csv';
+$summaryPath = $reportsDir . '/sniff-coverage-summary.md';
+
+$rows = [];
+foreach ($enabledSniffs as $sniff) {
+	$directories = $refDirectories[$sniff] ?? [];
+	$fixtures = [];
+	$hasGood = false;
+	$hasBad = false;
+	$customCount = 0;
+
+	foreach ($directories as $directory) {
+		$phpFiles = glob($directory . '/*.php') ?: [];
+		foreach ($phpFiles as $phpFile) {
+			$fixtures[$phpFile] = true;
+			$fileName = basename($phpFile);
+			if ($fileName === 'good.php') {
+				$hasGood = true;
+			} elseif ($fileName === 'bad.php') {
+				$hasBad = true;
+			} elseif (str_starts_with($fileName, 'custom')) {
+				$customCount++;
+			}
+		}
+	}
+
+	$rows[] = [
+		'enabled_sniff' => $sniff,
+		'has_tests' => count($directories) > 0,
+		'fixture_count' => count($fixtures),
+		'has_good' => $hasGood,
+		'has_bad' => $hasBad,
+		'custom_count' => $customCount,
+		'directories' => implode(';', array_map(static fn(string $path): string => ltrim(str_replace($root, '', $path), '/'), $directories)),
+	];
+}
+
+$csv = fopen($csvPath, 'wb');
+if ($csv === false) {
+	fwrite(STDERR, "Cannot open CSV report for writing: {$csvPath}\n");
+	exit(1);
+}
+
+fputcsv($csv, ['enabled_sniff', 'has_tests', 'fixture_count', 'has_good', 'has_bad', 'custom_count', 'directories']);
+foreach ($rows as $row) {
+	fputcsv($csv, [
+		$row['enabled_sniff'],
+		$row['has_tests'] ? 'true' : 'false',
+		(string) $row['fixture_count'],
+		$row['has_good'] ? 'true' : 'false',
+		$row['has_bad'] ? 'true' : 'false',
+		(string) $row['custom_count'],
+		$row['directories'],
+	]);
+}
+fclose($csv);
+
+$total = count($rows);
+$tested = count(array_filter($rows, static fn(array $row): bool => $row['has_tests'] === true));
+$missing = array_values(array_filter($rows, static fn(array $row): bool => $row['has_tests'] === false));
+$weak = array_values(array_filter(
+	$rows,
+	static fn(array $row): bool => $row['has_tests'] === true && (!$row['has_good'] || !$row['has_bad'] || $row['fixture_count'] === 1)
+));
+$strong = count(array_filter(
+	$rows,
+	static fn(array $row): bool => $row['has_tests'] === true && $row['has_good'] && $row['has_bad'] && $row['custom_count'] > 0
+));
+
+$summary = [];
+$summary[] = '# Sniff Coverage Summary';
+$summary[] = '';
+$summary[] = '- Total enabled sniffs: ' . $total;
+$summary[] = '- With dedicated tests: ' . $tested;
+$summary[] = '- Missing dedicated tests: ' . count($missing);
+$summary[] = '- Weak examples: ' . count($weak);
+$summary[] = '- Strong examples (good+bad+custom): ' . $strong;
+$summary[] = '';
+
+if ($missing !== []) {
+	$summary[] = '## Missing';
+	foreach ($missing as $row) {
+		$summary[] = '- `' . $row['enabled_sniff'] . '`';
+	}
+	$summary[] = '';
+}
+
+if ($weak !== []) {
+	$summary[] = '## Weak Examples';
+	foreach ($weak as $row) {
+		$summary[] = sprintf(
+			'- `%s` (fixtures=%d, good=%s, bad=%s)',
+			$row['enabled_sniff'],
+			$row['fixture_count'],
+			$row['has_good'] ? 'true' : 'false',
+			$row['has_bad'] ? 'true' : 'false'
+		);
+	}
+	$summary[] = '';
+}
+
+file_put_contents($summaryPath, implode("\n", $summary));
+
+fwrite(STDOUT, "Generated {$csvPath}\n");
+fwrite(STDOUT, "Generated {$summaryPath}\n");

.docs/README.md

@@ -31,8 +31,8 @@ Take a look at our template repository [contributte/bare](https://github.com/con
   <arg name="parallel" value="16"/>
 
   <!-- Rulesets -->
-  <rule ref="./vendor/contributte/qa/ruleset-8.1.xml"/>
-  <!-- <rule ref="./vendor/contributte/qa/ruleset-8.0.xml"/> -->
+  <rule ref="./vendor/contributte/qa/ruleset-8.5.xml"/>
+  <!-- <rule ref="./vendor/contributte/qa/ruleset-8.4.xml"/> -->
   <!-- <rule ref="./vendor/contributte/qa/ruleset-next.xml"/> -->
 
   <!-- Rules -->
@@ -66,7 +66,7 @@ vendor/bin/phpcs --standard=ruleset.xml -e
 Example output:
 
 ```
-The Contributte standard contains 186 sniffs
+The Contributte standard contains 185 sniffs
 
 Generic (27 sniffs)
 -------------------