From 933a2602c32fd1fc575b781a41a80e72c93aa6ae Mon Sep 17 00:00:00 2001 From: mohammed ahmed Date: Sat, 4 Apr 2026 00:55:03 +0000 Subject: [PATCH] Fix: Make coverage error messages framework-agnostic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Error messages in coverage_utils.py hardcoded "Jest" even when the test framework was Vitest. This caused confusion in logs when Vitest tests failed (e.g., "Jest coverage file not found" when using Vitest). The JestCoverageUtils class is used for both Jest and Vitest since they share the same Istanbul/v8 coverage format. Error messages should be framework-agnostic. Changes: - "Jest coverage file not found" → "JavaScript coverage file not found" - "Failed to parse Jest coverage file" → "Failed to parse JavaScript coverage file" - "No coverage data found for X in Jest coverage" → "No coverage data found for X in JavaScript coverage" - "Function X not found in Jest fnMap" → "Function X not found in JavaScript fnMap" Affected trace IDs: 37e5a406, 735555fa, 940dfe80, c1e1de0e, dbec6c33, de96b1ab, fcf08c6b (7 logs from Apr 4 00:50 batch) Co-Authored-By: Claude Sonnet 4.5 --- codeflash/verification/coverage_utils.py | 8 +- .../test_coverage_utils_framework_agnostic.py | 91 +++++++++++++++++++ 2 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 tests/verification/test_coverage_utils_framework_agnostic.py diff --git a/codeflash/verification/coverage_utils.py b/codeflash/verification/coverage_utils.py index 1b2341680..e92c95947 100644 --- a/codeflash/verification/coverage_utils.py +++ b/codeflash/verification/coverage_utils.py @@ -43,14 +43,14 @@ def load_from_jest_json( """ if not coverage_json_path or not coverage_json_path.exists(): - logger.debug(f"Jest coverage file not found: {coverage_json_path}") + logger.debug(f"JavaScript coverage file not found: {coverage_json_path}") return CoverageData.create_empty(source_code_path, function_name, code_context) try: with coverage_json_path.open(encoding="utf-8") as f: coverage_data = json.load(f) except (json.JSONDecodeError, OSError) as e: - logger.warning(f"Failed to parse Jest coverage file: {e}") + logger.warning(f"Failed to parse JavaScript coverage file: {e}") return CoverageData.create_empty(source_code_path, function_name, code_context) # Find the file entry in coverage data @@ -66,7 +66,7 @@ def load_from_jest_json( break if not file_coverage: - logger.debug(f"No coverage data found for {source_code_path} in Jest coverage") + logger.debug(f"No coverage data found for {source_code_path} in JavaScript coverage") return CoverageData.create_empty(source_code_path, function_name, code_context) # Extract line coverage from statement map and execution counts @@ -94,7 +94,7 @@ def load_from_jest_json( # If function not found in fnMap, use entire file fn_start_line = 1 fn_end_line = 999999 - logger.debug(f"Function {function_name} not found in Jest fnMap, using file coverage") + logger.debug(f"Function {function_name} not found in JavaScript fnMap, using file coverage") # Calculate executed and unexecuted lines within the function executed_lines = [] diff --git a/tests/verification/test_coverage_utils_framework_agnostic.py b/tests/verification/test_coverage_utils_framework_agnostic.py new file mode 100644 index 000000000..fa29a0b9b --- /dev/null +++ b/tests/verification/test_coverage_utils_framework_agnostic.py @@ -0,0 +1,91 @@ +"""Test that coverage error messages are framework-agnostic.""" + +import tempfile +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from codeflash.languages.language_enum import Language +from codeflash.models.models import CodeOptimizationContext +from codeflash.verification.coverage_utils import JestCoverageUtils + + +class TestCoverageUtilsFrameworkAgnostic: + """Test that error messages don't hardcode 'Jest' when used for Vitest.""" + + def test_missing_coverage_file_message_is_framework_agnostic(self, caplog): + """When coverage file is missing, error message should not say 'Jest' specifically. + + This class is used for both Jest and Vitest (they use the same Istanbul/v8 format). + Error messages should be generic, not hardcode 'Jest'. + """ + # Set log level to DEBUG to capture all messages + caplog.set_level("DEBUG") + + # Create minimal context + context = MagicMock(spec=CodeOptimizationContext) + context.language = Language.JAVASCRIPT + context.target_code = "export function test() {}" + context.helper_functions = [] + + nonexistent_path = Path("/tmp/nonexistent_coverage_12345.json") + + # Load coverage from non-existent file + result = JestCoverageUtils.load_from_jest_json( + coverage_json_path=nonexistent_path, + function_name="testFunc", + code_context=context, + source_code_path=Path("/tmp/test.ts") + ) + + # Should return empty coverage data + assert result.status.name in ("NOT_FOUND", "EMPTY") + + # Error message should NOT hardcode "Jest" - it should be framework-agnostic + # since this util is used for both Jest and Vitest + log_messages = [record.message for record in caplog.records] + + # Check that if there's a message about coverage file, it doesn't say "Jest" + coverage_messages = [msg for msg in log_messages if "coverage file not found" in msg.lower()] + if coverage_messages: + # The message should NOT contain "Jest" specifically + # It should say something like "Coverage file not found" or "JavaScript coverage file not found" + for msg in coverage_messages: + assert "Jest" not in msg, ( + f"Error message should not hardcode 'Jest' since this util is used for Vitest too. " + f"Got: {msg}" + ) + + def test_parse_error_message_is_framework_agnostic(self, tmp_path, caplog): + """When coverage file is malformed, error should not say 'Jest' specifically.""" + # Set log level to capture all messages + caplog.set_level("DEBUG") + + # Create invalid JSON file + coverage_file = tmp_path / "invalid_coverage.json" + coverage_file.write_text("{invalid json") + + context = MagicMock(spec=CodeOptimizationContext) + context.language = Language.JAVASCRIPT + context.target_code = "export function test() {}" + context.helper_functions = [] + + result = JestCoverageUtils.load_from_jest_json( + coverage_json_path=coverage_file, + function_name="testFunc", + code_context=context, + source_code_path=Path("/tmp/test.ts") + ) + + # Should return empty coverage + assert result.status.name in ("NOT_FOUND", "EMPTY") + + # Check log messages don't hardcode "Jest" + log_messages = [record.message for record in caplog.records] + parse_error_messages = [msg for msg in log_messages if "parse" in msg.lower() and "coverage" in msg.lower()] + + for msg in parse_error_messages: + assert "Jest" not in msg, ( + f"Parse error message should not hardcode 'Jest'. Got: {msg}" + )