From 2960f2b4ca7c8af1e36972c43c0de5e0a02bf764 Mon Sep 17 00:00:00 2001 From: omchillure Date: Sun, 12 Apr 2026 09:50:11 +0530 Subject: [PATCH] Fix skipping tests with run ignored --- crates/karva/src/commands/test/mod.rs | 1 + crates/karva/tests/it/filterset.rs | 2 +- crates/karva/tests/it/main.rs | 1 + crates/karva/tests/it/run_ignored.rs | 193 ++++++++++++++++++ crates/karva_metadata/src/filter.rs | 110 +++++++++- .../src/extensions/tags/mod.rs | 5 + .../src/runner/package_runner.rs | 10 + 7 files changed, 320 insertions(+), 2 deletions(-) create mode 100644 crates/karva/tests/it/run_ignored.rs diff --git a/crates/karva/src/commands/test/mod.rs b/crates/karva/src/commands/test/mod.rs index 73d5b43d..d73369f1 100644 --- a/crates/karva/src/commands/test/mod.rs +++ b/crates/karva/src/commands/test/mod.rs @@ -217,6 +217,7 @@ fn collect_test_names( let ctx = EvalContext { test_name: &qualified, tags: &[], + has_skip_tag: false, }; if filter.matches(&ctx) { tests.push((module_name.clone(), function_name)); diff --git a/crates/karva/tests/it/filterset.rs b/crates/karva/tests/it/filterset.rs index d8ca9bb3..a6e2969e 100644 --- a/crates/karva/tests/it/filterset.rs +++ b/crates/karva/tests/it/filterset.rs @@ -1031,7 +1031,7 @@ fn filterset_unknown_predicate() { ----- stderr ----- Karva failed Cause: invalid `--filter` expression - Cause: unknown predicate `package` in filter expression `package(foo)` (expected `test` or `tag`) + Cause: unknown predicate `package` in filter expression `package(foo)` (expected `test`, `tag`, or `runignored`) " ); } diff --git a/crates/karva/tests/it/main.rs b/crates/karva/tests/it/main.rs index 8f31a426..3115737e 100644 --- a/crates/karva/tests/it/main.rs +++ b/crates/karva/tests/it/main.rs @@ -9,5 +9,6 @@ mod durations; mod extensions; mod filterset; mod last_failed; +mod run_ignored; mod version; mod watch; diff --git a/crates/karva/tests/it/run_ignored.rs b/crates/karva/tests/it/run_ignored.rs new file mode 100644 index 00000000..1db85c95 --- /dev/null +++ b/crates/karva/tests/it/run_ignored.rs @@ -0,0 +1,193 @@ +use insta_cmd::assert_cmd_snapshot; + +use crate::common::TestContext; + +const MIXED_TESTS: &str = r" +import karva + +@karva.tags.skip +def test_skipped(): + assert False + +@karva.tags.skip('reason here') +def test_skipped_with_reason(): + assert False + +def test_normal(): + assert True +"; + +#[test] +fn runignored_runs_only_skipped_tests() { + let context = TestContext::with_file("test.py", MIXED_TESTS); + assert_cmd_snapshot!(context.command_no_parallel().arg("-E").arg("runignored(only)"), @" + success: false + exit_code: 1 + ----- stdout ----- + Starting 3 tests across 1 worker + FAIL [TIME] test::test_skipped + FAIL [TIME] test::test_skipped_with_reason + SKIP [TIME] test::test_normal + + diagnostics: + + error[test-failure]: Test `test_skipped` failed + --> test.py:5:5 + | + 4 | @karva.tags.skip + 5 | def test_skipped(): + | ^^^^^^^^^^^^ + 6 | assert False + | + info: Test failed here + --> test.py:6:5 + | + 4 | @karva.tags.skip + 5 | def test_skipped(): + 6 | assert False + | ^^^^^^^^^^^^ + 7 | + 8 | @karva.tags.skip('reason here') + | + + error[test-failure]: Test `test_skipped_with_reason` failed + --> test.py:9:5 + | + 8 | @karva.tags.skip('reason here') + 9 | def test_skipped_with_reason(): + | ^^^^^^^^^^^^^^^^^^^^^^^^ + 10 | assert False + | + info: Test failed here + --> test.py:10:5 + | + 8 | @karva.tags.skip('reason here') + 9 | def test_skipped_with_reason(): + 10 | assert False + | ^^^^^^^^^^^^ + 11 | + 12 | def test_normal(): + | + + ──────────── + Summary [TIME] 3 tests run: 0 passed, 2 failed, 1 skipped + + ----- stderr ----- + "); +} + +#[test] +fn runignored_all_runs_skipped_alongside_normal() { + let context = TestContext::with_file("test.py", MIXED_TESTS); + assert_cmd_snapshot!(context.command_no_parallel().arg("-E").arg("runignored(all)"), @" + success: false + exit_code: 1 + ----- stdout ----- + Starting 3 tests across 1 worker + FAIL [TIME] test::test_skipped + FAIL [TIME] test::test_skipped_with_reason + PASS [TIME] test::test_normal + + diagnostics: + + error[test-failure]: Test `test_skipped` failed + --> test.py:5:5 + | + 4 | @karva.tags.skip + 5 | def test_skipped(): + | ^^^^^^^^^^^^ + 6 | assert False + | + info: Test failed here + --> test.py:6:5 + | + 4 | @karva.tags.skip + 5 | def test_skipped(): + 6 | assert False + | ^^^^^^^^^^^^ + 7 | + 8 | @karva.tags.skip('reason here') + | + + error[test-failure]: Test `test_skipped_with_reason` failed + --> test.py:9:5 + | + 8 | @karva.tags.skip('reason here') + 9 | def test_skipped_with_reason(): + | ^^^^^^^^^^^^^^^^^^^^^^^^ + 10 | assert False + | + info: Test failed here + --> test.py:10:5 + | + 8 | @karva.tags.skip('reason here') + 9 | def test_skipped_with_reason(): + 10 | assert False + | ^^^^^^^^^^^^ + 11 | + 12 | def test_normal(): + | + + ──────────── + Summary [TIME] 3 tests run: 1 passed, 2 failed, 0 skipped + + ----- stderr ----- + "); +} + +#[test] +fn runignored_with_no_skipped_tests_skips_all() { + let context = TestContext::with_file( + "test.py", + r" +def test_alpha(): + assert True + +def test_beta(): + assert True +", + ); + assert_cmd_snapshot!(context.command_no_parallel().arg("-E").arg("runignored(only)"), @" + success: true + exit_code: 0 + ----- stdout ----- + Starting 2 tests across 1 worker + SKIP [TIME] test::test_alpha + SKIP [TIME] test::test_beta + + ──────────── + Summary [TIME] 2 tests run: 0 passed, 2 skipped + + ----- stderr ----- + "); +} + +#[test] +fn runignored_skipif_false_not_matched() { + let context = TestContext::with_file( + "test.py", + r" +import karva + +@karva.tags.skip(False, reason='Condition is false') +def test_conditional(): + assert True + +def test_normal(): + assert True +", + ); + assert_cmd_snapshot!(context.command_no_parallel().arg("-E").arg("runignored(only)"), @" + success: true + exit_code: 0 + ----- stdout ----- + Starting 2 tests across 1 worker + PASS [TIME] test::test_conditional + SKIP [TIME] test::test_normal + + ──────────── + Summary [TIME] 2 tests run: 1 passed, 1 skipped + + ----- stderr ----- + "); +} diff --git a/crates/karva_metadata/src/filter.rs b/crates/karva_metadata/src/filter.rs index 0354a676..12adf949 100644 --- a/crates/karva_metadata/src/filter.rs +++ b/crates/karva_metadata/src/filter.rs @@ -29,6 +29,13 @@ impl Matcher { } } +/// The mode for the `runignored(mode)` predicate. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RunIgnoredMode { + Only, + All, +} + /// A single predicate in the filter DSL, e.g. `test(~login)` or `tag(slow)`. #[derive(Debug, Clone)] pub enum Predicate { @@ -36,6 +43,8 @@ pub enum Predicate { Test(Matcher), /// Evaluated against each custom tag on the test; matches if any tag matches. Tag(Matcher), + /// Matches tests based on their `@karva.tags.skip` decorator. + RunIgnored(RunIgnoredMode), } /// The value a [`Filterset`] is evaluated against. @@ -43,6 +52,7 @@ pub enum Predicate { pub struct EvalContext<'a> { pub test_name: &'a str, pub tags: &'a [&'a str], + pub has_skip_tag: bool, } #[derive(Debug, Clone)] @@ -60,11 +70,24 @@ impl Expr { Self::Predicate(Predicate::Tag(matcher)) => { ctx.tags.iter().any(|tag| matcher.matches(tag)) } + Self::Predicate(Predicate::RunIgnored(RunIgnoredMode::Only)) => ctx.has_skip_tag, + Self::Predicate(Predicate::RunIgnored(RunIgnoredMode::All)) => true, Self::Not(inner) => !inner.matches(ctx), Self::And(lhs, rhs) => lhs.matches(ctx) && rhs.matches(ctx), Self::Or(lhs, rhs) => lhs.matches(ctx) || rhs.matches(ctx), } } + + fn contains_runignored(&self) -> bool { + match self { + Self::Predicate(Predicate::RunIgnored(_)) => true, + Self::Predicate(_) => false, + Self::Not(inner) => inner.contains_runignored(), + Self::And(lhs, rhs) | Self::Or(lhs, rhs) => { + lhs.contains_runignored() || rhs.contains_runignored() + } + } + } } /// A parsed filterset expression that can be evaluated against a test. @@ -90,6 +113,10 @@ impl Filterset { pub fn matches(&self, ctx: &EvalContext<'_>) -> bool { self.expr.matches(ctx) } + + pub fn contains_runignored(&self) -> bool { + self.expr.contains_runignored() + } } /// A set of filterset expressions combined with OR semantics (matches if any @@ -108,6 +135,10 @@ impl FiltersetSet { Ok(Self { filters }) } + pub fn contains_runignored(&self) -> bool { + self.filters.iter().any(|filter| filter.contains_runignored()) + } + pub fn is_empty(&self) -> bool { self.filters.is_empty() } @@ -146,13 +177,17 @@ pub enum FilterError { expression: String, }, #[error( - "unknown predicate `{name}` in filter expression `{expression}` (expected `test` or `tag`)" + "unknown predicate `{name}` in filter expression `{expression}` (expected `test`, `tag`, or `runignored`)" )] UnknownPredicate { name: String, expression: String }, #[error("expected `(` after predicate in filter expression `{expression}`")] ExpectedPredicateOpenParen { expression: String }, #[error("expected a matcher body in filter expression `{expression}`")] ExpectedMatcher { expression: String }, + #[error( + "expected `only` or `all` inside `runignored(...)` in filter expression `{expression}`" + )] + ExpectedRunIgnoredMode { expression: String }, } #[derive(Debug, Clone, Copy)] @@ -418,6 +453,37 @@ impl<'a> Parser<'a> { let kind = match name.as_str() { "test" => PredicateKind::Test, "tag" => PredicateKind::Tag, + "runignored" => { + self.advance(); + if self.peek() != Some(&Token::LParen) { + return Err(FilterError::ExpectedPredicateOpenParen { + expression: self.expr_str(), + }); + } + self.advance(); + let mode = match self.peek() { + Some(Token::Ident(s)) if s == "only" => { + self.advance(); + RunIgnoredMode::Only + } + Some(Token::Ident(s)) if s == "all" => { + self.advance(); + RunIgnoredMode::All + } + _ => { + return Err(FilterError::ExpectedRunIgnoredMode { + expression: self.expr_str(), + }); + } + }; + if self.peek() != Some(&Token::RParen) { + return Err(FilterError::UnclosedParenthesis { + expression: self.expr_str(), + }); + } + self.advance(); + return Ok(Expr::Predicate(Predicate::RunIgnored(mode))); + } _ => { return Err(FilterError::UnknownPredicate { name, @@ -531,6 +597,7 @@ mod tests { EvalContext { test_name, tags: tag_list, + has_skip_tag: false, } } @@ -856,4 +923,45 @@ mod tests { let f = Filterset::new("test(tag)").expect("parse"); assert!(f.matches(&ctx("mod::test_tag_something", &[]))); } + + fn ctx_with_skip<'a>(test_name: &'a str, tag_list: &'a [&'a str]) -> EvalContext<'a> { + EvalContext { + test_name, + tags: tag_list, + has_skip_tag: true, + } + } + + #[test] + fn runignored_matches_skip_tagged() { + let f = Filterset::new("runignored(only)").expect("parse"); + assert!(f.matches(&ctx_with_skip("mod::test_slow", &[]))); + assert!(!f.matches(&ctx("mod::test_fast", &[]))); + } + + #[test] + fn runignored_all_matches_everything() { + let f = Filterset::new("runignored(all)").expect("parse"); + assert!(f.matches(&ctx_with_skip("mod::test_slow", &[]))); + assert!(f.matches(&ctx("mod::test_fast", &[]))); + } + + #[test] + fn runignored_with_not() { + let f = Filterset::new("not runignored(only)").expect("parse"); + assert!(!f.matches(&ctx_with_skip("mod::test_slow", &[]))); + assert!(f.matches(&ctx("mod::test_fast", &[]))); + } + + #[test] + fn runignored_requires_valid_mode() { + assert!(matches!( + Filterset::new("runignored()"), + Err(FilterError::ExpectedRunIgnoredMode { .. }) + )); + assert!(matches!( + Filterset::new("runignored(foo)"), + Err(FilterError::ExpectedRunIgnoredMode { .. }) + )); + } } diff --git a/crates/karva_test_semantic/src/extensions/tags/mod.rs b/crates/karva_test_semantic/src/extensions/tags/mod.rs index 79e1d8b1..c968b758 100644 --- a/crates/karva_test_semantic/src/extensions/tags/mod.rs +++ b/crates/karva_test_semantic/src/extensions/tags/mod.rs @@ -262,6 +262,11 @@ impl Tags { .collect() } + /// Returns true if the test has any skip decorator, regardless of conditions. + pub(crate) fn has_skip_tag(&self) -> bool { + self.inner.iter().any(|tag| matches!(tag, Tag::Skip(_))) + } + /// Returns true if any skip tag should be skipped. pub(crate) fn should_skip(&self) -> (bool, Option) { for tag in &self.inner { diff --git a/crates/karva_test_semantic/src/runner/package_runner.rs b/crates/karva_test_semantic/src/runner/package_runner.rs index f37d91b9..2aa4b0ca 100644 --- a/crates/karva_test_semantic/src/runner/package_runner.rs +++ b/crates/karva_test_semantic/src/runner/package_runner.rs @@ -219,7 +219,9 @@ impl<'ctx, 'a> PackageRunner<'ctx, 'a> { let ctx = EvalContext { test_name: &display_name, tags: &custom_names, + has_skip_tag: tags.has_skip_tag(), }; + let should_skip = tags.should_skip(); if !filter.matches(&ctx) { return Some(self.context.register_test_case_result( &qualified, @@ -227,6 +229,14 @@ impl<'ctx, 'a> PackageRunner<'ctx, 'a> { std::time::Duration::ZERO, )); } + if should_skip.0 && !filter.contains_runignored() { + return Some(self.context.register_test_case_result( + &qualified, + IndividualTestResultKind::Skipped { reason: None }, + std::time::Duration::ZERO, + )); + } + return None; } if let (true, reason) = tags.should_skip() {