From f6a90495a58a1d7a9f13441ba68aecd0ba4aaea2 Mon Sep 17 00:00:00 2001
From: Tim Fennis <fennis.tim@gmail.com>
Date: Fri, 27 Feb 2026 17:27:55 +0100
Subject: [PATCH] =?UTF-8?q?=F0=9F=93=A6=20Extract=20lexer=20into=20standal?=
 =?UTF-8?q?one=20ndc=5Flexer=20crate?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 Cargo.lock                                       | 12 ++++++++++++
 Cargo.toml                                       |  3 ++-
 ndc_bin/Cargo.toml                               |  1 +
 ndc_bin/src/diagnostic.rs                        |  2 +-
 ndc_bin/src/highlighter.rs                       |  2 +-
 ndc_lexer/Cargo.toml                             |  9 +++++++++
 ndc_lib/src/lexer/mod.rs => ndc_lexer/src/lib.rs |  0
 {ndc_lib/src/lexer => ndc_lexer/src}/number.rs   |  0
 {ndc_lib/src/lexer => ndc_lexer/src}/span.rs     |  0
 {ndc_lib/src/lexer => ndc_lexer/src}/string.rs   |  0
 {ndc_lib/src/lexer => ndc_lexer/src}/token.rs    |  2 +-
 ndc_lib/Cargo.toml                               |  1 +
 ndc_lib/src/ast/expression.rs                    |  2 +-
 ndc_lib/src/ast/operator.rs                      |  2 +-
 ndc_lib/src/ast/parser.rs                        |  2 +-
 ndc_lib/src/interpreter/evaluate/index.rs        |  2 +-
 ndc_lib/src/interpreter/evaluate/mod.rs          |  2 +-
 ndc_lib/src/interpreter/function.rs              |  2 +-
 ndc_lib/src/interpreter/mod.rs                   |  4 ++--
 ndc_lib/src/interpreter/num.rs                   |  2 +-
 ndc_lib/src/interpreter/semantic/analyser.rs     |  2 +-
 ndc_lib/src/lib.rs                               |  1 -
 ndc_lsp/Cargo.toml                               |  1 +
 ndc_lsp/src/backend.rs                           |  4 ++--
 24 files changed, 41 insertions(+), 17 deletions(-)
 create mode 100644 ndc_lexer/Cargo.toml
 rename ndc_lib/src/lexer/mod.rs => ndc_lexer/src/lib.rs (100%)
 rename {ndc_lib/src/lexer => ndc_lexer/src}/number.rs (100%)
 rename {ndc_lib/src/lexer => ndc_lexer/src}/span.rs (100%)
 rename {ndc_lib/src/lexer => ndc_lexer/src}/string.rs (100%)
 rename {ndc_lib/src/lexer => ndc_lexer/src}/token.rs (99%)

diff --git a/Cargo.lock b/Cargo.lock
index 7d9781f6..ca9db30b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1247,6 +1247,7 @@ dependencies = [
  "clap",
  "itertools 0.14.0",
  "miette",
+ "ndc_lexer",
  "ndc_lib",
  "ndc_lsp",
  "owo-colors",
@@ -1257,6 +1258,15 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "ndc_lexer"
+version = "0.2.1"
+dependencies = [
+ "num",
+ "ryu",
+ "thiserror",
+]
+
 [[package]]
 name = "ndc_lib"
 version = "0.2.1"
@@ -1268,6 +1278,7 @@ dependencies = [
  "factorial",
  "itertools 0.14.0",
  "md5",
+ "ndc_lexer",
  "ndc_macros",
  "num",
  "once_cell",
@@ -1286,6 +1297,7 @@ dependencies = [
 name = "ndc_lsp"
 version = "0.2.1"
 dependencies = [
+ "ndc_lexer",
  "ndc_lib",
  "tokio",
  "tower-lsp",
diff --git a/Cargo.toml b/Cargo.toml
index 095b6c01..01df384f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [workspace]
 resolver = "3"
-members = ["ndc_macros", "ndc_bin", "ndc_lib", "ndc_lsp", "benches", "tests"]
+members = ["ndc_macros", "ndc_bin", "ndc_lib", "ndc_lsp", "ndc_lexer", "benches", "tests"]
 
 [workspace.package]
 edition = "2024"
@@ -17,6 +17,7 @@ derive_builder = { version = "0.20.2" }
 derive_more = { version = "2.1.1", features = ["deref", "deref_mut", "from", "display", "constructor"] }
 factorial = "0.4.0"
 itertools = "0.14.0"
+ndc_lexer = { path = "ndc_lexer" }
 ndc_lib = { path = "ndc_lib" }
 ndc_lsp = { path = "ndc_lsp" }
 ndc_macros = { path = "ndc_macros" }
diff --git a/ndc_bin/Cargo.toml b/ndc_bin/Cargo.toml
index ce86853e..d9649483 100644
--- a/ndc_bin/Cargo.toml
+++ b/ndc_bin/Cargo.toml
@@ -14,6 +14,7 @@ clap.workspace = true
 itertools.workspace = true
 strsim.workspace = true
 miette = { version = "7.6.0", features = ["fancy"] }
+ndc_lexer.workspace = true
 ndc_lib.workspace = true
 ndc_lsp.workspace = true
 owo-colors.workspace = true
diff --git a/ndc_bin/src/diagnostic.rs b/ndc_bin/src/diagnostic.rs
index eeed1162..c6e79f1a 100644
--- a/ndc_bin/src/diagnostic.rs
+++ b/ndc_bin/src/diagnostic.rs
@@ -1,6 +1,6 @@
 use miette::{Diagnostic, LabeledSpan, SourceSpan};
 use ndc_lib::interpreter::InterpreterError;
-use ndc_lib::lexer::Span;
+use ndc_lexer::Span;
 use std::fmt;
 
 fn span_to_source_span(span: Span) -> SourceSpan {
diff --git a/ndc_bin/src/highlighter.rs b/ndc_bin/src/highlighter.rs
index b4514b73..f374f95d 100644
--- a/ndc_bin/src/highlighter.rs
+++ b/ndc_bin/src/highlighter.rs
@@ -3,7 +3,7 @@ use miette::{
     SpanContents,
     highlighters::{Highlighter, HighlighterState},
 };
-use ndc_lib::lexer::{Lexer, Token};
+use ndc_lexer::{Lexer, Token};
 use owo_colors::{Rgb, Style, Styled};
 
 #[derive(Default)]
diff --git a/ndc_lexer/Cargo.toml b/ndc_lexer/Cargo.toml
new file mode 100644
index 00000000..74abd4e8
--- /dev/null
+++ b/ndc_lexer/Cargo.toml
@@ -0,0 +1,9 @@
+[package]
+name = "ndc_lexer"
+edition.workspace = true
+version.workspace = true
+
+[dependencies]
+num.workspace = true
+ryu.workspace = true
+thiserror.workspace = true
diff --git a/ndc_lib/src/lexer/mod.rs b/ndc_lexer/src/lib.rs
similarity index 100%
rename from ndc_lib/src/lexer/mod.rs
rename to ndc_lexer/src/lib.rs
diff --git a/ndc_lib/src/lexer/number.rs b/ndc_lexer/src/number.rs
similarity index 100%
rename from ndc_lib/src/lexer/number.rs
rename to ndc_lexer/src/number.rs
diff --git a/ndc_lib/src/lexer/span.rs b/ndc_lexer/src/span.rs
similarity index 100%
rename from ndc_lib/src/lexer/span.rs
rename to ndc_lexer/src/span.rs
diff --git a/ndc_lib/src/lexer/string.rs b/ndc_lexer/src/string.rs
similarity index 100%
rename from ndc_lib/src/lexer/string.rs
rename to ndc_lexer/src/string.rs
diff --git a/ndc_lib/src/lexer/token.rs b/ndc_lexer/src/token.rs
similarity index 99%
rename from ndc_lib/src/lexer/token.rs
rename to ndc_lexer/src/token.rs
index da3c93b6..add7c55e 100644
--- a/ndc_lib/src/lexer/token.rs
+++ b/ndc_lexer/src/token.rs
@@ -331,7 +331,7 @@ impl From<String> for Token {
 
 #[cfg(test)]
 mod test {
-    use crate::lexer::Span;
+    use crate::Span;
 
     #[test]
     fn test_merge() {
diff --git a/ndc_lib/Cargo.toml b/ndc_lib/Cargo.toml
index 3c684b8b..624c53e8 100644
--- a/ndc_lib/Cargo.toml
+++ b/ndc_lib/Cargo.toml
@@ -11,6 +11,7 @@ derive_more.workspace = true
 derive_builder.workspace = true
 factorial.workspace = true
 itertools.workspace = true
+ndc_lexer.workspace = true
 ndc_macros.workspace = true
 num.workspace = true
 once_cell.workspace = true
diff --git a/ndc_lib/src/ast/expression.rs b/ndc_lib/src/ast/expression.rs
index 84289da6..574e302a 100644
--- a/ndc_lib/src/ast/expression.rs
+++ b/ndc_lib/src/ast/expression.rs
@@ -2,7 +2,7 @@ use crate::ast::operator::LogicalOperator;
 use crate::ast::parser::Error as ParseError;
 use crate::interpreter::evaluate::EvaluationError;
 use crate::interpreter::function::StaticType;
-use crate::lexer::Span;
+use ndc_lexer::Span;
 use num::BigInt;
 use num::complex::Complex64;
 
diff --git a/ndc_lib/src/ast/operator.rs b/ndc_lib/src/ast/operator.rs
index 04504841..d976b731 100644
--- a/ndc_lib/src/ast/operator.rs
+++ b/ndc_lib/src/ast/operator.rs
@@ -1,5 +1,5 @@
 use crate::ast::Error as ParseError;
-use crate::lexer::{Token, TokenLocation};
+use ndc_lexer::{Token, TokenLocation};
 use std::fmt;
 use std::fmt::Formatter;
 
diff --git a/ndc_lib/src/ast/parser.rs b/ndc_lib/src/ast/parser.rs
index c6cb7442..af09d1ec 100644
--- a/ndc_lib/src/ast/parser.rs
+++ b/ndc_lib/src/ast/parser.rs
@@ -3,7 +3,7 @@ use std::fmt::Write;
 use crate::ast::Expression;
 use crate::ast::expression::{Binding, ExpressionLocation, ForBody, ForIteration, Lvalue};
 use crate::ast::operator::{BinaryOperator, LogicalOperator, UnaryOperator};
-use crate::lexer::{Span, Token, TokenLocation};
+use ndc_lexer::{Span, Token, TokenLocation};
 
 pub struct Parser {
     tokens: Vec<TokenLocation>,
diff --git a/ndc_lib/src/interpreter/evaluate/index.rs b/ndc_lib/src/interpreter/evaluate/index.rs
index 5f904153..bff61c19 100644
--- a/ndc_lib/src/interpreter/evaluate/index.rs
+++ b/ndc_lib/src/interpreter/evaluate/index.rs
@@ -14,8 +14,8 @@ use crate::interpreter::environment::Environment;
 use crate::{
     ast::{Expression, ExpressionLocation},
     interpreter::{function::FunctionCarrier, sequence::Sequence, value::Value},
-    lexer::Span,
 };
+use ndc_lexer::Span;
 use itertools::Itertools;
 use std::cell::RefCell;
 use std::cmp::min;
diff --git a/ndc_lib/src/interpreter/evaluate/mod.rs b/ndc_lib/src/interpreter/evaluate/mod.rs
index 90253e0b..3d7b1c50 100644
--- a/ndc_lib/src/interpreter/evaluate/mod.rs
+++ b/ndc_lib/src/interpreter/evaluate/mod.rs
@@ -9,7 +9,7 @@ use crate::interpreter::iterator::mut_value_to_iterator;
 use crate::interpreter::num::Number;
 use crate::interpreter::sequence::Sequence;
 use crate::interpreter::value::Value;
-use crate::lexer::Span;
+use ndc_lexer::Span;
 use index::{Offset, evaluate_as_index, get_at_index, set_at_index};
 use itertools::Itertools;
 use std::cell::RefCell;
diff --git a/ndc_lib/src/interpreter/function.rs b/ndc_lib/src/interpreter/function.rs
index 9228e7f3..d9a489e2 100644
--- a/ndc_lib/src/interpreter/function.rs
+++ b/ndc_lib/src/interpreter/function.rs
@@ -7,7 +7,7 @@ use crate::interpreter::evaluate::{
 use crate::interpreter::num::{BinaryOperatorError, Number};
 use crate::interpreter::sequence::Sequence;
 use crate::interpreter::value::Value;
-use crate::lexer::Span;
+use ndc_lexer::Span;
 use derive_builder::Builder;
 use itertools::Itertools;
 use std::cell::{BorrowError, BorrowMutError, RefCell};
diff --git a/ndc_lib/src/interpreter/mod.rs b/ndc_lib/src/interpreter/mod.rs
index 76764aed..54d9a43c 100644
--- a/ndc_lib/src/interpreter/mod.rs
+++ b/ndc_lib/src/interpreter/mod.rs
@@ -7,7 +7,7 @@ use crate::interpreter::evaluate::{EvaluationError, evaluate_expression};
 use crate::interpreter::function::FunctionCarrier;
 use crate::interpreter::semantic::analyser::{Analyser, ScopeTree};
 use crate::interpreter::value::Value;
-use crate::lexer::{Lexer, TokenLocation};
+use ndc_lexer::{Lexer, TokenLocation};
 pub mod environment;
 pub mod evaluate;
 pub mod function;
@@ -118,7 +118,7 @@ pub enum InterpreterError {
     #[error("Error while lexing source")]
     Lexer {
         #[from]
-        cause: crate::lexer::Error,
+        cause: ndc_lexer::Error,
     },
     #[error("Error while parsing source")]
     Parser {
diff --git a/ndc_lib/src/interpreter/num.rs b/ndc_lib/src/interpreter/num.rs
index de664006..0259dd30 100644
--- a/ndc_lib/src/interpreter/num.rs
+++ b/ndc_lib/src/interpreter/num.rs
@@ -8,7 +8,7 @@ use crate::ast::BinaryOperator;
 use crate::interpreter::evaluate::EvaluationError;
 use crate::interpreter::function::StaticType;
 use crate::interpreter::int::Int;
-use crate::lexer::Span;
+use ndc_lexer::Span;
 use num::bigint::TryFromBigIntError;
 use num::complex::{Complex64, ComplexFloat};
 use num::{BigInt, BigRational, Complex, FromPrimitive, Signed, ToPrimitive, Zero};
diff --git a/ndc_lib/src/interpreter/semantic/analyser.rs b/ndc_lib/src/interpreter/semantic/analyser.rs
index 05952b8f..38c19328 100644
--- a/ndc_lib/src/interpreter/semantic/analyser.rs
+++ b/ndc_lib/src/interpreter/semantic/analyser.rs
@@ -2,7 +2,7 @@ use crate::ast::{
     Binding, Expression, ExpressionLocation, ForBody, ForIteration, Lvalue, ResolvedVar,
 };
 use crate::interpreter::function::StaticType;
-use crate::lexer::Span;
+use ndc_lexer::Span;
 use itertools::Itertools;
 use std::fmt::{Debug, Formatter};
 
diff --git a/ndc_lib/src/lib.rs b/ndc_lib/src/lib.rs
index 259e2917..da4362fb 100644
--- a/ndc_lib/src/lib.rs
+++ b/ndc_lib/src/lib.rs
@@ -2,5 +2,4 @@ pub mod ast;
 mod compare;
 mod hash_map;
 pub mod interpreter;
-pub mod lexer;
 pub mod stdlib;
diff --git a/ndc_lsp/Cargo.toml b/ndc_lsp/Cargo.toml
index be7c0ab0..6331bd4d 100644
--- a/ndc_lsp/Cargo.toml
+++ b/ndc_lsp/Cargo.toml
@@ -6,6 +6,7 @@ version.workspace = true
 
 [dependencies]
 tokio = { version = "1.49.0", features = ["full"] }
+ndc_lexer.workspace = true
 ndc_lib.workspace = true
 tower-lsp.workspace = true
 
diff --git a/ndc_lsp/src/backend.rs b/ndc_lsp/src/backend.rs
index 7c47ccfb..db769b82 100644
--- a/ndc_lsp/src/backend.rs
+++ b/ndc_lsp/src/backend.rs
@@ -2,7 +2,7 @@ use std::collections::HashMap;
 
 use ndc_lib::ast::{Expression, ExpressionLocation, ForBody, ForIteration, Lvalue};
 use ndc_lib::interpreter::Interpreter;
-use ndc_lib::lexer::{Lexer, Span, TokenLocation};
+use ndc_lexer::{Lexer, Span, TokenLocation};
 use tokio::sync::Mutex;
 use tower_lsp::jsonrpc::Result as JsonRPCResult;
 use tower_lsp::lsp_types::{
@@ -31,7 +31,7 @@ impl Backend {
     async fn validate(&self, uri: &Url, text: &str) {
         let scanner = Lexer::new(text);
         let tokens = scanner
-            .collect::<Result<Vec<TokenLocation>, ndc_lib::lexer::Error>>()
+            .collect::<Result<Vec<TokenLocation>, ndc_lexer::Error>>()
             .map_err(|err| {
                 let span: Span = err.location();