feat(lexer): Scan alphanumerics & whitespace

This commit is contained in:
Tibo De Peuter 2025-03-27 17:25:58 +01:00
parent e0754650bc
commit e1f632ca40
Signed by: tdpeuter
GPG key ID: 38297DE43F75FFE2
5 changed files with 60 additions and 40 deletions

View file

@ -53,6 +53,8 @@ class LexerScanTest {
assertEquals(TokenType.ALPHANUMERIC, tokens[0].type, "Expected ALPHANUMERIC token, got ${tokens[0].type}")
assertEquals(TokenType.EOF, tokens[1].type, "Expected EOF token, got ${tokens[1].type}")
assertEquals(0, tokens[0].position.line, "Expected line 0, got ${tokens[0].position.line}")
assertEquals(0, tokens[0].position.column, "Expected column 0, got ${tokens[0].position.column}")
assertEquals(1, tokens[0].position.length, "Expected length 1, got ${tokens[0].position.length}")
}
@ -67,5 +69,17 @@ class LexerScanTest {
assertEquals(TokenType.EOF, tokens[1].type, "Expected EOF token, got ${tokens[1].type}")
assertEquals(4, tokens[0].position.length, "Expected length 4, got ${tokens[0].position.length}")
assertEquals("word", tokens[0].value, "Expected 'word', got ${tokens[0].value}")
}
@Test
fun scan_whitespace_returns_nothing() {
val lexer = Lexer(" ")
val tokens = lexer.scan()
assertEquals(1, tokens.size)
assertEquals(TokenType.EOF, tokens[0].type, "Expected EOF token, got ${tokens[0].type}")
}
}