feat(lexer): Scan alphanumerics & whitespace
This commit is contained in:
parent
e0754650bc
commit
e1f632ca40
5 changed files with 60 additions and 40 deletions
|
@ -53,6 +53,8 @@ class LexerScanTest {
|
|||
assertEquals(TokenType.ALPHANUMERIC, tokens[0].type, "Expected ALPHANUMERIC token, got ${tokens[0].type}")
|
||||
assertEquals(TokenType.EOF, tokens[1].type, "Expected EOF token, got ${tokens[1].type}")
|
||||
|
||||
assertEquals(0, tokens[0].position.line, "Expected line 0, got ${tokens[0].position.line}")
|
||||
assertEquals(0, tokens[0].position.column, "Expected column 0, got ${tokens[0].position.column}")
|
||||
assertEquals(1, tokens[0].position.length, "Expected length 1, got ${tokens[0].position.length}")
|
||||
}
|
||||
|
||||
|
@ -67,5 +69,17 @@ class LexerScanTest {
|
|||
assertEquals(TokenType.EOF, tokens[1].type, "Expected EOF token, got ${tokens[1].type}")
|
||||
|
||||
assertEquals(4, tokens[0].position.length, "Expected length 4, got ${tokens[0].position.length}")
|
||||
|
||||
assertEquals("word", tokens[0].value, "Expected 'word', got ${tokens[0].value}")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun scan_whitespace_returns_nothing() {
|
||||
val lexer = Lexer(" ")
|
||||
val tokens = lexer.scan()
|
||||
|
||||
assertEquals(1, tokens.size)
|
||||
|
||||
assertEquals(TokenType.EOF, tokens[0].type, "Expected EOF token, got ${tokens[0].type}")
|
||||
}
|
||||
}
|
||||
|
|
Reference in a new issue