diff --git a/src/lexer/Lexer.kt b/src/lexer/Lexer.kt index 173a8a2..8d0a1fa 100644 --- a/src/lexer/Lexer.kt +++ b/src/lexer/Lexer.kt @@ -9,25 +9,25 @@ class Lexer(private val source: String) { fun scan(): List { while (hasNext()) { - val token = scanToken() - tokens += token + tokens += scanToken() } + position.length = 0 tokens += Token(TokenType.EOF, position) return tokens } private fun scanToken(): Token { - val c = peek() - val token = when (c) { - '.' -> Token(TokenType.DOT, position) - else -> throw Error("Unknown symbol: $c", position) - } + val char: Char = peek() - offset++ - position.column++ - return token + position.length = 1 + + return when { + char == '.' -> scanDot() + char.isLetterOrDigit() -> scanAlphanumeric() + else -> throw Error("Unknown symbol: $char", position) + } } private fun hasNext(): Boolean { @@ -41,4 +41,27 @@ class Lexer(private val source: String) { return source[offset] } + + // Scanners + + private fun scanDot(): Token { + val token = Token(TokenType.DOT, position) + offset++ + position.column++ + return token + } + + private fun scanAlphanumeric(): Token { + val token = Token(TokenType.ALPHANUMERIC, position) + offset++ + position.column++ + + while (hasNext() && peek().isLetterOrDigit()) { + offset++ + position.column++ + position.length++ + } + + return token + } } diff --git a/src/lexer/LexerPosition.kt b/src/lexer/LexerPosition.kt index 4c39451..49cce6c 100644 --- a/src/lexer/LexerPosition.kt +++ b/src/lexer/LexerPosition.kt @@ -1,5 +1,5 @@ package lexer -class LexerPosition(val line: Int, var column: Int, val length: Int) { +class LexerPosition(val line: Int, var column: Int, var length: Int) { // Do nothing } diff --git a/src/lexer/TokenType.kt b/src/lexer/TokenType.kt index 1863986..0d88a45 100644 --- a/src/lexer/TokenType.kt +++ b/src/lexer/TokenType.kt @@ -1,6 +1,8 @@ package lexer enum class TokenType { + ALPHANUMERIC, + DOT, EOF diff --git a/tests/lexer/LexerScanTest.kt b/tests/lexer/LexerScanTest.kt index 68aa9e3..8133800 100644 --- a/tests/lexer/LexerScanTest.kt +++ b/tests/lexer/LexerScanTest.kt @@ -9,15 +9,15 @@ import kotlin.test.assertEquals class LexerScanTest { @Test - fun scan_emptyString_returnsEOF() { + fun scan_emptyString_returns_EOF() { val lexer = Lexer("") val tokens = lexer.scan() - assertEquals(1, tokens.size) - assertEquals(TokenType.EOF, tokens[0].type) + assertEquals(1, tokens.size, "Expected 1 token, got ${tokens.size}") + assertEquals(TokenType.EOF, tokens[0].type, "Expected EOF token, got ${tokens[0].type}") } @Test - fun scan_unknownSymbol_returnsError() { + fun scan_unknownSymbol_returns_Error() { val lexer = Lexer("€") assertThrows({ val tokens = lexer.scan() @@ -25,11 +25,47 @@ class LexerScanTest { } @Test - fun scan_dot_returnsDot() { + fun scan_dot_returns_Dot() { val lexer = Lexer(".") val tokens = lexer.scan() assertEquals(2, tokens.size) assertEquals(TokenType.DOT, tokens[0].type, "Expected DOT token, got ${tokens[0].type}") assertEquals(TokenType.EOF, tokens[1].type, "Expected EOF token, got ${tokens[1].type}") } + + @Test + fun scan_two_dots_returns_two_dots() { + val lexer = Lexer("..") + val tokens = lexer.scan() + assertEquals(3, tokens.size) + assertEquals(TokenType.DOT, tokens[0].type, "Expected DOT token, got ${tokens[0].type}") + assertEquals(TokenType.DOT, tokens[1].type, "Expected DOT token, got ${tokens[1].type}") + assertEquals(TokenType.EOF, tokens[2].type, "Expected EOF token, got ${tokens[2].type}") + } + + @Test + fun scan_letter_returns_letter() { + val lexer = Lexer("a") + val tokens = lexer.scan() + + assertEquals(2, tokens.size) + + assertEquals(TokenType.ALPHANUMERIC, tokens[0].type, "Expected ALPHANUMERIC token, got ${tokens[0].type}") + assertEquals(TokenType.EOF, tokens[1].type, "Expected EOF token, got ${tokens[1].type}") + + assertEquals(1, tokens[0].position.length, "Expected length 1, got ${tokens[0].position.length}") + } + + @Test + fun scan_word_returns_alphanumerics() { + val lexer = Lexer("word") + val tokens = lexer.scan() + + assertEquals(2, tokens.size) + + assertEquals(TokenType.ALPHANUMERIC, tokens[0].type, "Expected ALPHANUMERIC token, got ${tokens[0].type}") + assertEquals(TokenType.EOF, tokens[1].type, "Expected EOF token, got ${tokens[1].type}") + + assertEquals(4, tokens[0].position.length, "Expected length 4, got ${tokens[0].position.length}") + } }