tdpeuter/2025LogProg-project-GhentProlog
tdpeuter/2025LogProg-project-GhentProlog
Archived
1
Fork 0
Code Issues Releases Activity Actions

Remove old lexer+parser implementation

Browse source
This commit is contained in:
Tibo De Peuter 2025-04-27 18:22:03 +02:00
parent d5632e9217
commit a4ec29f084
Signed by: tdpeuter
GPG key ID: 38297DE43F75FFE2
15 changed files with 0 additions and 706 deletions
Download patch file Download diff file Expand all files Collapse all files

127
src/lexer/Lexer.kt
View file

@ -1,127 +0,0 @@
package lexer
import lexer.errors.LexingError
import lexer.errors.LexingErrorType
import lexer.state.LexerPosition
import lexer.state.TokenPosition
class Lexer(private val source: String) {
private var tokens: List<Token> = emptyList()
private val position = LexerPosition(0, 0, -1)
/**
* Scans the source code and returns a list of tokens.
* @return List of [Token]s
*/
fun scan(): List<Token> {
while (hasNext()) {
val char: Char = peek()
tokens += when {
char == '(' -> scanSymbol(TokenType.PARENTHESIS_LEFT)
char == ')' -> scanSymbol(TokenType.PARENTHESIS_RIGHT)
char == '.' -> scanSymbol(TokenType.DOT)
char == '"' -> scanQuotedString()
char == '%' -> { scanComment(); continue }
char.isLetterOrDigit() -> scanAlphanumeric()
char.isWhitespace() -> { scanWhitespace(); continue }
else -> throw LexingError(LexingErrorType.UNKNOWN_TOKEN, "Did not recognize $char", position)
}
}
tokens += Token(TokenType.EOF, "EOF", getPosition(0))
return tokens
}
private fun hasNext(): Boolean {
// Check if the position is within the source length
return position.offset < source.length
}
private fun peek(): Char {
// Peek should only be called if there is a next character
require(hasNext()) {
LexingError(LexingErrorType.UNEXPECTED_END_OF_INPUT, "Expected additional character", position)
}
return source[position.offset]
}
private fun next(): Char {
// Advance the position and return the character
val char = peek()
position.offset++
position.column++
return char
}
private fun getPosition(length: Int = 1): TokenPosition {
// Return a new TokenPosition based on the current LexerPosition
return TokenPosition(position.line, position.column, length)
}
/* * * * * * *
* Scanners *
* * * * * * */
/**
* Scans a symbol token, given the expected [TokenType].
* @param tokenType The expected [TokenType]
* @return The scanned [Token]
*/
private fun scanSymbol(tokenType: TokenType): Token {
return Token(tokenType, next().toString(), getPosition(1))
}
private fun scanAlphanumeric(): Token {
// Scan all alphanumeric characters
var length = 0
while (hasNext() && peek().isLetterOrDigit()) {
next()
length++
}
val value = source.substring(position.offset - length, position.offset)
return Token(TokenType.ALPHANUMERIC, value, getPosition(length))
}
private fun scanQuotedString(): Token {
// "Assert" that the next character is the start of a quoted string
require(next() == '"') {
LexingError(LexingErrorType.UNEXPECTED_TOKEN, "Expected opening quote '('", position)
}
var length = 0
while (hasNext() && peek() != '"') {
next()
length++
}
// "Assert" that the next character is the end of the quoted string
require(next() == '"') {
LexingError(LexingErrorType.UNEXPECTED_TOKEN, "Expected closing quote ')'", position)
}
val value = source.substring(position.offset - length - 1, position.offset - 1)
return Token(TokenType.ALPHANUMERIC, value, getPosition(length))
}
private fun scanComment() {
// "Assert" that the next character is the start of a comment
require(next() == '%') {
LexingError(LexingErrorType.UNEXPECTED_TOKEN, "Expected opening comment '%'", position)
}
// Skip all characters until the end of the line
while (hasNext() && peek() != '\n') {
next()
}
}
private fun scanWhitespace() {
// Skip all whitespace characters
while (hasNext() && peek().isWhitespace()) {
if (next() == '\n') {
position.line++
position.column = 0
}
}
}
}

9
src/lexer/Token.kt
View file

@ -1,9 +0,0 @@
package lexer
import lexer.state.TokenPosition
data class Token(
val type: TokenType,
val value: String,
val position: TokenPosition
)

15
src/lexer/TokenType.kt
View file

@ -1,15 +0,0 @@
package lexer
enum class TokenType {
ALPHANUMERIC,
// TODO Replace with SMALL_LETTER, CAPITAL_LETTER, DIGIT, HEX_DIGIT, ... ?
// Structure
COMMA,
DOT,
PARENTHESIS_LEFT, PARENTHESIS_RIGHT,
// Special
EOF
}

13
src/lexer/errors/LexingError.kt
View file

@ -1,13 +0,0 @@
package lexer.errors
import lexer.state.LexerPosition
data class LexingError(
val type: LexingErrorType,
override val message: String,
val position: LexerPosition
) : Throwable(
"""
${position.line}:${position.column + 1} ${type}: $message
""".trimIndent()
)

7
src/lexer/errors/LexingErrorType.kt
View file

@ -1,7 +0,0 @@
package lexer.errors
enum class LexingErrorType {
UNKNOWN_TOKEN,
UNEXPECTED_TOKEN,
UNEXPECTED_END_OF_INPUT,
}

3
src/lexer/state/LexerPosition.kt
View file

@ -1,3 +0,0 @@
package lexer.state
data class LexerPosition(var offset: Int, var line: Int, var column: Int)

3
src/lexer/state/TokenPosition.kt
View file

@ -1,3 +0,0 @@
package lexer.state
data class TokenPosition(val line: Int, val column: Int, val length: Int)

137
src/parser/Parser.kt
View file

@ -1,137 +0,0 @@
package parser
import lexer.Token
import lexer.TokenType
import parser.errors.ParsingError
import parser.errors.ParsingErrorType
import parser.state.ParserPosition
import prolog.ast.logic.Clause
import prolog.ast.logic.Fact
import prolog.ast.logic.Rule
import prolog.ast.terms.Atom
import prolog.ast.terms.Structure
import prolog.ast.terms.Term
class Parser(private val tokens: List<Token>) {
private val position: ParserPosition = ParserPosition(0)
fun parse(): List<Term> {
val terms = mutableListOf<Term>()
while (hasNext()) {
position.save()
var term: Term? = null
while (term == null) {
// Try each parser rule in order
}
require(term != null) {
ParsingError(ParsingErrorType.UNEXPECTED_TOKEN, "Expected a term", position)
}
terms.add(term)
}
return terms
}
/**
* Matches the current token with any of the expected types.
* If it matches, it consumes the token and returns true.
*
* @param types The list of expected token types.
* @return True if the current token matches any of the expected types, false otherwise.
*/
private fun match(types: List<TokenType>): Boolean {
for (type in types) {
if (check(type)) {
next()
return true
}
}
return false
}
/**
* Checks if the current token matches the expected type.
*/
private fun check(type: TokenType): Boolean {
return hasNext() && peek().type == type
}
private fun hasNext(): Boolean {
// Check if the position is within the tokens list
// TODO Check for EOF instead?
return position.offset < tokens.size
}
private fun peek(): Token {
require(hasNext()) { "Unexpected end of input" }
return tokens[position.offset]
}
private fun next(): Token {
val token = peek()
position.offset++
return token
}
private fun previous(): Token {
require(0 < position.offset) { "No previous token" }
return tokens[position.offset - 1]
}
/* * * * * *
* Parsers *
* * * * * */
private fun parseWithTry(parseRule: () -> Term): Term {
try {
return parseRule()
} catch (e: Exception) {
throw ParsingError(ParsingErrorType.UNEXPECTED_TOKEN, "Unexpected token", position)
}
}
private fun parseClause(): Clause {
return try {
Fact(parseStructure())
} catch (e: Exception) {
Fact(parseAtom())
}
}
private fun parseStructure(): Structure {
val name = parseAtom()
val args = mutableListOf<Term>()
require(match(listOf(TokenType.PARENTHESIS_LEFT))) {
ParsingError(ParsingErrorType.UNEXPECTED_TOKEN, "Expected '(' after structure name", position)
}
// TODO Handle arguments
require(match(listOf(TokenType.PARENTHESIS_RIGHT))) {
ParsingError(ParsingErrorType.UNEXPECTED_TOKEN, "Expected ')' after structure arguments", position)
}
return Structure(name, args)
}
private fun parseAtom(): Atom {
return Atom(parseLetterDigit())
}
private fun parseLetterDigit(): String {
require(match(listOf(TokenType.ALPHANUMERIC)) && previous().value[0].isLowerCase()) {
ParsingError(ParsingErrorType.UNEXPECTED_TOKEN, "Expected lowercase letter", position)
}
return previous().value
}
}

12
src/parser/errors/ParsingError.kt
View file

@ -1,12 +0,0 @@
package parser.errors
import parser.state.ParserPosition
class ParsingError(private val type: ParsingErrorType, override val message: String, private val position: ParserPosition) :
Throwable() {
override fun toString(): String {
return """
($position) ${type}: $message
""".trimIndent()
}
}

7
src/parser/errors/ParsingErrorType.kt
View file

@ -1,7 +0,0 @@
package parser.errors
enum class ParsingErrorType {
UNEXPECTED_TOKEN,
INTERNAL_ERROR,
}

25
src/parser/state/ParserPosition.kt
View file

@ -1,25 +0,0 @@
package parser.state
import parser.errors.ParsingError
import parser.errors.ParsingErrorType
data class ParserPosition(var offset: Int) {
private val checkpoints: ArrayDeque<ParserPosition> = ArrayDeque()
fun save() {
checkpoints.addLast(this.copy())
}
fun reload() {
require(checkpoints.isNotEmpty()) {
ParsingError(ParsingErrorType.INTERNAL_ERROR, "No checkpoint to reload from", this)
}
val checkpoint = checkpoints.removeLast()
offset = checkpoint.offset
}
override fun toString(): String {
return "at $offset"
}
}

62
tests/lexer/ScanPrologParserTests.kt
View file

@ -1,62 +0,0 @@
package lexer
import lexer.errors.LexingError
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import kotlin.test.assertEquals
/**
* Tests for the Prolog lexer.
*
* These tests are based on the Prolog syntax.
*/
class ScanPrologParserTests {
@Test
fun scan_simple_atom() {
val tokens = Lexer("atom.").scan()
assertEquals(3, tokens.size)
assertEquals(TokenType.ALPHANUMERIC, tokens[0].type, "Expected ALPHANUMERIC token, got ${tokens[0].type}")
assertEquals(TokenType.DOT, tokens[1].type, "Expected DOT token, got ${tokens[1].type}")
assertEquals(TokenType.EOF, tokens[2].type, "Expected EOF token, got ${tokens[2].type}")
}
@Test
fun scan_variable() {
val tokens = Lexer("X.").scan()
assertEquals(3, tokens.size)
assertEquals(TokenType.ALPHANUMERIC, tokens[0].type, "Expected ALPHANUMERIC token, got ${tokens[0].type}")
assertEquals(TokenType.DOT, tokens[1].type, "Expected DOT token, got ${tokens[1].type}")
assertEquals(TokenType.EOF, tokens[2].type, "Expected EOF token, got ${tokens[2].type}")
}
@Test
fun scan_variable_with_number() {
val tokens = Lexer("X1.").scan()
assertEquals(3, tokens.size)
assertEquals(TokenType.ALPHANUMERIC, tokens[0].type, "Expected ALPHANUMERIC token, got ${tokens[0].type}")
assertEquals(TokenType.DOT, tokens[1].type, "Expected DOT token, got ${tokens[1].type}")
assertEquals(TokenType.EOF, tokens[2].type, "Expected EOF token, got ${tokens[2].type}")
}
@Test
fun scan_variable_with_underscore() {
val tokens = Lexer("X_1.").scan()
assertEquals(3, tokens.size)
assertEquals(TokenType.ALPHANUMERIC, tokens[0].type, "Expected ALPHANUMERIC token, got ${tokens[0].type}")
assertEquals(TokenType.DOT, tokens[1].type, "Expected DOT token, got ${tokens[1].type}")
assertEquals(TokenType.EOF, tokens[2].type, "Expected EOF token, got ${tokens[2].type}")
}
@Test
fun scan_variable_that_starts_with_a_number() {
assertThrows<LexingError> { Lexer("1X.").scan() }
}
}

191
tests/lexer/ScanTests.kt
View file

@ -1,191 +0,0 @@
package lexer
import lexer.errors.LexingError
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import org.junit.jupiter.api.Assertions.*
class ScanTests {
@Test
fun scan_emptyString_returns_EOF() {
val tokens = Lexer("").scan()
assertEquals(1, tokens.size, "Expected 1 token, got ${tokens.size}")
assertEquals(TokenType.EOF, tokens[0].type, "Expected EOF token, got ${tokens[0].type}")
}
@Test
fun scan_unknownSymbol_returns_Error() {
assertThrows<LexingError> { Lexer("€").scan() }
}
@Test
fun scan_dot_returns_Dot() {
val tokens = Lexer(".").scan()
assertEquals(2, tokens.size)
assertEquals(TokenType.DOT, tokens[0].type, "Expected DOT token, got ${tokens[0].type}")
assertEquals(TokenType.EOF, tokens[1].type, "Expected EOF token, got ${tokens[1].type}")
}
@Test
fun scan_two_dots_returns_two_dots() {
val tokens = Lexer("..").scan()
assertEquals(3, tokens.size)
assertEquals(TokenType.DOT, tokens[0].type, "Expected DOT token, got ${tokens[0].type}")
assertEquals(TokenType.DOT, tokens[1].type, "Expected DOT token, got ${tokens[1].type}")
assertEquals(TokenType.EOF, tokens[2].type, "Expected EOF token, got ${tokens[2].type}")
}
@Test
fun scan_letter_returns_letter() {
val tokens = Lexer("a").scan()
assertEquals(2, tokens.size)
assertEquals(TokenType.ALPHANUMERIC, tokens[0].type, "Expected ALPHANUMERIC token, got ${tokens[0].type}")
assertEquals(TokenType.EOF, tokens[1].type, "Expected EOF token, got ${tokens[1].type}")
assertEquals(0, tokens[0].position.line, "Expected line 0, got ${tokens[0].position.line}")
assertEquals(0, tokens[0].position.column, "Expected column 0, got ${tokens[0].position.column}")
assertEquals(1, tokens[0].position.length, "Expected length 1, got ${tokens[0].position.length}")
}
@Test
fun scan_word_returns_alphanumerics() {
val lexer = Lexer("word")
val tokens = lexer.scan()
assertEquals(2, tokens.size)
assertEquals(TokenType.ALPHANUMERIC, tokens[0].type, "Expected ALPHANUMERIC token, got ${tokens[0].type}")
assertEquals(TokenType.EOF, tokens[1].type, "Expected EOF token, got ${tokens[1].type}")
assertEquals(4, tokens[0].position.length, "Expected length 4, got ${tokens[0].position.length}")
assertEquals("word", tokens[0].value, "Expected 'word', got ${tokens[0].value}")
}
@Test
fun scan_space_returns_nothing() {
val lexer = Lexer(" ")
val tokens = lexer.scan()
assertEquals(1, tokens.size)
assertEquals(TokenType.EOF, tokens[0].type, "Expected EOF token, got ${tokens[0].type}")
}
@Test
fun scan_whitespace_various_returns_nothing() {
val lexer = Lexer(" \t\n\r")
val tokens = lexer.scan()
assertEquals(1, tokens.size)
assertEquals(TokenType.EOF, tokens[0].type, "Expected EOF token, got ${tokens[0].type}")
}
@Test
fun scan_separated_words() {
val tokens = Lexer("word1 word2").scan()
assertEquals(3, tokens.size, "Expected 3 tokens, got ${tokens.size}")
assertEquals(TokenType.ALPHANUMERIC, tokens[0].type, "Expected ALPHANUMERIC token, got ${tokens[0].type}")
assertEquals("word1", tokens[0].value, "Expected 'word1', got ${tokens[0].value}")
assertEquals(5, tokens[0].position.length, "Expected length 5, got ${tokens[0].position.length}")
assertEquals(TokenType.ALPHANUMERIC, tokens[1].type, "Expected ALPHANUMERIC token, got ${tokens[1].type}")
assertEquals("word2", tokens[1].value, "Expected 'word2', got ${tokens[1].value}")
assertEquals(5, tokens[1].position.length, "Expected length 5, got ${tokens[1].position.length}")
}
@Test
fun scan_multiline() {
val tokens = Lexer(
"""
word1
word2
""".trimIndent()
).scan()
assertEquals(3, tokens.size, "Expected 3 tokens, got ${tokens.size}")
assertEquals(TokenType.ALPHANUMERIC, tokens[0].type, "Expected ALPHANUMERIC token, got ${tokens[0].type}")
assertEquals("word1", tokens[0].value, "Expected 'word1', got ${tokens[0].value}")
assertEquals(5, tokens[0].position.length, "Expected length 5, got ${tokens[0].position.length}")
assertEquals(TokenType.ALPHANUMERIC, tokens[1].type, "Expected ALPHANUMERIC token, got ${tokens[1].type}")
assertEquals("word2", tokens[1].value, "Expected 'word2', got ${tokens[1].value}")
assertEquals(5, tokens[1].position.length, "Expected length 5, got ${tokens[1].position.length}")
}
@Test
fun scan_parenthesis_returns_parenthesis() {
val lexer = Lexer("()")
val tokens = lexer.scan()
assertEquals(3, tokens.size)
assertEquals(
TokenType.PARENTHESIS_LEFT,
tokens[0].type,
"Expected LEFT_PARENTHESES token, got ${tokens[0].type}"
)
assertEquals(
TokenType.PARENTHESIS_RIGHT,
tokens[1].type,
"Expected RIGHT_PARENTHESES token, got ${tokens[1].type}"
)
assertEquals(TokenType.EOF, tokens[2].type, "Expected EOF token, got ${tokens[2].type}")
}
@Test
fun scan_simple_quoted_string_returns_string() {
val lexer = Lexer("\"string\"")
val tokens = lexer.scan()
assertEquals(2, tokens.size)
assertEquals(TokenType.ALPHANUMERIC, tokens[0].type, "Expected ALPHANUMERIC token, got ${tokens[0].type}")
assertEquals(TokenType.EOF, tokens[1].type, "Expected EOF token, got ${tokens[1].type}")
assertEquals("string", tokens[0].value, "Expected 'string', got ${tokens[0].value}")
}
@Test
fun scan_quoted_string_with_space_returns_string() {
val lexer = Lexer("\"string with space\"")
val tokens = lexer.scan()
assertEquals(2, tokens.size)
assertEquals(TokenType.ALPHANUMERIC, tokens[0].type, "Expected ALPHANUMERIC token, got ${tokens[0].type}")
assertEquals(TokenType.EOF, tokens[1].type, "Expected EOF token, got ${tokens[1].type}")
assertEquals("string with space", tokens[0].value, "Expected 'string with space', got ${tokens[0].value}")
}
@Test
fun scan_comments_returns_nothing() {
val lexer = Lexer("% comment")
val tokens = lexer.scan()
assertEquals(1, tokens.size)
assertEquals(TokenType.EOF, tokens[0].type, "Expected EOF token, got ${tokens[0].type}")
}
@Test
fun scan_comment_and_sentence_returns_sentence() {
val tokens = Lexer("""
% comment
sentence
""".trimIndent()).scan()
assertEquals(2, tokens.size)
assertEquals(TokenType.ALPHANUMERIC, tokens[0].type, "Expected ALPHANUMERIC token, got ${tokens[0].type}")
assertEquals("sentence", tokens[0].value, "Expected 'sentence', got ${tokens[0].value}")
}
}

4
tests/parser/ParseFromTextTests.kt
View file

@ -1,4 +0,0 @@
package parser
class ParseFromTextTests {
}

91
tests/parser/ParseTests.kt
View file

@ -1,91 +0,0 @@
package parser
import lexer.Token
import lexer.state.TokenPosition
import lexer.TokenType
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
import prolog.ast.terms.Atom
import prolog.ast.terms.CompoundTerm
class ParseTests {
@Test
fun `parse atom a`() {
val input = Token(TokenType.ALPHANUMERIC, "a", TokenPosition(0, 0, 1))
val result = Parser(listOf(input)).parse()
assertEquals(1, result.size, "Expected 1 term")
assertEquals(Atom("a"), result[0], "Expected atom 'a'")
}
@Test
fun `parse atom foo`() {
val input = Token(TokenType.ALPHANUMERIC, "foo", TokenPosition(0, 0, 3))
val result = Parser(listOf(input)).parse()
assertEquals(1, result.size, "Expected 1 term")
assertEquals(Atom("foo"), result[0], "Expected atom 'foo'")
}
@Test
fun `parse atom foo1`() {
val input = Token(TokenType.ALPHANUMERIC, "foo1", TokenPosition(0, 0, 4))
val result = Parser(listOf(input)).parse()
assertEquals(1, result.size, "Expected 1 term")
assertEquals(Atom("foo1"), result[0], "Expected atom 'foo1'")
}
@Test
fun `parse atom fooBar`() {
val name = "fooBar"
val input = Token(TokenType.ALPHANUMERIC, name, TokenPosition(0, 0, 6))
val result = Parser(listOf(input)).parse()
assertEquals(1, result.size, "Expected 1 term")
assertEquals(Atom(name), result[0], "Expected atom 'fooBar'")
}
@Test
fun `parse atom foo_bar`() {
val name = "foo_bar"
val input = Token(TokenType.ALPHANUMERIC, name, TokenPosition(0, 0, 7))
val result = Parser(listOf(input)).parse()
assertEquals(1, result.size, "Expected 1 term")
assertEquals(Atom(name), result[0], "Expected atom 'foo_bar'")
}
@Test
fun `parse atom my_FooBar1`() {
val name = "my_FooBar1"
val input = Token(TokenType.ALPHANUMERIC, name, TokenPosition(0, 0, 11))
val result = Parser(listOf(input)).parse()
assertEquals(1, result.size, "Expected 1 term")
assertEquals(Atom(name), result[0], "Expected atom 'my_FooBar1'")
}
@Test
fun `parse compound term f()`() {
val input = listOf(
Token(TokenType.ALPHANUMERIC, "f", TokenPosition(0, 0, 1)),
Token(TokenType.PARENTHESIS_LEFT, "(", TokenPosition(0, 1, 2)),
Token(TokenType.PARENTHESIS_RIGHT, ")", TokenPosition(0, 3, 4))
)
val result = Parser(input).parse()
assertEquals(1, result.size, "Expected 1 term")
assertTrue(result[0] is CompoundTerm)
assertEquals("f", (result[0] as CompoundTerm).name)
assertEquals(0, (result[0] as CompoundTerm).arguments.size)
}
}