Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ internal interface Node {
}

/**
* An binary operator node
* A binary operator node
*
* @property left is its left child
* @property op is the actual operator
Expand Down
24 changes: 17 additions & 7 deletions src/commonMain/kotlin/com/notkamui/keval/Tokenizer.kt
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,10 @@ private fun Sequence<String>.normalizeTokens(symbols: Map<String, KevalOperator>
ret.add(token)
} else {
throw KevalInvalidSymbolException(
token, joinToString(""), currentPos, "comma can only be used in the context of a function"
token,
joinToString(""),
currentPos,
"comma can only be used in the context of a function"
)
}
}
Expand Down Expand Up @@ -85,13 +88,20 @@ internal fun String.isKevalOperator(symbolsSet: Set<String>): Boolean = this in
* @return the list of tokens
* @throws KevalInvalidSymbolException if the expression contains an invalid symbol
*/
internal fun String.tokenize(symbolsSet: Map<String, KevalOperator>): List<String> =
TOKENIZER_REGEX.findAll(this)
internal fun String.tokenize(symbolsSet: Map<String, KevalOperator>): List<String> {
val constantsSet = symbolsSet.filter { it.value is KevalConstant }

val tokenizerRegex = if (constantsSet.isNotEmpty()) {
val constantsRegexString = constantsSet.keys.joinToString(separator = "|")
"""(\d+\.\d+|\d+|$constantsRegexString|[a-zA-Z_]\w*|[^\w\s])"""
} else {
"""(\d+\.\d+|\d+|[a-zA-Z_]\w*|[^\w\s])"""
}.toRegex()

return tokenizerRegex.findAll(this)
.map(MatchResult::value)
.filter(String::isNotBlank)
.map { SANITIZE_REGEX.replace(it, "") }
.normalizeTokens(symbolsSet)

private val SANITIZE_REGEX = """\s+""".toRegex()

private val TOKENIZER_REGEX = """(\d+\.\d+|\d+|[a-zA-Z_]\w*|[^\w\s])""".toRegex()
}
private val SANITIZE_REGEX = """\s+""".toRegex()
20 changes: 20 additions & 0 deletions src/commonTest/kotlin/com/notkamui/keval/TokenizerTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -103,4 +103,24 @@ class TokenizerTest {
val nodes = "f(((1)))".tokenize(k.resourcesView())
assertEquals("f(((1)))", nodes.joinToString(separator = ""))
}

@Test
fun symbolsAndNumbers() {
listOf(
listOf("PI","9"),
listOf("9", "e"),
listOf("PI", "e"),
listOf("PI", "PI"),
listOf("PI", "e", "e", "PI")
).forEach { tokenList ->
// Intersperses the assumed "*"
val expected = tokenList.dropLast(1).flatMap { listOf(it, "*") } + tokenList.last()
assertEquals(
expected,
tokenList
.joinToString(separator = "")
.tokenize(KevalBuilder.DEFAULT_RESOURCES)
)
}
}
}