usbharu
/
markdown-kt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: 段落をちゃんと処理できるように

This commit is contained in:
usbharu 2024-11-16 23:04:12 +09:00
parent 579343145e
commit 8d2e2246cb
Signed by: usbharu
GPG Key ID: 95CBCF7046307B77
7 changed files with 128 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
library/src/commonMain/kotlin/dev/usbharu/markdown/AstNode.kt
View File

@ -106,5 +106,17 @@ sealed class AstNode {
return name
}
}
data class InlineCodeNode(val code: String) : InlineNode() {
override fun print(): String {
return "`$code`"
}
}
data class SimpleUrlNode(val url: String) : InlineNode() {
override fun print(): String {
return url
}
}
}

24
library/src/commonMain/kotlin/dev/usbharu/markdown/Lexer.kt
View File

@ -89,20 +89,24 @@ class Lexer {
} else if (htmlNest != 0) {
codeBuffer.append(" ")
} else {
tokens.add(Break(1))
addBreak(tokens)
}
}
inQuote = false
}
val lastToken = tokens.lastOrNull()
if (lastToken is Break) {
if (lastToken is LineBreak) {
if (lastToken.count == 1) {
tokens.removeLast()
} else {
lastToken.count--
}
}
if (lastToken is BlockBreak) {
tokens.removeLast()
tokens.add(LineBreak(1))
}
return tokens
}
@ -517,15 +521,19 @@ class Lexer {
lines: PeekableStringIterator,
tokens: MutableList<Token>,
) {
var count = 0
while (lines.peekOrNull() == "") {
lines.next()
count++
lines.skip()
addBreak(tokens)
}
if (tokens.lastOrNull() is Break) {
tokens[tokens.lastIndex] = Break(count + 1)
}
fun addBreak(tokens: MutableList<Token>) {
val lastOrNull = tokens.lastOrNull()
if (lastOrNull is LineBreak && 1 <= lastOrNull.count) {
tokens.removeLast()
tokens.add(BlockBreak)
} else {
tokens.add(Break(count))
tokens.add(LineBreak(1))
}
}
}

49
library/src/commonMain/kotlin/dev/usbharu/markdown/Parser.kt
View File

@ -16,12 +16,12 @@ class Parser {
val node = when (val next = iterator.next()) {
is Asterisk, is InlineCodeBlock, is Strike,
is Text, is Whitespace, Exclamation, ParenthesesEnd, ParenthesesStart,
SquareBracketStart, SquareBracketEnd, is Url, is UrlTitle -> paragraph(
SquareBracketStart, SquareBracketEnd, is Url, is UrlTitle, is LineBreak -> paragraph(
next,
iterator
)
is Break -> null //todo ただの改行と段落分けの改行のトークンを分ける
is BlockBreak -> null
is CheckBox -> TODO()
is CodeBlock -> TODO()
is CodeBlockLanguage -> TODO()
@ -53,8 +53,31 @@ class Parser {
return HeaderNode(header.count, headerTextNode)
}
fun paragraph(token: Token, iterator: PeekableTokenIterator): AstNode {
return ParagraphNode(inline(token, iterator))
fun paragraph(token: Token, iterator: PeekableTokenIterator): AstNode? {
val list = mutableListOf<InlineNode>()
var token2: Token? = token
do {
list.addAll(inline(token2!!, iterator))
if (iterator.hasNext() && isInline(iterator.peekOrNull())) {
token2 = iterator.next()
} else {
token2 = iterator.peekOrNull()
}
} while (iterator.hasNext() && isInline(token2))
if (list.isEmpty()) {
return null
}
return ParagraphNode(list)
}
fun isInline(token: Token?): Boolean {
return when (token) {
is Asterisk, is InlineCodeBlock, is Strike,
is Text, is Whitespace, Exclamation, ParenthesesEnd, ParenthesesStart,
SquareBracketStart, SquareBracketEnd, is Url, is UrlTitle, is LineBreak -> true
else -> false
}
}
fun inline(token: Token, iterator: PeekableTokenIterator): MutableList<InlineNode> {
@ -63,21 +86,33 @@ class Parser {
val node = when (token) {
is Asterisk -> asterisk(token, iterator)
Exclamation -> image(Exclamation, iterator)
is InlineCodeBlock -> TODO()
is InlineCodeBlock -> inlineCodeBlock(token, iterator)
ParenthesesEnd -> PlainText(")")
ParenthesesStart -> PlainText("(")
SquareBracketEnd -> PlainText("]")
SquareBracketStart -> url(SquareBracketStart, iterator)
is Strike -> TODO()
is Text -> plainText(token, iterator)
is Url -> TODO()
is Url -> inlineUrl(token, iterator)
is UrlTitle -> PlainText("\"${token.title}\"")
is Whitespace -> whitespace(token, iterator)
is LineBreak -> null
else -> TODO()
}
if (node != null) {
return mutableListOf(node)
}
return mutableListOf()
}
fun inlineUrl(url: Url, iterator: PeekableTokenIterator): SimpleUrlNode {
return SimpleUrlNode(url.url)
}
fun inlineCodeBlock(inlineCodeBlock: InlineCodeBlock, iterator: PeekableTokenIterator): InlineCodeNode {
return InlineCodeNode(inlineCodeBlock.text)
}
fun whitespace(token: Whitespace, iterator: PeekableTokenIterator): InlineNode {
return PlainText(token.whitespace.toString().repeat(token.count))
@ -207,7 +242,7 @@ class Parser {
var counter = 0
val tokens = mutableListOf<Token>()
while (iterator.peekOrNull(counter) != null &&
iterator.peekOrNull(counter) !is Break &&
iterator.peekOrNull(counter) !is LineBreak &&
iterator.peekOrNull(counter) !is Asterisk
) {
tokens.add(iterator.peekOrNull(counter)!!)

4
library/src/commonMain/kotlin/dev/usbharu/markdown/PeekableIterator.kt
View File

@ -33,6 +33,10 @@ class PeekableStringIterator(private val list: List<String>) : Iterator<String>
fun peekOrNull(): String? = list.getOrNull(index)
fun current(): Int = index
fun skip(count: Int = 1) {
index += count
}
}
class PeekableTokenIterator(private val tokens: List<Token>) : Iterator<Token> {

3
library/src/commonMain/kotlin/dev/usbharu/markdown/Token.kt
View File

@ -5,7 +5,8 @@ import kotlin.js.JsExport
@JsExport
sealed class Token {
data class Text(var text: String) : Token()
data class Break(var count: Int) : Token()
data class LineBreak(var count: Int) : Token()
data object BlockBreak : Token()
data class Header(var count: Int) : Token()
data class Quote(var count: Int) : Token()
data class Separator(var count: Int, val char: Char) : Token()

30
library/src/commonTest/kotlin/dev/usbharu/markdown/LexerTest.kt
View File

@ -15,7 +15,7 @@ class LexerTest {
println(actual)
assertContentEquals(listOf(Break(1)), actual)
assertContentEquals(listOf(LineBreak(1)), actual)
}
@Test
@ -26,7 +26,7 @@ class LexerTest {
println(actual)
assertContentEquals(listOf(Break(1)), actual)
assertContentEquals(listOf(LineBreak(1)), actual)
}
@Test
@ -37,7 +37,7 @@ class LexerTest {
println(actual)
assertContentEquals(listOf(Break(2)), actual)
assertContentEquals(listOf(BlockBreak), actual)
}
@Test
@ -59,7 +59,7 @@ class LexerTest {
println(actual)
assertContentEquals(listOf(Text("abcd"), Break(1), Text("efgh")), actual)
assertContentEquals(listOf(Text("abcd"), LineBreak(1), Text("efgh")), actual)
}
@Test
@ -70,7 +70,7 @@ class LexerTest {
println(actual)
assertContentEquals(listOf(Text("abcd"), Break(2), Text("efgh")), actual)
assertContentEquals(listOf(Text("abcd"), BlockBreak, Text("efgh")), actual)
}
@Test
@ -141,7 +141,7 @@ class LexerTest {
listOf(
Header(1),
Text("a"),
Break(1),
LineBreak(1),
Header(1),
Text("b")
), actual
@ -331,15 +331,15 @@ class LexerTest {
DiscList,
CheckBox(false),
Text("a"),
Break(1),
LineBreak(1),
DiscList,
CheckBox(true),
Text("b"),
Break(1),
LineBreak(1),
DiscList,
CheckBox(false),
Text("c"),
Break(1),
LineBreak(1),
DiscList,
CheckBox(true),
Text("d"),
@ -366,7 +366,7 @@ class LexerTest {
println(actual)
assertContentEquals(listOf(DiscList, Text("aiueo"), Break(1), DiscList, Text("abcd")), actual)
assertContentEquals(listOf(DiscList, Text("aiueo"), LineBreak(1), DiscList, Text("abcd")), actual)
}
@Test
@ -378,7 +378,7 @@ class LexerTest {
println(actual)
assertContentEquals(
listOf(DiscList, Text("aiueo"), Break(1), Whitespace(4, ' '), DiscList, Text("abcd")), actual
listOf(DiscList, Text("aiueo"), LineBreak(1), Whitespace(4, ' '), DiscList, Text("abcd")), actual
)
}
@ -392,7 +392,7 @@ class LexerTest {
assertContentEquals(
listOf(
DecimalList('1'), Text("aiueo"), Break(1), Whitespace(4, ' '), DecimalList('2'), Text("abcd")
DecimalList('1'), Text("aiueo"), LineBreak(1), Whitespace(4, ' '), DecimalList('2'), Text("abcd")
), actual
)
}
@ -407,7 +407,7 @@ class LexerTest {
assertContentEquals(
listOf(
DecimalList('１'), Text("aiueo"), Break(1), Whitespace(4, ' '), DecimalList('２'), Text("abcd")
DecimalList('１'), Text("aiueo"), LineBreak(1), Whitespace(4, ' '), DecimalList('２'), Text("abcd")
), actual
)
}
@ -422,7 +422,7 @@ class LexerTest {
assertContentEquals(
listOf(
DecimalList('1'), Text("aiueo"), Break(1), Whitespace(4, ' '), DecimalList('2'), Text("abcd")
DecimalList('1'), Text("aiueo"), LineBreak(1), Whitespace(4, ' '), DecimalList('2'), Text("abcd")
), actual
)
}
@ -463,7 +463,7 @@ class LexerTest {
assertContentEquals(
listOf(
Text("こんにちは～"), Whitespace(1, ' '), Url("https://example.com"), Break(1), Text("あいうえお")
Text("こんにちは～"), Whitespace(1, ' '), Url("https://example.com"), LineBreak(1), Text("あいうえお")
), actual
)
}

38
library/src/commonTest/kotlin/dev/usbharu/markdown/ParserTest.kt
View File

@ -30,7 +30,7 @@ class ParserTest {
fun header複数() {
val parser = Parser()
val actual = parser.parse(listOf(Header(1), Text("a b c"), Break(1), Header(2), Text("d e f")))
val actual = parser.parse(listOf(Header(1), Text("a b c"), LineBreak(1), Header(2), Text("d e f")))
println(actual)
println(actual.print())
@ -235,7 +235,7 @@ class ParserTest {
fun separator2() {
val parser = Parser()
val actual = parser.parse(listOf(Separator(3, '-'), Break(1), Separator(3, '-')))
val actual = parser.parse(listOf(Separator(3, '-'), LineBreak(1), Separator(3, '-')))
println(actual)
println(actual.print())
@ -309,4 +309,38 @@ class ParserTest {
), actual
)
}
@Test
fun 複数段落() {
val parser = Parser()
val actual = parser.parse(
listOf(
Text("aiueo"), LineBreak(1), Text("abcd"), BlockBreak, Text("hoge")
)
)
println(actual)
println(actual.print())
assertEquals(
RootNode(
BodyNode(
listOf(
ParagraphNode(
listOf(
PlainText("aiueo"),
PlainText(("abcd"))
)
),
ParagraphNode(
listOf(
PlainText("hoge")
)
)
)
)
), actual
)
}
}