From 8d2e2246cb55afe08927fceb55331f77604c390b Mon Sep 17 00:00:00 2001 From: usbharu Date: Sat, 16 Nov 2024 23:04:12 +0900 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=AE=B5=E8=90=BD=E3=82=92=E3=81=A1?= =?UTF-8?q?=E3=82=83=E3=82=93=E3=81=A8=E5=87=A6=E7=90=86=E3=81=A7=E3=81=8D?= =?UTF-8?q?=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kotlin/dev/usbharu/markdown/AstNode.kt | 12 +++++ .../kotlin/dev/usbharu/markdown/Lexer.kt | 24 ++++++--- .../kotlin/dev/usbharu/markdown/Parser.kt | 51 ++++++++++++++++--- .../dev/usbharu/markdown/PeekableIterator.kt | 4 ++ .../kotlin/dev/usbharu/markdown/Token.kt | 3 +- .../kotlin/dev/usbharu/markdown/LexerTest.kt | 30 +++++------ .../kotlin/dev/usbharu/markdown/ParserTest.kt | 38 +++++++++++++- 7 files changed, 128 insertions(+), 34 deletions(-) diff --git a/library/src/commonMain/kotlin/dev/usbharu/markdown/AstNode.kt b/library/src/commonMain/kotlin/dev/usbharu/markdown/AstNode.kt index f834e96..c2fb37f 100644 --- a/library/src/commonMain/kotlin/dev/usbharu/markdown/AstNode.kt +++ b/library/src/commonMain/kotlin/dev/usbharu/markdown/AstNode.kt @@ -106,5 +106,17 @@ sealed class AstNode { return name } } + + data class InlineCodeNode(val code: String) : InlineNode() { + override fun print(): String { + return "`$code`" + } + } + + data class SimpleUrlNode(val url: String) : InlineNode() { + override fun print(): String { + return url + } + } } diff --git a/library/src/commonMain/kotlin/dev/usbharu/markdown/Lexer.kt b/library/src/commonMain/kotlin/dev/usbharu/markdown/Lexer.kt index 696169a..eb553aa 100644 --- a/library/src/commonMain/kotlin/dev/usbharu/markdown/Lexer.kt +++ b/library/src/commonMain/kotlin/dev/usbharu/markdown/Lexer.kt @@ -89,20 +89,24 @@ class Lexer { } else if (htmlNest != 0) { codeBuffer.append(" ") } else { - tokens.add(Break(1)) + addBreak(tokens) } } inQuote = false } val lastToken = tokens.lastOrNull() - if (lastToken is Break) { + if (lastToken is LineBreak) { if (lastToken.count == 1) { tokens.removeLast() } else { lastToken.count-- } } + if (lastToken is BlockBreak) { + tokens.removeLast() + tokens.add(LineBreak(1)) + } return tokens } @@ -517,15 +521,19 @@ class Lexer { lines: PeekableStringIterator, tokens: MutableList, ) { - var count = 0 while (lines.peekOrNull() == "") { - lines.next() - count++ + lines.skip() + addBreak(tokens) } - if (tokens.lastOrNull() is Break) { - tokens[tokens.lastIndex] = Break(count + 1) + } + + fun addBreak(tokens: MutableList) { + val lastOrNull = tokens.lastOrNull() + if (lastOrNull is LineBreak && 1 <= lastOrNull.count) { + tokens.removeLast() + tokens.add(BlockBreak) } else { - tokens.add(Break(count)) + tokens.add(LineBreak(1)) } } } \ No newline at end of file diff --git a/library/src/commonMain/kotlin/dev/usbharu/markdown/Parser.kt b/library/src/commonMain/kotlin/dev/usbharu/markdown/Parser.kt index bb978ae..21acf07 100644 --- a/library/src/commonMain/kotlin/dev/usbharu/markdown/Parser.kt +++ b/library/src/commonMain/kotlin/dev/usbharu/markdown/Parser.kt @@ -16,12 +16,12 @@ class Parser { val node = when (val next = iterator.next()) { is Asterisk, is InlineCodeBlock, is Strike, is Text, is Whitespace, Exclamation, ParenthesesEnd, ParenthesesStart, - SquareBracketStart, SquareBracketEnd, is Url, is UrlTitle -> paragraph( + SquareBracketStart, SquareBracketEnd, is Url, is UrlTitle, is LineBreak -> paragraph( next, iterator ) - is Break -> null //todo ただの改行と段落分けの改行のトークンを分ける + is BlockBreak -> null is CheckBox -> TODO() is CodeBlock -> TODO() is CodeBlockLanguage -> TODO() @@ -53,8 +53,31 @@ class Parser { return HeaderNode(header.count, headerTextNode) } - fun paragraph(token: Token, iterator: PeekableTokenIterator): AstNode { - return ParagraphNode(inline(token, iterator)) + fun paragraph(token: Token, iterator: PeekableTokenIterator): AstNode? { + val list = mutableListOf() + var token2: Token? = token + do { + list.addAll(inline(token2!!, iterator)) + if (iterator.hasNext() && isInline(iterator.peekOrNull())) { + token2 = iterator.next() + } else { + token2 = iterator.peekOrNull() + } + } while (iterator.hasNext() && isInline(token2)) + if (list.isEmpty()) { + return null + } + return ParagraphNode(list) + } + + fun isInline(token: Token?): Boolean { + return when (token) { + is Asterisk, is InlineCodeBlock, is Strike, + is Text, is Whitespace, Exclamation, ParenthesesEnd, ParenthesesStart, + SquareBracketStart, SquareBracketEnd, is Url, is UrlTitle, is LineBreak -> true + + else -> false + } } fun inline(token: Token, iterator: PeekableTokenIterator): MutableList { @@ -63,20 +86,32 @@ class Parser { val node = when (token) { is Asterisk -> asterisk(token, iterator) Exclamation -> image(Exclamation, iterator) - is InlineCodeBlock -> TODO() + is InlineCodeBlock -> inlineCodeBlock(token, iterator) ParenthesesEnd -> PlainText(")") ParenthesesStart -> PlainText("(") SquareBracketEnd -> PlainText("]") SquareBracketStart -> url(SquareBracketStart, iterator) is Strike -> TODO() is Text -> plainText(token, iterator) - is Url -> TODO() + is Url -> inlineUrl(token, iterator) is UrlTitle -> PlainText("\"${token.title}\"") is Whitespace -> whitespace(token, iterator) + is LineBreak -> null else -> TODO() } - return mutableListOf(node) + if (node != null) { + return mutableListOf(node) + } + return mutableListOf() + } + + fun inlineUrl(url: Url, iterator: PeekableTokenIterator): SimpleUrlNode { + return SimpleUrlNode(url.url) + } + + fun inlineCodeBlock(inlineCodeBlock: InlineCodeBlock, iterator: PeekableTokenIterator): InlineCodeNode { + return InlineCodeNode(inlineCodeBlock.text) } fun whitespace(token: Whitespace, iterator: PeekableTokenIterator): InlineNode { @@ -207,7 +242,7 @@ class Parser { var counter = 0 val tokens = mutableListOf() while (iterator.peekOrNull(counter) != null && - iterator.peekOrNull(counter) !is Break && + iterator.peekOrNull(counter) !is LineBreak && iterator.peekOrNull(counter) !is Asterisk ) { tokens.add(iterator.peekOrNull(counter)!!) diff --git a/library/src/commonMain/kotlin/dev/usbharu/markdown/PeekableIterator.kt b/library/src/commonMain/kotlin/dev/usbharu/markdown/PeekableIterator.kt index c0972d4..d773f9d 100644 --- a/library/src/commonMain/kotlin/dev/usbharu/markdown/PeekableIterator.kt +++ b/library/src/commonMain/kotlin/dev/usbharu/markdown/PeekableIterator.kt @@ -33,6 +33,10 @@ class PeekableStringIterator(private val list: List) : Iterator fun peekOrNull(): String? = list.getOrNull(index) fun current(): Int = index + + fun skip(count: Int = 1) { + index += count + } } class PeekableTokenIterator(private val tokens: List) : Iterator { diff --git a/library/src/commonMain/kotlin/dev/usbharu/markdown/Token.kt b/library/src/commonMain/kotlin/dev/usbharu/markdown/Token.kt index c9025c2..d155a2e 100644 --- a/library/src/commonMain/kotlin/dev/usbharu/markdown/Token.kt +++ b/library/src/commonMain/kotlin/dev/usbharu/markdown/Token.kt @@ -5,7 +5,8 @@ import kotlin.js.JsExport @JsExport sealed class Token { data class Text(var text: String) : Token() - data class Break(var count: Int) : Token() + data class LineBreak(var count: Int) : Token() + data object BlockBreak : Token() data class Header(var count: Int) : Token() data class Quote(var count: Int) : Token() data class Separator(var count: Int, val char: Char) : Token() diff --git a/library/src/commonTest/kotlin/dev/usbharu/markdown/LexerTest.kt b/library/src/commonTest/kotlin/dev/usbharu/markdown/LexerTest.kt index 1c95516..92fa9e3 100644 --- a/library/src/commonTest/kotlin/dev/usbharu/markdown/LexerTest.kt +++ b/library/src/commonTest/kotlin/dev/usbharu/markdown/LexerTest.kt @@ -15,7 +15,7 @@ class LexerTest { println(actual) - assertContentEquals(listOf(Break(1)), actual) + assertContentEquals(listOf(LineBreak(1)), actual) } @Test @@ -26,7 +26,7 @@ class LexerTest { println(actual) - assertContentEquals(listOf(Break(1)), actual) + assertContentEquals(listOf(LineBreak(1)), actual) } @Test @@ -37,7 +37,7 @@ class LexerTest { println(actual) - assertContentEquals(listOf(Break(2)), actual) + assertContentEquals(listOf(BlockBreak), actual) } @Test @@ -59,7 +59,7 @@ class LexerTest { println(actual) - assertContentEquals(listOf(Text("abcd"), Break(1), Text("efgh")), actual) + assertContentEquals(listOf(Text("abcd"), LineBreak(1), Text("efgh")), actual) } @Test @@ -70,7 +70,7 @@ class LexerTest { println(actual) - assertContentEquals(listOf(Text("abcd"), Break(2), Text("efgh")), actual) + assertContentEquals(listOf(Text("abcd"), BlockBreak, Text("efgh")), actual) } @Test @@ -141,7 +141,7 @@ class LexerTest { listOf( Header(1), Text("a"), - Break(1), + LineBreak(1), Header(1), Text("b") ), actual @@ -331,15 +331,15 @@ class LexerTest { DiscList, CheckBox(false), Text("a"), - Break(1), + LineBreak(1), DiscList, CheckBox(true), Text("b"), - Break(1), + LineBreak(1), DiscList, CheckBox(false), Text("c"), - Break(1), + LineBreak(1), DiscList, CheckBox(true), Text("d"), @@ -366,7 +366,7 @@ class LexerTest { println(actual) - assertContentEquals(listOf(DiscList, Text("aiueo"), Break(1), DiscList, Text("abcd")), actual) + assertContentEquals(listOf(DiscList, Text("aiueo"), LineBreak(1), DiscList, Text("abcd")), actual) } @Test @@ -378,7 +378,7 @@ class LexerTest { println(actual) assertContentEquals( - listOf(DiscList, Text("aiueo"), Break(1), Whitespace(4, ' '), DiscList, Text("abcd")), actual + listOf(DiscList, Text("aiueo"), LineBreak(1), Whitespace(4, ' '), DiscList, Text("abcd")), actual ) } @@ -392,7 +392,7 @@ class LexerTest { assertContentEquals( listOf( - DecimalList('1'), Text("aiueo"), Break(1), Whitespace(4, ' '), DecimalList('2'), Text("abcd") + DecimalList('1'), Text("aiueo"), LineBreak(1), Whitespace(4, ' '), DecimalList('2'), Text("abcd") ), actual ) } @@ -407,7 +407,7 @@ class LexerTest { assertContentEquals( listOf( - DecimalList('１'), Text("aiueo"), Break(1), Whitespace(4, ' '), DecimalList('２'), Text("abcd") + DecimalList('１'), Text("aiueo"), LineBreak(1), Whitespace(4, ' '), DecimalList('２'), Text("abcd") ), actual ) } @@ -422,7 +422,7 @@ class LexerTest { assertContentEquals( listOf( - DecimalList('1'), Text("aiueo"), Break(1), Whitespace(4, ' '), DecimalList('2'), Text("abcd") + DecimalList('1'), Text("aiueo"), LineBreak(1), Whitespace(4, ' '), DecimalList('2'), Text("abcd") ), actual ) } @@ -463,7 +463,7 @@ class LexerTest { assertContentEquals( listOf( - Text("こんにちは～"), Whitespace(1, ' '), Url("https://example.com"), Break(1), Text("あいうえお") + Text("こんにちは～"), Whitespace(1, ' '), Url("https://example.com"), LineBreak(1), Text("あいうえお") ), actual ) } diff --git a/library/src/commonTest/kotlin/dev/usbharu/markdown/ParserTest.kt b/library/src/commonTest/kotlin/dev/usbharu/markdown/ParserTest.kt index c3eca2d..48dfca4 100644 --- a/library/src/commonTest/kotlin/dev/usbharu/markdown/ParserTest.kt +++ b/library/src/commonTest/kotlin/dev/usbharu/markdown/ParserTest.kt @@ -30,7 +30,7 @@ class ParserTest { fun header複数() { val parser = Parser() - val actual = parser.parse(listOf(Header(1), Text("a b c"), Break(1), Header(2), Text("d e f"))) + val actual = parser.parse(listOf(Header(1), Text("a b c"), LineBreak(1), Header(2), Text("d e f"))) println(actual) println(actual.print()) @@ -235,7 +235,7 @@ class ParserTest { fun separator2() { val parser = Parser() - val actual = parser.parse(listOf(Separator(3, '-'), Break(1), Separator(3, '-'))) + val actual = parser.parse(listOf(Separator(3, '-'), LineBreak(1), Separator(3, '-'))) println(actual) println(actual.print()) @@ -309,4 +309,38 @@ class ParserTest { ), actual ) } + + @Test + fun 複数段落() { + val parser = Parser() + + val actual = parser.parse( + listOf( + Text("aiueo"), LineBreak(1), Text("abcd"), BlockBreak, Text("hoge") + ) + ) + + println(actual) + println(actual.print()) + + assertEquals( + RootNode( + BodyNode( + listOf( + ParagraphNode( + listOf( + PlainText("aiueo"), + PlainText(("abcd")) + ) + ), + ParagraphNode( + listOf( + PlainText("hoge") + ) + ) + ) + ) + ), actual + ) + } } \ No newline at end of file