From 985df8ed8224f6aaef75fc4b66a2b92c837d18f4 Mon Sep 17 00:00:00 2001 From: usbharu Date: Thu, 14 Nov 2024 11:46:45 +0900 Subject: [PATCH] =?UTF-8?q?=E3=82=B3=E3=83=BC=E3=83=89=E3=83=96=E3=83=AD?= =?UTF-8?q?=E3=83=83=E3=82=AF=E3=81=AB=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kotlin/dev/usbharu/markdown/Lexer.kt | 76 ++++++++- .../kotlin/dev/usbharu/markdown/Token.kt | 5 +- .../kotlin/dev/usbharu/markdown/LexerTest.kt | 152 ++++++++++++++++++ 3 files changed, 230 insertions(+), 3 deletions(-) diff --git a/library/src/commonMain/kotlin/dev/usbharu/markdown/Lexer.kt b/library/src/commonMain/kotlin/dev/usbharu/markdown/Lexer.kt index e982e60..af08faf 100644 --- a/library/src/commonMain/kotlin/dev/usbharu/markdown/Lexer.kt +++ b/library/src/commonMain/kotlin/dev/usbharu/markdown/Lexer.kt @@ -8,6 +8,9 @@ class Lexer { val lines = PeekableStringIterator(input.lines()) var inQuote = false + var inCode = false + + val codeBuffer = StringBuilder() line@ while (lines.hasNext()) { @@ -20,6 +23,68 @@ class Lexer { char@ while (iterator.hasNext()) { val next = iterator.next() when { + next == '`' || next == '｀' -> { + //todo ````` のようなやつが来たときのことを考える + if (iterator.peekOrNull() == next) { + val codeBlockBuilder = StringBuilder() + codeBlockBuilder.append(next) + codeBlockBuilder.append(iterator.next()) + if (iterator.peekOrNull() == next) { + codeBlockBuilder.append(iterator.next()) + if (iterator.peekOrNull() == next) { + tokens.add(Text(codeBlockBuilder.toString())) + } else { + if (inCode) { + inCode = false + tokens.add(CodeBlock(codeBuffer.toString().trimStart('\n').trimEnd('\n'))) + codeBuffer.clear() + } else { + inCode = true + var inFilename = false + val language = StringBuilder() + val filename = StringBuilder() + if (iterator.hasNext()) { + codeBlock@ while (iterator.hasNext()) { + val nextLanguage = iterator.next() + if ((nextLanguage == ':' || nextLanguage == '：') && !inFilename) { + inFilename = true + continue@codeBlock + } + if (inFilename) { + filename.append(nextLanguage) + } else { + language.append(nextLanguage) + } + + } + tokens.add(CodeBlockLanguage(language.toString(), filename.toString())) + } + + } + } + + } else if (iterator.peekOrNull() == null) { + tokens.add(Text(codeBlockBuilder.toString())) + } + + } else { + val codeBuilder = StringBuilder() + while (iterator.hasNext() && iterator.peekOrNull() != next) { + codeBuilder.append(iterator.next()) + } + if (iterator.hasNext() && iterator.next() == next) { //インラインコードブロックかと思ったら違った + tokens.add(InlineCodeBlock(codeBuilder.toString())) + } else { + tokens.add(Text(codeBuilder.insert(0, next).toString())) + } + + } + } + + inCode -> { + codeBuffer.append(next) + } + next == '#' || next == '＃' -> header(iterator, tokens) (next == '>' || next == '＞') && !inQuote -> { inQuote = true @@ -58,6 +123,7 @@ class Lexer { } } + else -> { val lastToken = tokens.lastOrNull() if (lastToken is Text) { @@ -69,7 +135,12 @@ class Lexer { } } - tokens.add(Break(1)) + + if (inCode) { + codeBuffer.append("\n") + } else { + tokens.add(Break(1)) + } } inQuote = false } @@ -137,9 +208,10 @@ class Lexer { while (iterator.hasNext() && iterator.peekOrNull() != doubleQuotation && iterator.peekOrNull() != ')') { titleBuilder.append(iterator.next()) } - if (iterator.peekOrNull() == '"') { + if (iterator.peekOrNull() == doubleQuotation) { iterator.next() } + tokens.add(UrlTitle(titleBuilder.toString())) } } } diff --git a/library/src/commonMain/kotlin/dev/usbharu/markdown/Token.kt b/library/src/commonMain/kotlin/dev/usbharu/markdown/Token.kt index a5947e5..5fd7130 100644 --- a/library/src/commonMain/kotlin/dev/usbharu/markdown/Token.kt +++ b/library/src/commonMain/kotlin/dev/usbharu/markdown/Token.kt @@ -25,4 +25,7 @@ data object ParenthesesEnd : Token() data class Url(var url: String) : Token() data class Asterisk(var count: Int, var char: Char) : Token() data object Exclamation : Token() -data class UrlTitle(val title: String) : Token() \ No newline at end of file +data class UrlTitle(val title: String) : Token() +data class InlineCodeBlock(val text: String) : Token() +data class CodeBlock(val text: String) : Token() +data class CodeBlockLanguage(val language: String, val filename: String) : Token() \ No newline at end of file diff --git a/library/src/commonTest/kotlin/dev/usbharu/markdown/LexerTest.kt b/library/src/commonTest/kotlin/dev/usbharu/markdown/LexerTest.kt index 7d4b03f..1b11ce9 100644 --- a/library/src/commonTest/kotlin/dev/usbharu/markdown/LexerTest.kt +++ b/library/src/commonTest/kotlin/dev/usbharu/markdown/LexerTest.kt @@ -608,4 +608,156 @@ class LexerTest { ), actual ) } + + @Test + fun urlとタイトル() { + val lexer = Lexer() + + val actual = lexer.lex("[alt](https://example.com \"example\")") + + println(actual) + + assertContentEquals( + listOf( + SquareBracketStart, + Text("alt"), + SquareBracketEnd, + ParenthesesStart, + Url("https://example.com"), + UrlTitle("example"), + ParenthesesEnd + ), actual + ) + } + + @Test + fun urlとタイトル全角() { + val lexer = Lexer() + + val actual = lexer.lex("[alt](https://example.com \"example)") + + println(actual) + + assertContentEquals( + listOf( + SquareBracketStart, + Text("alt"), + SquareBracketEnd, + ParenthesesStart, + Url("https://example.com"), + UrlTitle("example"), + ParenthesesEnd + ), actual + ) + } + + @Test + fun インラインコードブロック() { + val lexer = Lexer() + + val actual = lexer.lex("`code`") + + println(actual) + + assertContentEquals( + listOf( + InlineCodeBlock("code"), + ), actual + ) + } + + @Test + fun インラインコードブロック2() { + val lexer = Lexer() + + val actual = lexer.lex("aiueo`code`abcd") + + println(actual) + + assertContentEquals( + listOf( + Text("aiueo"), + InlineCodeBlock("code"), + Text("abcd"), + ), actual + ) + } + + + @Test + fun コードブロック() { + val lexer = Lexer() + + val actual = lexer.lex( + """``` + |code + |``` + """.trimMargin() + ) + + println(actual) + + assertContentEquals( + listOf( + CodeBlock("code"), + ), actual + ) + } + + @Test + fun 言語指定付きコードブロック() { + val lexer = Lexer() + + val actual = lexer.lex( + """```hoge + |code + |``` + """.trimMargin() + ) + + println(actual) + + assertContentEquals( + listOf( + CodeBlockLanguage("hoge", ""), + CodeBlock("code"), + ), actual + ) + } + + @Test + fun ファイル名と言語コードブロック() { + val lexer = Lexer() + + val actual = lexer.lex( + """```hoge:fuga + |code + |``` + """.trimMargin() + ) + + println(actual) + + assertContentEquals( + listOf( + CodeBlockLanguage("hoge", "fuga"), + CodeBlock("code"), + ), actual + ) + } + + @Test + fun コードブロックかと思ったら違った() { + val lexer = Lexer() + + val actual = lexer.lex("````aiueo") + + println(actual) + + assertContentEquals( + listOf( + Text("```"), Text("`aiueo") + ), actual + ) + } } \ No newline at end of file