基于 Go 实现一个 Markdown 解析器

Tag: Go Posted on 2020-12-06 21:59:01 Edited on 2020-12-06 22:11:15 Views: 185

概述

仓库链接:https://github.com/songquanpeng/md2html

简单说一下流程:

  1. 把 Markdown 字符串分解为 Token 流。
  2. 从 Token 流中构建出抽象语法树(AST)。
  3. 遍历抽象语法树,生成 HTML 文档。

词法分析部分

核心代码:

func nextToken() (textToken, otherToken Token) {
    textToken.Type = TextToken
    for {
        if pos >= len(input) {
            otherToken.Type = EofToken
            return
        }
        c := input[pos]
        if len(textToken.Value) == 0 && (lastTokenType == NewlineToken || lastTokenType == TabToken) {
            switch c {
            case '#':
                n := countSymbol(c)
                otherToken.Type = TitleToken
                otherToken.Value = append(otherToken.Value, rune(n))
                pos += n
                if input[pos] == ' ' {
                    pos++
                }
                return
            case '\t':
                otherToken.Type = TabToken
                pos++
                return
            case '\n':
                otherToken.Type = NewlineToken
                pos++
                return
            case '-':
                fallthrough
            case '+':
                fallthrough
            case '*':
                if isSpaceBehind() {
                    otherToken.Type = UnorderedListToken
                    pos += 2
                    yes, completed := isTaskSymbol()
                    if yes {
                        pos += 2
                        if isSpaceBehind() {
                            pos += 2
                            if completed {
                                otherToken.Type = CompletedTaskToken
                            } else {
                                otherToken.Type = UncompletedTaskToken
                            }
                            return
                        }
                        pos -= 2
                    }
                    return
                } else { // Consider if this is a dividing line
                    if nextIsSameTo(c) {
                        pos++
                        if nextIsSameTo(c) {
                            pos += 2
                            otherToken.Type = DividingLineToken
                            return
                        }
                        pos--
                    }
                }
            case '>':
                if isSpaceBehind() {
                    otherToken.Type = QuoteToken
                    pos += 2
                    return
                }
            case '`':
                if nextIsSameTo(c) {
                    pos++
                    if nextIsSameTo(c) {
                        pos += 2
                        otherToken.Type = CodeBlockToken
                        start, end := getCodeBlockStartEnd()
                        otherToken.Value = input[start:end]
                        pos = end + 3
                        return
                    }
                    pos--
                }
            case '\r':
                fallthrough
            case ' ':
                n := countSymbol(c)
                if n >= 2 {
                    pos += n
                    otherToken.Type = TabToken
                    return
                } else {
                    pos++
                }
            }
            if isNumDotSpace() {
                otherToken.Type = OrderedListToken
                return
            }
        }
        // Update c because pos maybe updated due to black symbol.
        c = input[pos]

        // Now we have to return the text token before the below token.
        switch c {
        case '*':
            if nextIsSameTo(c) {
                pos += 2
                otherToken.Type = DoubleStarToken
                otherToken.Value = []rune("**")
            } else {
                pos += 1
                otherToken.Type = SingleStarToken
                otherToken.Value = []rune("*")
            }
            return
        case '_':
            if nextIsSameTo(c) {
                pos += 2
                otherToken.Type = DoubleUnderscoreToken
                otherToken.Value = []rune("__")
            } else {
                pos += 1
                otherToken.Type = SingleUnderscoreToken
                otherToken.Value = []rune("_")
            }
            return
        case '~':
            if nextIsSameTo(c) {
                pos += 2
                otherToken.Type = DoubleTildeToken
                otherToken.Value = []rune("~~")
                return
            }
        case '`':
            otherToken.Type = SingleBacktickToken
            otherToken.Value = []rune("`")
            pos++
            return
        case '!':
            if nextIsSameTo('[') {
                pos += 2
                otherToken.Type = ImageHeadToken
                return
            }
        case '[':
            pos++
            otherToken.Type = LinkHeadToken
            return
        case ']':
            if nextIsSameTo('(') {
                pos += 2
                for i := pos; i < len(input) && input[i] != '\n'; i++ {
                    if input[i] == ')' {
                        otherToken.Type = LinkBodyToken
                        otherToken.Value = input[pos:i]
                        pos = i + 1
                        return
                    }
                }
                pos -= 2
            }
        case '\n':
            otherToken.Type = NewlineToken
            pos++
            return
        case '\t':
            otherToken.Type = TabToken
            pos++
            return
        }
        pos++
        if c != '\r' {
            textToken.Value = append(textToken.Value, c)
        }
    }
}

语法分析部分

核心代码:

func parseSectionList() (root *Node) {
    node := Node{}
    root = &node
    for {
        token := getToken()
        restoreToken()
        current := &Node{}
        switch token.Type {
        case lexer.TitleToken:
            current = parseTitle()
        case lexer.DividingLineToken:
            current = parseDividingLine()
        case lexer.CodeBlockToken:
            current = parseCodeBlock()
        case lexer.UncompletedTaskToken:
            fallthrough
        case lexer.CompletedTaskToken:
            fallthrough
        case lexer.UnorderedListToken:
            fallthrough
        case lexer.OrderedListToken:
            current = parseList()
        case lexer.QuoteToken:
            current = parseQuote()
        case lexer.NewlineToken:
            _ = getToken()
            tabCounter = 0
            continue
        case lexer.TabToken:
            tabCounter++
            _ = getToken()
            continue
        case lexer.EofToken:
            return
        default:
            current = parseContent(false)
        }
        root.Children = append(root.Children, current)
    }
}

HTML 代码生成部分

这部分最好玩也最简单,主要代码:

func Convert(markdown string, fullPage bool) (html string) {
    ast := parser.Parse(markdown)
    if os.Getenv("MODE") == "debug" {
        parser.PrintAST(ast)
    }
    html = processArticleNode(ast)
    if fullPage {
        html = fmt.Sprintf(HtmlTemplate, Style, html)
    }
    return html
}

func processArticleNode(node *parser.Node) (html string) {
    for _, child := range node.Children {
        switch child.Type {
        case parser.TitleNode:
            html += processTitleNode(child)
        case parser.DividingLineNode:
            html += processDividingLineNode(child)
        case parser.ContentNode:
            content := processContentNode(child)
            html += fmt.Sprintf("<div>%s</div>\n", content)
        case parser.ListNode:
            html += processListNode(child)
        case parser.QuoteNode:
            html += processQuoteNode(child)
        case parser.CodeBlockNode:
            html += processCodeBlockNode(child)
        }
    }
    html = fmt.Sprintf("<div class='article'>\n%s\n</div>", html)
    return
}

func processTitleNode(node *parser.Node) (html string) {
    content := processContentNode(node.Children[0])
    level := int(node.Value[0])
    html = fmt.Sprintf("<h%d>%s</h%d>\n", level, content, level)
    return
}

未经允许,禁止转载,本文源站链接:https://iamazing.cn/