Add external markup render support (#2570)

* add external markup render support

* bug fixed

* refacotr codes and fix wrong error log

* fix comments and add check to prevent leaks

* add check for config file and improve the example

* check file close error

* use ioutil.TempFile instead uuid

* correct Render -> Parser

* improve warning when incorrect markup setting

* fix typos

Author: lunny

cmd/web.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/markup/external"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/routers"
 	"code.gitea.io/gitea/routers/routes"
@@ -59,6 +60,8 @@ func runWeb(ctx *cli.Context) error {
 
 	routers.GlobalInit()
 
+	external.RegisterParsers()
+
 	m := routes.NewMacaron()
 	routes.RegisterRoutes(m)
 

conf/app.ini

@@ -573,3 +573,12 @@ SHOW_FOOTER_BRANDING = false
 SHOW_FOOTER_VERSION = true
 ; Show time of template execution in the footer
 SHOW_FOOTER_TEMPLATE_LOAD_TIME = true
+
+[markup.asciidoc]
+ENABLED = false
+; List of file extensions that should be rendered by an external command
+FILE_EXTENSIONS = .adoc,.asciidoc
+; External command to render all matching extensions
+RENDER_COMMAND = "asciidoc --out-file=- -"
+; Input is not a standard input but a file
+IS_INPUT_FILE = false

modules/markup/external/external.go

@@ -0,0 +1,88 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package external
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strings"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/markup"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+// RegisterParsers registers all supported third part parsers according settings
+func RegisterParsers() {
+	for _, parser := range setting.ExternalMarkupParsers {
+		if parser.Enabled && parser.Command != "" && len(parser.FileExtensions) > 0 {
+			markup.RegisterParser(&Parser{parser})
+		}
+	}
+}
+
+// Parser implements markup.Parser for external tools
+type Parser struct {
+	setting.MarkupParser
+}
+
+// Name returns the external tool name
+func (p *Parser) Name() string {
+	return p.MarkupName
+}
+
+// Extensions returns the supported extensions of the tool
+func (p *Parser) Extensions() []string {
+	return p.FileExtensions
+}
+
+// Render renders the data of the document to HTML via the external tool.
+func (p *Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
+	var (
+		bs       []byte
+		buf      = bytes.NewBuffer(bs)
+		rd       = bytes.NewReader(rawBytes)
+		commands = strings.Fields(p.Command)
+		args     = commands[1:]
+	)
+
+	if p.IsInputFile {
+		// write to temp file
+		f, err := ioutil.TempFile("", "gitea_input")
+		if err != nil {
+			log.Error(4, "%s create temp file when rendering %s failed: %v", p.Name(), p.Command, err)
+			return []byte("")
+		}
+		defer os.Remove(f.Name())
+
+		_, err = io.Copy(f, rd)
+		if err != nil {
+			f.Close()
+			log.Error(4, "%s write data to temp file when rendering %s failed: %v", p.Name(), p.Command, err)
+			return []byte("")
+		}
+
+		err = f.Close()
+		if err != nil {
+			log.Error(4, "%s close temp file when rendering %s failed: %v", p.Name(), p.Command, err)
+			return []byte("")
+		}
+		args = append(args, f.Name())
+	}
+
+	cmd := exec.Command(commands[0], args...)
+	if !p.IsInputFile {
+		cmd.Stdin = rd
+	}
+	cmd.Stdout = buf
+	if err := cmd.Run(); err != nil {
+		log.Error(4, "%s render run command %s %v failed: %v", p.Name(), commands[0], args, err)
+		return []byte("")
+	}
+	return buf.Bytes()
+}

modules/setting/setting.go

@@ -60,6 +60,15 @@ const (
 	LandingPageExplore LandingPage = "/explore"
 )
 
+// MarkupParser defines the external parser configured in ini
+type MarkupParser struct {
+	Enabled        bool
+	MarkupName     string
+	Command        string
+	FileExtensions []string
+	IsInputFile    bool
+}
+
 // settings
 var (
 	// AppVer settings
@@ -515,6 +524,8 @@ var (
 	HasRobotsTxt      bool
 	InternalToken     string // internal access token
 	IterateBufferSize int
+
+	ExternalMarkupParsers []MarkupParser
 )
 
 // DateLang transforms standard language locale name to corresponding value in datetime plugin.
@@ -1073,6 +1084,44 @@ func NewContext() {
 	UI.ShowUserEmail = Cfg.Section("ui").Key("SHOW_USER_EMAIL").MustBool(true)
 
 	HasRobotsTxt = com.IsFile(path.Join(CustomPath, "robots.txt"))
+
+	extensionReg := regexp.MustCompile(`\.\w`)
+	for _, sec := range Cfg.Section("markup").ChildSections() {
+		name := strings.TrimLeft(sec.Name(), "markup.")
+		if name == "" {
+			log.Warn("name is empty, markup " + sec.Name() + "ignored")
+			continue
+		}
+
+		extensions := sec.Key("FILE_EXTENSIONS").Strings(",")
+		var exts = make([]string, 0, len(extensions))
+		for _, extension := range extensions {
+			if !extensionReg.MatchString(extension) {
+				log.Warn(sec.Name() + " file extension " + extension + " is invalid. Extension ignored")
+			} else {
+				exts = append(exts, extension)
+			}
+		}
+
+		if len(exts) == 0 {
+			log.Warn(sec.Name() + " file extension is empty, markup " + name + " ignored")
+			continue
+		}
+
+		command := sec.Key("RENDER_COMMAND").MustString("")
+		if command == "" {
+			log.Warn(" RENDER_COMMAND is empty, markup " + name + " ignored")
+			continue
+		}
+
+		ExternalMarkupParsers = append(ExternalMarkupParsers, MarkupParser{
+			Enabled:        sec.Key("ENABLED").MustBool(false),
+			MarkupName:     name,
+			FileExtensions: exts,
+			Command:        command,
+			IsInputFile:    sec.Key("IS_INPUT_FILE").MustBool(false),
+		})
+	}
 }
 
 // Service settings
@@ -1133,7 +1182,6 @@ func newService() {
 			Service.OpenIDBlacklist[i] = regexp.MustCompilePOSIX(p)
 		}
 	}
-
 }
 
 var logLevels = map[string]string{