diff --git a/.travis.yml b/.travis.yml index 6caeabf..b85cad5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,3 +2,5 @@ sudo: false script: - sbt test +jdk: + - oraclejdk8 diff --git a/project/build.scala b/project/build.scala index 6249a92..f8e111e 100644 --- a/project/build.scala +++ b/project/build.scala @@ -38,7 +38,8 @@ scalaVersion := ScalaVersion, resolvers ++= Seq( Classpaths.typesafeReleases, - "amateras-repo" at "http://amateras.sourceforge.jp/mvn/" + "amateras-repo" at "http://amateras.sourceforge.jp/mvn/", + "amateras-snapshot-repo" at "http://amateras.sourceforge.jp/mvn-snapshot/" ), scalacOptions := Seq("-deprecation", "-language:postfixOps"), libraryDependencies ++= Seq( @@ -50,7 +51,7 @@ "org.json4s" %% "json4s-jackson" % "3.2.11", "jp.sf.amateras" %% "scalatra-forms" % "0.1.0", "commons-io" % "commons-io" % "2.4", - "org.pegdown" % "pegdown" % "1.5.0", + "io.github.gitbucket" % "markedj" % "1.0.1", "org.apache.commons" % "commons-compress" % "1.9", "org.apache.commons" % "commons-email" % "1.3.3", "org.apache.httpcomponents" % "httpclient" % "4.3.6", diff --git a/src/main/scala/gitbucket/core/controller/IssuesController.scala b/src/main/scala/gitbucket/core/controller/IssuesController.scala index 8f2bfaf..1bee1bf 100644 --- a/src/main/scala/gitbucket/core/controller/IssuesController.scala +++ b/src/main/scala/gitbucket/core/controller/IssuesController.scala @@ -233,7 +233,7 @@ org.json4s.jackson.Serialization.write( Map("title" -> x.title, "content" -> Markdown.toHtml(x.content getOrElse "No description given.", - repository, false, true, true, isEditable(x.userName, x.repositoryName, x.openedUserName)) + repository, false, true, true, true, isEditable(x.userName, x.repositoryName, x.openedUserName)) )) } } else Unauthorized diff --git a/src/main/scala/gitbucket/core/view/Markdown.scala b/src/main/scala/gitbucket/core/view/Markdown.scala index 3a80b8f..2e126eb 100644 --- a/src/main/scala/gitbucket/core/view/Markdown.scala +++ b/src/main/scala/gitbucket/core/view/Markdown.scala @@ -1,18 +1,14 @@ package gitbucket.core.view import java.text.Normalizer -import java.util.Locale import java.util.regex.Pattern +import java.util.{Optional, Locale} import gitbucket.core.controller.Context -import gitbucket.core.service.{RepositoryService, RequestCache, WikiService} +import gitbucket.core.service.{RepositoryService, RequestCache} import gitbucket.core.util.StringUtil -import org.parboiled.common.StringUtils -import org.pegdown.LinkRenderer.Rendering -import org.pegdown._ -import org.pegdown.ast._ - -import scala.collection.JavaConverters._ +import io.github.gitbucket.markedj._ +import io.github.gitbucket.markedj.Utils._ object Markdown { @@ -24,7 +20,7 @@ * @param enableRefsLink if true then issue reference (e.g. #123) is rendered as link * @param enableAnchor if true then anchor for headline is generated * @param enableTaskList if true then task list syntax is available - * @param hasWritePermission + * @param hasWritePermission true if user has writable to ths given repository * @param pages the list of existing Wiki pages */ def toHtml(markdown: String, @@ -35,7 +31,6 @@ enableTaskList: Boolean = false, hasWritePermission: Boolean = false, pages: List[String] = Nil)(implicit context: Context): String = { - // escape issue id val s = if(enableRefsLink){ markdown.replaceAll("(?<=(\\W|^))#(\\d+)(?=(\\W|$))", "issue:$2") @@ -43,252 +38,145 @@ // escape task list val source = if(enableTaskList){ - GitBucketHtmlSerializer.escapeTaskList(s) + escapeTaskList(s) } else s - val rootNode = new PegDownProcessor( - Extensions.AUTOLINKS | Extensions.WIKILINKS | Extensions.FENCED_CODE_BLOCKS | - Extensions.TABLES | Extensions.HARDWRAPS | Extensions.SUPPRESS_ALL_HTML | Extensions.STRIKETHROUGH - ).parseMarkdown(source.toCharArray) - - new GitBucketHtmlSerializer( - markdown, repository, enableWikiLink, enableRefsLink, enableAnchor, enableTaskList, - hasWritePermission, pages - ).toHtml(rootNode) + val options = new Options() + val renderer = new GitBucketMarkedRenderer(options, repository, enableWikiLink, enableRefsLink, enableAnchor, enableTaskList, hasWritePermission, pages) + Marked.marked(source, options, renderer) } -} -class GitBucketLinkRender( - context: Context, - repository: RepositoryService.RepositoryInfo, - enableWikiLink: Boolean, - pages: List[String]) extends LinkRenderer with WikiService { + /** + * Extends markedj Renderer for GitBucket + */ + class GitBucketMarkedRenderer(options: Options, repository: RepositoryService.RepositoryInfo, + enableWikiLink: Boolean, enableRefsLink: Boolean, enableAnchor: Boolean, enableTaskList: Boolean, hasWritePermission: Boolean, + pages: List[String]) + (implicit val context: Context) extends Renderer(options) with LinkConverter with RequestCache { - override def render(node: WikiLinkNode): Rendering = { - if(enableWikiLink){ - try { - val text = node.getText - val (label, page) = if(text.contains('|')){ - val i = text.indexOf('|') - (text.substring(0, i), text.substring(i + 1)) + override def heading(text: String, level: Int, raw: String): String = { + val id = generateAnchorName(text) + val out = new StringBuilder() + + out.append("") + + if(enableAnchor){ + out.append("") + out.append("") + } + + out.append(text) + out.append("\n") + out.toString() + } + + override def code(code: String, lang: Optional[String], escaped: Boolean): String = { + "
" +
+        (if(escaped) code else escape(code, true)) + "
" + } + + override def list(body: String, ordered: Boolean): String = { + var listType: String = null + if (ordered) { + listType = "ol" + } + else { + listType = "ul" + } + if(body.contains("""class="task-list-item-checkbox"""")){ + return "<" + listType + " class=\"task-list\">\n" + body + "\n" + } else { + return "<" + listType + ">\n" + body + "\n" + } + } + + override def listitem(text: String): String = { + if(text.contains("""class="task-list-item-checkbox" """)){ + return "
  • " + text + "
  • \n" + } else { + return "
  • " + text + "
  • \n" + } + } + + override def text(text: String): String = { + // convert commit id and username to link. + val t1 = if(enableRefsLink) convertRefsLinks(text, repository, "issue:") else text + + // convert task list to checkbox. + val t2 = if(enableTaskList) convertCheckBox(t1, hasWritePermission) else t1 + + t2 + } + + override def link(href: String, title: Optional[String], text: String): String = { + super.link(fixUrl(href, true), title, text) + } + + override def image(href: String, title: Optional[String], text: String): String = { + super.image(fixUrl(href, true), title, text) + } + + override def nolink(text: String): String = { + if(enableWikiLink && text.startsWith("[[") && text.endsWith("]]")){ + val link = text.replaceAll("(^\\[\\[|\\]\\]$)", "") + + val (label, page) = if(link.contains('|')){ + val i = link.indexOf('|') + (link.substring(0, i), link.substring(i + 1)) } else { - (text, text) + (link, link) } val url = repository.httpUrl.replaceFirst("/git/", "/").stripSuffix(".git") + "/wiki/" + StringUtil.urlEncode(page) - if(pages.contains(page)){ - new Rendering(url, label) + "" + escape(label) + "" } else { - new Rendering(url, label).withAttribute("class", "absent") + "" + escape(label) + "" } - } catch { - case e: java.io.UnsupportedEncodingException => throw new IllegalStateException - } - } else { - super.render(node) - } - } -} - -class GitBucketVerbatimSerializer extends VerbatimSerializer { - def serialize(node: VerbatimNode, printer: Printer): Unit = { - printer.println.print("") - var text: String = node.getText - while (text.charAt(0) == '\n') { - printer.print("
    ") - text = text.substring(1) - } - printer.printEncoded(text) - printer.print("") - } -} - -class GitBucketHtmlSerializer( - markdown: String, - repository: RepositoryService.RepositoryInfo, - enableWikiLink: Boolean, - enableRefsLink: Boolean, - enableAnchor: Boolean, - enableTaskList: Boolean, - hasWritePermission: Boolean, - pages: List[String] - )(implicit val context: Context) extends ToHtmlSerializer( - new GitBucketLinkRender(context, repository, enableWikiLink, pages), - Map[String, VerbatimSerializer](VerbatimSerializer.DEFAULT -> new GitBucketVerbatimSerializer).asJava - ) with LinkConverter with RequestCache { - - override protected def printImageTag(rendering: LinkRenderer.Rendering): Unit = { - printer.print("") - .print("\"").printEncoded(rendering.text).print("\"/") - } - - override protected def printLink(rendering: LinkRenderer.Rendering): Unit = { - printer.print('<').print('a') - printAttribute("href", fixUrl(rendering.href)) - for (attr <- rendering.attributes.asScala) { - printAttribute(attr.name, attr.value) - } - printer.print('>').print(rendering.text).print("") - } - - private def fixUrl(url: String, isImage: Boolean = false): String = { - if(url.startsWith("http://") || url.startsWith("https://") || url.startsWith("/")){ - url - } else if(url.startsWith("#")){ - ("#" + GitBucketHtmlSerializer.generateAnchorName(url.substring(1))) - } else if(!enableWikiLink){ - if(context.currentPath.contains("/blob/")){ - url + (if(isImage) "?raw=true" else "") - } else if(context.currentPath.contains("/tree/")){ - val paths = context.currentPath.split("/") - val branch = if(paths.length > 3) paths.drop(4).mkString("/") else repository.repository.defaultBranch - repository.httpUrl.replaceFirst("/git/", "/").stripSuffix(".git") + "/blob/" + branch + "/" + url + (if(isImage) "?raw=true" else "") } else { - val paths = context.currentPath.split("/") - val branch = if(paths.length > 3) paths.last else repository.repository.defaultBranch - repository.httpUrl.replaceFirst("/git/", "/").stripSuffix(".git") + "/blob/" + branch + "/" + url + (if(isImage) "?raw=true" else "") + escape(text) } - } else { - repository.httpUrl.replaceFirst("/git/", "/").stripSuffix(".git") + "/wiki/_blob/" + url - } - } - - private def printAttribute(name: String, value: String): Unit = { - printer.print(' ').print(name).print('=').print('"').print(value).print('"') - } - - private def printHeaderTag(node: HeaderNode): Unit = { - val tag = s"h${node.getLevel}" - val child = node.getChildren.asScala.headOption - val anchorName = child match { - case Some(x: AnchorLinkNode) => x.getName - case Some(x: TextNode) => x.getText - case _ => GitBucketHtmlSerializer.generateAnchorName(extractText(node)) // TODO } - printer.print(s"""<$tag class="markdown-head">""") - if(enableAnchor){ - printer.print(s"""""") - printer.print(s"""""") + private def fixUrl(url: String, isImage: Boolean = false): String = { + if(url.startsWith("http://") || url.startsWith("https://") || url.startsWith("/")){ + url + } else if(url.startsWith("#")){ + ("#" + generateAnchorName(url.substring(1))) + } else if(!enableWikiLink){ + if(context.currentPath.contains("/blob/")){ + url + (if(isImage) "?raw=true" else "") + } else if(context.currentPath.contains("/tree/")){ + val paths = context.currentPath.split("/") + val branch = if(paths.length > 3) paths.drop(4).mkString("/") else repository.repository.defaultBranch + repository.httpUrl.replaceFirst("/git/", "/").stripSuffix(".git") + "/blob/" + branch + "/" + url + (if(isImage) "?raw=true" else "") + } else { + val paths = context.currentPath.split("/") + val branch = if(paths.length > 3) paths.last else repository.repository.defaultBranch + repository.httpUrl.replaceFirst("/git/", "/").stripSuffix(".git") + "/blob/" + branch + "/" + url + (if(isImage) "?raw=true" else "") + } + } else { + repository.httpUrl.replaceFirst("/git/", "/").stripSuffix(".git") + "/wiki/_blob/" + url + } } - child match { - case Some(x: AnchorLinkNode) => printer.print(x.getText) - case _ => visitChildren(node) - } - printer.print(s"") - } - private def extractText(node: Node): String = { - val sb = new StringBuilder() - node.getChildren.asScala.map { - case x: TextNode => sb.append(x.getText) - case x: Node => sb.append(extractText(x)) - } - sb.toString() - } - - override def visit(node: HeaderNode): Unit = { - printHeaderTag(node) - } - - override def visit(node: TextNode): Unit = { - // convert commit id and username to link. - val t = if(enableRefsLink) convertRefsLinks(node.getText, repository, "issue:") else node.getText - - // convert task list to checkbox. - val text = if(enableTaskList) GitBucketHtmlSerializer.convertCheckBox(t, hasWritePermission) else t - - if (abbreviations.isEmpty) { - printer.print(text) - } else { - printWithAbbreviations(text) - } - } - - override def visit(node: VerbatimNode) { - val printer = new Printer() - val serializer = verbatimSerializers.get(VerbatimSerializer.DEFAULT) - serializer.serialize(node, printer) - val html = printer.getString - - // convert commit id and username to link. - val t = if(enableRefsLink) convertRefsLinks(html, repository, "issue:", escapeHtml = false) else html - - this.printer.print(t) - } - - override def visit(node: BulletListNode): Unit = { - if (printChildrenToString(node).contains("""class="task-list-item-checkbox" """)) { - printer.println().print("""") - } else { - printIndentedTag(node, "ul") - } - } - - override def visit(node: ListItemNode): Unit = { - if (printChildrenToString(node).contains("""class="task-list-item-checkbox" """)) { - printer.println() - printer.print("""
  • """) - visitChildren(node) - printer.print("
  • ") - } else { - printer.println() - printTag(node, "li") - } - } - - override def visit(node: ExpLinkNode) { - printLink(linkRenderer.render(node, printLinkChildrenToString(node))) - } - - def printLinkChildrenToString(node: SuperNode) = { - val priorPrinter = printer - printer = new Printer() - visitLinkChildren(node) - val result = printer.getString() - printer = priorPrinter - result - } - - def visitLinkChildren(node: SuperNode) { - import scala.collection.JavaConversions._ - node.getChildren.foreach(child => child match { - case node: ExpImageNode => visitLinkChild(node) - case node: SuperNode => visitLinkChildren(node) - case _ => child.accept(this) - }) - } - - def visitLinkChild(node: ExpImageNode) { - printer.print("\"").printEncoded(printChildrenToString(node)).print("\"/") - } -} - -object GitBucketHtmlSerializer { - - private val Whitespace = "[\\s]".r - - def generateAnchorName(text: String): String = { - val noWhitespace = Whitespace.replaceAllIn(text, "-") - val normalized = Normalizer.normalize(noWhitespace, Normalizer.Form.NFD) - val noSpecialChars = StringUtil.urlEncode(normalized) - noSpecialChars.toLowerCase(Locale.ENGLISH) } def escapeTaskList(text: String): String = { Pattern.compile("""^( *)- \[([x| ])\] """, Pattern.MULTILINE).matcher(text).replaceAll("$1* task:$2: ") } + def generateAnchorName(text: String): String = { + val normalized = Normalizer.normalize(text.replaceAll("<.*>", "").replaceAll("[\\s]", "-"), Normalizer.Form.NFD) + val encoded = StringUtil.urlEncode(normalized) + encoded.toLowerCase(Locale.ENGLISH) + } + def convertCheckBox(text: String, hasWritePermission: Boolean): String = { val disabled = if (hasWritePermission) "" else "disabled" text.replaceAll("task:x:", """") - .replaceAll("task: :", """") + .replaceAll("task: :", """") } + } + diff --git a/src/test/scala/gitbucket/core/view/GitBucketHtmlSerializerSpec.scala b/src/test/scala/gitbucket/core/view/GitBucketHtmlSerializerSpec.scala deleted file mode 100644 index 82fb3d9..0000000 --- a/src/test/scala/gitbucket/core/view/GitBucketHtmlSerializerSpec.scala +++ /dev/null @@ -1,93 +0,0 @@ -package gitbucket.core.view - -import org.specs2.mutable._ - -class GitBucketHtmlSerializerSpec extends Specification { - - import GitBucketHtmlSerializer._ - - "generateAnchorName" should { - "convert whitespace characters to hyphens" in { - val before = "foo bar baz" - val after = generateAnchorName(before) - after mustEqual "foo-bar-baz" - } - - "normalize characters with diacritics" in { - val before = "Dónde estará mi vida" - val after = generateAnchorName(before) - after mustEqual "do%cc%81nde-estara%cc%81-mi-vida" - } - - "omit special characters" in { - val before = "foo!bar@baz>9000" - val after = generateAnchorName(before) - after mustEqual "foo%21bar%40baz%3e9000" - } - } - - "escapeTaskList" should { - "convert '- [ ] ' to '* task: :'" in { - val before = "- [ ] aaaa" - val after = escapeTaskList(before) - after mustEqual "* task: : aaaa" - } - - "convert ' - [ ] ' to ' * task: :'" in { - val before = " - [ ] aaaa" - val after = escapeTaskList(before) - after mustEqual " * task: : aaaa" - } - - "convert only first '- [ ] '" in { - val before = " - [ ] aaaa - [ ] bbb" - val after = escapeTaskList(before) - after mustEqual " * task: : aaaa - [ ] bbb" - } - - "convert '- [x] ' to '* task:x:'" in { - val before = " - [x] aaaa" - val after = escapeTaskList(before) - after mustEqual " * task:x: aaaa" - } - - "convert multi lines" in { - val before = """ -tasks -- [x] aaaa -- [ ] bbb -""" - val after = escapeTaskList(before) - after mustEqual """ -tasks -* task:x: aaaa -* task: : bbb -""" - } - - "no convert if inserted before '- [ ] '" in { - val before = " a - [ ] aaaa" - val after = escapeTaskList(before) - after mustEqual " a - [ ] aaaa" - } - - "no convert '- [] '" in { - val before = " - [] aaaa" - val after = escapeTaskList(before) - after mustEqual " - [] aaaa" - } - - "no convert '- [ ]a'" in { - val before = " - [ ]a aaaa" - val after = escapeTaskList(before) - after mustEqual " - [ ]a aaaa" - } - - "no convert '-[ ] '" in { - val before = " -[ ] aaaa" - val after = escapeTaskList(before) - after mustEqual " -[ ] aaaa" - } - } -} - diff --git a/src/test/scala/gitbucket/core/view/MarkdownSpec.scala b/src/test/scala/gitbucket/core/view/MarkdownSpec.scala new file mode 100644 index 0000000..a2ec2e9 --- /dev/null +++ b/src/test/scala/gitbucket/core/view/MarkdownSpec.scala @@ -0,0 +1,93 @@ +package gitbucket.core.view + +import org.specs2.mutable._ + +class MarkdownSpec extends Specification { + + import Markdown._ + + "generateAnchorName" should { + "convert whitespace characters to hyphens" in { + val before = "foo bar baz" + val after = generateAnchorName(before) + after mustEqual "foo-bar-baz" + } + + "normalize characters with diacritics" in { + val before = "Dónde estará mi vida" + val after = generateAnchorName(before) + after mustEqual "do%cc%81nde-estara%cc%81-mi-vida" + } + + "omit special characters" in { + val before = "foo!bar@baz>9000" + val after = generateAnchorName(before) + after mustEqual "foo%21bar%40baz%3e9000" + } + } + + "escapeTaskList" should { + "convert '- [ ] ' to '* task: :'" in { + val before = "- [ ] aaaa" + val after = escapeTaskList(before) + after mustEqual "* task: : aaaa" + } + + "convert ' - [ ] ' to ' * task: :'" in { + val before = " - [ ] aaaa" + val after = escapeTaskList(before) + after mustEqual " * task: : aaaa" + } + + "convert only first '- [ ] '" in { + val before = " - [ ] aaaa - [ ] bbb" + val after = escapeTaskList(before) + after mustEqual " * task: : aaaa - [ ] bbb" + } + + "convert '- [x] ' to '* task:x:'" in { + val before = " - [x] aaaa" + val after = escapeTaskList(before) + after mustEqual " * task:x: aaaa" + } + + "convert multi lines" in { + val before = """ +tasks +- [x] aaaa +- [ ] bbb +""" + val after = escapeTaskList(before) + after mustEqual """ +tasks +* task:x: aaaa +* task: : bbb +""" + } + + "no convert if inserted before '- [ ] '" in { + val before = " a - [ ] aaaa" + val after = escapeTaskList(before) + after mustEqual " a - [ ] aaaa" + } + + "no convert '- [] '" in { + val before = " - [] aaaa" + val after = escapeTaskList(before) + after mustEqual " - [] aaaa" + } + + "no convert '- [ ]a'" in { + val before = " - [ ]a aaaa" + val after = escapeTaskList(before) + after mustEqual " - [ ]a aaaa" + } + + "no convert '-[ ] '" in { + val before = " -[ ] aaaa" + val after = escapeTaskList(before) + after mustEqual " -[ ] aaaa" + } + } +} +