diff --git a/src/main/scala/gitbucket/core/controller/ControllerBase.scala b/src/main/scala/gitbucket/core/controller/ControllerBase.scala index eeedd1c..5db473f 100644 --- a/src/main/scala/gitbucket/core/controller/ControllerBase.scala +++ b/src/main/scala/gitbucket/core/controller/ControllerBase.scala @@ -57,7 +57,7 @@ // Redirect to dashboard httpResponse.sendRedirect(baseUrl + "/") } - } else if(path.startsWith("/git/")){ + } else if(path.startsWith("/git/") || path.startsWith("/git-lfs/")){ // Git repository chain.doFilter(request, response) } else { diff --git a/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala b/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala index a6a460a..3d59d71 100644 --- a/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala +++ b/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala @@ -1,6 +1,7 @@ package gitbucket.core.controller -import javax.servlet.http.{HttpServletResponse, HttpServletRequest} +import java.io.FileInputStream +import javax.servlet.http.{HttpServletRequest, HttpServletResponse} import gitbucket.core.plugin.PluginRegistry import gitbucket.core.repo.html @@ -16,9 +17,8 @@ import gitbucket.core.service.WebHookService._ import gitbucket.core.view import gitbucket.core.view.helpers - import io.github.gitbucket.scalatra.forms._ -import org.apache.commons.io.FileUtils +import org.apache.commons.io.{FileUtils, IOUtils} import org.eclipse.jgit.api.{ArchiveCommand, Git} import org.eclipse.jgit.archive.{TgzFormat, ZipFormat} import org.eclipse.jgit.dircache.DirCache @@ -255,13 +255,9 @@ val (id, path) = repository.splitPath(multiParams("splat").head) using(Git.open(getRepositoryDir(repository.owner, repository.name))){ git => val revCommit = JGitUtil.getRevCommitFromId(git, git.getRepository.resolve(id)) - getPathObjectId(git, path, revCommit).flatMap { objectId => - JGitUtil.getObjectLoaderFromId(git, objectId){ loader => - contentType = FileUtil.getMimeType(path) - response.setContentLength(loader.getSize.toInt) - loader.copyTo(response.outputStream) - () - } + + getPathObjectId(git, path, revCommit).map { objectId => + responseRawFile(git, objectId, path, repository) } getOrElse NotFound() } }) @@ -277,23 +273,62 @@ getPathObjectId(git, path, revCommit).map { objectId => if(raw){ // Download (This route is left for backword compatibility) - JGitUtil.getObjectLoaderFromId(git, objectId){ loader => - contentType = FileUtil.getMimeType(path) - response.setContentLength(loader.getSize.toInt) - loader.copyTo(response.outputStream) - () - } getOrElse NotFound() + responseRawFile(git, objectId, path, repository) } else { html.blob(id, repository, path.split("/").toList, JGitUtil.getContentInfo(git, path, objectId), new JGitUtil.CommitInfo(JGitUtil.getLastModifiedCommit(git, revCommit, path)), hasDeveloperRole(repository.owner, repository.name, context.loginAccount), - request.paths(2) == "blame") + request.paths(2) == "blame", + isLfsFile(git, objectId)) } } getOrElse NotFound() } }) + private def isLfsFile(git: Git, objectId: ObjectId): Boolean = { + JGitUtil.getObjectLoaderFromId(git, objectId){ loader => + if(loader.isLarge){ + false + } else { + new String(loader.getCachedBytes, "UTF-8").startsWith("version https://git-lfs.github.com/spec/v1") + } + }.getOrElse(false) + } + + private def responseRawFile(git: Git, objectId: ObjectId, path: String, + repository: RepositoryService.RepositoryInfo): Unit = { + JGitUtil.getObjectLoaderFromId(git, objectId){ loader => + contentType = FileUtil.getMimeType(path) + + if(loader.isLarge){ + response.setContentLength(loader.getSize.toInt) + loader.copyTo(response.outputStream) + } else { + val bytes = loader.getCachedBytes + val text = new String(bytes, "UTF-8") + + if(text.startsWith("version https://git-lfs.github.com/spec/v1")){ + // LFS objects + val attrs = text.split("\n").map { line => + val dim = line.split(" ") + dim(0) -> dim(1) + }.toMap + + response.setContentLength(attrs("size").toInt) + val oid = attrs("oid").split(":")(1) + + using(new FileInputStream(FileUtil.getLfsFilePath(repository.owner, repository.name, oid))){ in => + IOUtils.copy(in, response.getOutputStream) + } + } else { + response.setContentLength(loader.getSize.toInt) + response.getOutputStream.write(bytes) + } + } + } + } + get("/:owner/:repository/blame/*"){ blobRoute.action() } diff --git a/src/main/scala/gitbucket/core/service/AccessTokenService.scala b/src/main/scala/gitbucket/core/service/AccessTokenService.scala index 0a8109d..a2345be 100644 --- a/src/main/scala/gitbucket/core/service/AccessTokenService.scala +++ b/src/main/scala/gitbucket/core/service/AccessTokenService.scala @@ -20,7 +20,7 @@ def tokenToHash(token: String): String = StringUtil.sha1(token) /** - * @retuen (TokenId, Token) + * @return (TokenId, Token) */ def generateAccessToken(userName: String, note: String)(implicit s: Session): (Int, String) = { var token: String = null diff --git a/src/main/scala/gitbucket/core/service/SystemSettingsService.scala b/src/main/scala/gitbucket/core/service/SystemSettingsService.scala index c2aa28d..123c121 100644 --- a/src/main/scala/gitbucket/core/service/SystemSettingsService.scala +++ b/src/main/scala/gitbucket/core/service/SystemSettingsService.scala @@ -176,6 +176,9 @@ port:Int, genericUser:String) + case class Lfs( + serverUrl: Option[String]) + val DefaultSshPort = 29418 val DefaultSmtpPort = 25 val DefaultLdapPort = 389 diff --git a/src/main/scala/gitbucket/core/servlet/GitLfsTransferServlet.scala b/src/main/scala/gitbucket/core/servlet/GitLfsTransferServlet.scala new file mode 100644 index 0000000..acdb2f6 --- /dev/null +++ b/src/main/scala/gitbucket/core/servlet/GitLfsTransferServlet.scala @@ -0,0 +1,83 @@ +package gitbucket.core.servlet + +import java.io.{File, FileInputStream, FileOutputStream} +import java.text.MessageFormat +import javax.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse} + +import gitbucket.core.util.{Directory, FileUtil, StringUtil} +import org.apache.commons.io.{FileUtils, IOUtils} +import org.json4s.jackson.Serialization._ +import org.apache.http.HttpStatus +import gitbucket.core.util.ControlUtil._ + +/** + * Provides GitLFS Transfer API + * https://github.com/git-lfs/git-lfs/blob/master/docs/api/basic-transfers.md + */ +class GitLfsTransferServlet extends HttpServlet { + + private implicit val jsonFormats = gitbucket.core.api.JsonFormat.jsonFormats + private val LongObjectIdLength = 32 + private val LongObjectIdStringLength = LongObjectIdLength * 2 + + override protected def doGet(req: HttpServletRequest, res: HttpServletResponse): Unit = { + for { + (owner, repository, oid) <- getPathInfo(req, res) if checkToken(req, oid) + } yield { + val file = new File(FileUtil.getLfsFilePath(owner, repository, oid)) + if(file.exists()){ + res.setStatus(HttpStatus.SC_OK) + res.setContentType("application/octet-stream") + res.setContentLength(file.length.toInt) + using(new FileInputStream(file), res.getOutputStream){ (in, out) => + IOUtils.copy(in, out) + out.flush() + } + } else { + sendError(res, HttpStatus.SC_NOT_FOUND, + MessageFormat.format("Object ''{0}'' not found", oid)) + } + } + } + + override protected def doPut(req: HttpServletRequest, res: HttpServletResponse): Unit = { + for { + (owner, repository, oid) <- getPathInfo(req, res) if checkToken(req, oid) + } yield { + val file = new File(FileUtil.getLfsFilePath(owner, repository, oid)) + FileUtils.forceMkdir(file.getParentFile) + using(req.getInputStream, new FileOutputStream(file)){ (in, out) => + IOUtils.copy(in, out) + } + res.setStatus(HttpStatus.SC_OK) + } + } + + private def checkToken(req: HttpServletRequest, oid: String): Boolean = { + val token = req.getHeader("Authorization") + if(token != null){ + val Array(expireAt, targetOid) = StringUtil.decodeBlowfish(token).split(" ") + oid == targetOid && expireAt.toLong > System.currentTimeMillis + } else { + false + } + } + + private def getPathInfo(req: HttpServletRequest, res: HttpServletResponse): Option[(String, String, String)] = { + req.getRequestURI.substring(1).split("/") match { + case Array(_, owner, repository, oid) => Some((owner, repository, oid)) + case _ => None + } + } + + private def sendError(res: HttpServletResponse, status: Int, message: String): Unit = { + res.setStatus(status) + using(res.getWriter()){ out => + out.write(write(GitLfs.Error(message))) + out.flush() + } + } + +} + + diff --git a/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala b/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala index e85804e..d7b359a 100644 --- a/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala +++ b/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala @@ -1,6 +1,7 @@ package gitbucket.core.servlet import java.io.File +import java.util.Date import gitbucket.core.api import gitbucket.core.model.{Session, WebHook} @@ -11,16 +12,16 @@ import gitbucket.core.util.ControlUtil._ import gitbucket.core.util.Implicits._ import gitbucket.core.util._ - import org.eclipse.jgit.api.Git import org.eclipse.jgit.http.server.GitServlet import org.eclipse.jgit.lib._ import org.eclipse.jgit.transport._ import org.eclipse.jgit.transport.resolver._ import org.slf4j.LoggerFactory - import javax.servlet.ServletConfig -import javax.servlet.http.{HttpServletResponse, HttpServletRequest} +import javax.servlet.http.{HttpServletRequest, HttpServletResponse} + +import org.json4s.jackson.Serialization._ /** @@ -32,7 +33,8 @@ class GitRepositoryServlet extends GitServlet with SystemSettingsService { private val logger = LoggerFactory.getLogger(classOf[GitRepositoryServlet]) - + private implicit val jsonFormats = gitbucket.core.api.JsonFormat.jsonFormats + override def init(config: ServletConfig): Unit = { setReceivePackFactory(new GitBucketReceivePackFactory()) @@ -45,15 +47,73 @@ override def service(req: HttpServletRequest, res: HttpServletResponse): Unit = { val agent = req.getHeader("USER-AGENT") val index = req.getRequestURI.indexOf(".git") - if(index >= 0 && (agent == null || agent.toLowerCase.indexOf("git/") < 0)){ + if(index >= 0 && (agent == null || agent.toLowerCase.indexOf("git") < 0)){ // redirect for browsers val paths = req.getRequestURI.substring(0, index).split("/") res.sendRedirect(baseUrl(req) + "/" + paths.dropRight(1).last + "/" + paths.last) + + } else if(req.getMethod.toUpperCase == "POST" && req.getRequestURI.endsWith("/info/lfs/objects/batch")){ + serviceGitLfsBatchAPI(req, res) + } else { // response for git client super.service(req, res) } } + + /** + * Provides GitLFS Batch API + * https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md + */ + protected def serviceGitLfsBatchAPI(req: HttpServletRequest, res: HttpServletResponse): Unit = { + val batchRequest = read[GitLfs.BatchRequest](req.getInputStream) + val settings = loadSystemSettings() + + settings.baseUrl match { + case None => { + throw new IllegalStateException("lfs.server_url is not configured.") + } + case Some(baseUrl) => { + req.getRequestURI.substring(1).replace(".git/", "/").split("/") match { + case Array(_, owner, repository, _*) => { + val timeout = System.currentTimeMillis + (60000 * 10) // 10 min. + val batchResponse = batchRequest.operation match { + case "upload" => + GitLfs.BatchUploadResponse("basic", batchRequest.objects.map { requestObject => + GitLfs.BatchResponseObject(requestObject.oid, requestObject.size, true, + GitLfs.Actions( + upload = Some(GitLfs.Action( + href = baseUrl + "/git-lfs/" + owner + "/" + repository + "/" + requestObject.oid, + header = Map("Authorization" -> StringUtil.encodeBlowfish(timeout + " " + requestObject.oid)), + expires_at = new Date(timeout) + )) + ) + ) + }) + case "download" => + GitLfs.BatchUploadResponse("basic", batchRequest.objects.map { requestObject => + GitLfs.BatchResponseObject(requestObject.oid, requestObject.size, true, + GitLfs.Actions( + download = Some(GitLfs.Action( + href = baseUrl + "/git-lfs/" + owner + "/" + repository + "/" + requestObject.oid, + header = Map("Authorization" -> StringUtil.encodeBlowfish(timeout + " " + requestObject.oid)), + expires_at = new Date(timeout) + )) + ) + ) + }) + } + + res.setContentType("application/vnd.git-lfs+json") + using(res.getWriter){ out => + out.print(write(batchResponse)) + out.flush() + } + } + } + } + } + } } class GitBucketRepositoryResolver(parent: FileResolver[HttpServletRequest]) extends RepositoryResolver[HttpServletRequest] { @@ -232,3 +292,45 @@ } } + +object GitLfs { + + case class BatchRequest( + operation: String, + transfers: Seq[String], + objects: Seq[BatchRequestObject] + ) + + case class BatchRequestObject( + oid: String, + size: Long + ) + + case class BatchUploadResponse( + transfer: String, + objects: Seq[BatchResponseObject] + ) + + case class BatchResponseObject( + oid: String, + size: Long, + authenticated: Boolean, + actions: Actions + ) + + case class Actions( + download: Option[Action] = None, + upload: Option[Action] = None + ) + + case class Action( + href: String, + header: Map[String, String] = Map.empty, + expires_at: Date + ) + + case class Error( + message: String + ) + +} \ No newline at end of file diff --git a/src/main/scala/gitbucket/core/servlet/TransactionFilter.scala b/src/main/scala/gitbucket/core/servlet/TransactionFilter.scala index c25a2fe..5ace04b 100644 --- a/src/main/scala/gitbucket/core/servlet/TransactionFilter.scala +++ b/src/main/scala/gitbucket/core/servlet/TransactionFilter.scala @@ -21,8 +21,9 @@ def destroy(): Unit = {} def doFilter(req: ServletRequest, res: ServletResponse, chain: FilterChain): Unit = { - if(req.asInstanceOf[HttpServletRequest].getServletPath().startsWith("/assets/")){ - // assets don't need transaction + val servletPath = req.asInstanceOf[HttpServletRequest].getServletPath() + if(servletPath.startsWith("/assets/") || servletPath.startsWith("/git-lfs")){ + // assets and git-lfs don't need transaction chain.doFilter(req, res) } else { Database() withTransaction { session => diff --git a/src/main/scala/gitbucket/core/util/ControlUtil.scala b/src/main/scala/gitbucket/core/util/ControlUtil.scala index a323c42..74569ac 100644 --- a/src/main/scala/gitbucket/core/util/ControlUtil.scala +++ b/src/main/scala/gitbucket/core/util/ControlUtil.scala @@ -20,6 +20,20 @@ } } + def using[A <% { def close(): Unit }, B <% { def close(): Unit }, C](resource1: A, resource2: B)(f: (A, B) => C): C = + try f(resource1, resource2) finally { + if(resource1 != null){ + ignoring(classOf[Throwable]) { + resource1.close() + } + } + if(resource2 != null){ + ignoring(classOf[Throwable]) { + resource2.close() + } + } + } + def using[T](git: Git)(f: Git => T): T = try f(git) finally git.getRepository.close() diff --git a/src/main/scala/gitbucket/core/util/Directory.scala b/src/main/scala/gitbucket/core/util/Directory.scala index 6e24100..e73bca8 100644 --- a/src/main/scala/gitbucket/core/util/Directory.scala +++ b/src/main/scala/gitbucket/core/util/Directory.scala @@ -1,11 +1,9 @@ package gitbucket.core.util import java.io.File -import ControlUtil._ -import org.apache.commons.io.FileUtils /** - * Provides directories used by GitBucket. + * Provides directory locations used by GitBucket. */ object Directory { @@ -51,6 +49,12 @@ new File(s"${RepositoryHome}/${owner}/${repository}/comments") /** + * Directory for files which are attached to issue. + */ + def getLfsDir(owner: String, repository: String): File = + new File(s"${RepositoryHome}/${owner}/${repository}/lfs") + + /** * Directory for uploaded files by the specified user. */ def getUserUploadDir(userName: String): File = new File(s"${GitBucketHome}/data/${userName}/files") diff --git a/src/main/scala/gitbucket/core/util/FileUtil.scala b/src/main/scala/gitbucket/core/util/FileUtil.scala index f753a60..4836c23 100644 --- a/src/main/scala/gitbucket/core/util/FileUtil.scala +++ b/src/main/scala/gitbucket/core/util/FileUtil.scala @@ -62,4 +62,8 @@ "image/jpeg", "image/png", "text/plain") + + def getLfsFilePath(owner: String, repository: String, oid: String): String = + Directory.getLfsDir(owner, repository) + "/" + oid + } diff --git a/src/main/scala/gitbucket/core/util/StringUtil.scala b/src/main/scala/gitbucket/core/util/StringUtil.scala index 76cc389..f255b1d 100644 --- a/src/main/scala/gitbucket/core/util/StringUtil.scala +++ b/src/main/scala/gitbucket/core/util/StringUtil.scala @@ -1,14 +1,23 @@ package gitbucket.core.util import java.net.{URLDecoder, URLEncoder} + import org.mozilla.universalchardet.UniversalDetector import ControlUtil._ import org.apache.commons.io.input.BOMInputStream import org.apache.commons.io.IOUtils +import org.apache.commons.codec.binary.{Base64, StringUtils} + import scala.util.control.Exception._ object StringUtil { + private lazy val BlowfishKey = { + // last 4 numbers in current timestamp + val time = System.currentTimeMillis.toString + time.substring(time.length - 4) + } + def sha1(value: String): String = defining(java.security.MessageDigest.getInstance("SHA-1")){ md => md.update(value.getBytes) @@ -21,6 +30,20 @@ md.digest.map(b => "%02x".format(b)).mkString } + def encodeBlowfish(value: String): String = { + val spec = new javax.crypto.spec.SecretKeySpec(BlowfishKey.getBytes(), "Blowfish") + val cipher = javax.crypto.Cipher.getInstance("Blowfish") + cipher.init(javax.crypto.Cipher.ENCRYPT_MODE, spec) + new String(Base64.encodeBase64(cipher.doFinal(value.getBytes("UTF-8"))), "UTF-8") + } + + def decodeBlowfish(value: String): String = { + val spec = new javax.crypto.spec.SecretKeySpec(BlowfishKey.getBytes(), "Blowfish") + val cipher = javax.crypto.Cipher.getInstance("Blowfish") + cipher.init(javax.crypto.Cipher.DECRYPT_MODE, spec) + new String(cipher.doFinal(Base64.decodeBase64(value)), "UTF-8") + } + def urlEncode(value: String): String = URLEncoder.encode(value, "UTF-8").replace("+", "%20") def urlDecode(value: String): String = URLDecoder.decode(value, "UTF-8") diff --git a/src/main/twirl/gitbucket/core/admin/system.scala.html b/src/main/twirl/gitbucket/core/admin/system.scala.html index b2a582d..549f32e 100644 --- a/src/main/twirl/gitbucket/core/admin/system.scala.html +++ b/src/main/twirl/gitbucket/core/admin/system.scala.html @@ -123,27 +123,25 @@
- Both of SSH host and Base URL are required if SSH access is enabled. -
@@ -157,14 +155,14 @@- Enable notification not only SMTP configuration if you want to send notification email. -