diff --git a/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala b/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala index 7a79584..a593a0f 100644 --- a/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala +++ b/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala @@ -1,8 +1,8 @@ package gitbucket.core.controller import java.io.File -import javax.servlet.http.{HttpServletRequest, HttpServletResponse} +import javax.servlet.http.{HttpServletRequest, HttpServletResponse} import gitbucket.core.plugin.PluginRegistry import gitbucket.core.repo.html import gitbucket.core.helper @@ -17,6 +17,13 @@ import gitbucket.core.service.WebHookService._ import gitbucket.core.view import gitbucket.core.view.helpers +import org.apache.commons.compress.archivers.{ArchiveEntry, ArchiveOutputStream} +import org.apache.commons.compress.archivers.tar.{TarArchiveEntry, TarArchiveOutputStream} +import org.apache.commons.compress.archivers.zip.{ZipArchiveEntry, ZipArchiveOutputStream} +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream +import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream +import org.apache.commons.compress.utils.IOUtils import org.scalatra.forms._ import org.apache.commons.io.FileUtils import org.eclipse.jgit.api.{ArchiveCommand, Git} @@ -24,7 +31,10 @@ import org.eclipse.jgit.dircache.{DirCache, DirCacheBuilder} import org.eclipse.jgit.errors.MissingObjectException import org.eclipse.jgit.lib._ +import org.eclipse.jgit.revwalk.RevWalk import org.eclipse.jgit.transport.{ReceiveCommand, ReceivePack} +import org.eclipse.jgit.treewalk.TreeWalk +import org.eclipse.jgit.treewalk.filter.PathFilter import org.json4s.jackson.Serialization import org.scalatra._ import org.scalatra.i18n.Messages @@ -813,16 +823,31 @@ }) /** - * Download repository contents as an archive. + * Download repository contents as an archive as compatible URL. */ - get("/:owner/:repository/archive/*")(referrersOnly { repository => - multiParams("splat").head match { - case name if name.endsWith(".zip") => - archiveRepository(name, ".zip", repository) - case name if name.endsWith(".tar.gz") => - archiveRepository(name, ".tar.gz", repository) - case _ => BadRequest() - } + get("/:owner/:repository/archive/:branch.:suffix")(referrersOnly { repository => + val branch = params("branch") + val suffix = params("suffix") + archiveRepository(branch, branch + "." + suffix, repository, "") + }) + + /** + * Download all repository contents as an archive. + */ + get("/:owner/:repository/archive/:branch/:name")(referrersOnly { repository => + val branch = params("branch") + val name = params("name") + archiveRepository(branch, name, repository, "") + }) + + /** + * Download repositories subtree contents as an archive. + */ + get("/:owner/:repository/archive/:branch/*/:name")(referrersOnly { repository => + val branch = params("branch") + val name = params("name") + val path = multiParams("splat").head + archiveRepository(branch, name, repository, path) }) get("/:owner/:repository/network/members")(referrersOnly { repository => @@ -1110,26 +1135,88 @@ } } - private def archiveRepository(name: String, suffix: String, repository: RepositoryService.RepositoryInfo): Unit = { - val revision = name.stripSuffix(suffix) + def archiveRepository( + revision: String, + filename: String, + repository: RepositoryService.RepositoryInfo, + path: String + ): Unit = { + def archive(archiveFormat: String, archive: ArchiveOutputStream)( + entryCreator: (String, Long, Int) => ArchiveEntry + ): Unit = { + using(Git.open(getRepositoryDir(repository.owner, repository.name))) { git => + val oid = git.getRepository.resolve(revision) + val revCommit = JGitUtil.getRevCommitFromId(git, oid) + val sha1 = oid.getName() + val repositorySuffix = (if (sha1.startsWith(revision)) sha1 else revision).replace('/', '-') + val pathSuffix = if (path.isEmpty) "" else '-' + path.replace('/', '-') + val baseName = repository.name + "-" + repositorySuffix + pathSuffix + val filename = baseName + archiveFormat - using(Git.open(getRepositoryDir(repository.owner, repository.name))) { git => - val oid = git.getRepository.resolve(revision) - val revCommit = JGitUtil.getRevCommitFromId(git, oid) - val sha1 = oid.getName() - val repositorySuffix = (if (sha1.startsWith(revision)) sha1 else revision).replace('/', '-') - val filename = repository.name + "-" + repositorySuffix + suffix + using(new RevWalk(git.getRepository)) { revWalk => + using(new TreeWalk(git.getRepository)) { treeWalk => + treeWalk.addTree(revCommit.getTree) + treeWalk.setRecursive(true) + if (!path.isEmpty) { + treeWalk.setFilter(PathFilter.create(path)) + } + if (treeWalk != null) { + while (treeWalk.next()) { + val entryPath = + if (path.isEmpty) baseName + "/" + treeWalk.getPathString + else path.split("/").last + treeWalk.getPathString.substring(path.length) + val size = JGitUtil.getFileSize(git, repository, treeWalk) + val mode = treeWalk.getFileMode.getBits + val entry: ArchiveEntry = entryCreator(entryPath, size, mode) + JGitUtil.openFile(git, repository, revCommit.getTree, treeWalk.getPathString) { in => + archive.putArchiveEntry(entry) + IOUtils.copy(in, archive) + archive.closeArchiveEntry() + } + } + } + } + } + } + } + val tarRe = """\.tar\.(gz|bz2|xz)$""".r - contentType = "application/octet-stream" - response.setHeader("Content-Disposition", s"attachment; filename=${filename}") - response.setBufferSize(1024 * 1024); - - git.archive - .setFormat(suffix.tail) - .setPrefix(repository.name + "-" + repositorySuffix + "/") - .setTree(revCommit) - .setOutputStream(response.getOutputStream) - .call() + filename match { + case name if name.endsWith(".zip") => + response.setHeader("Content-Disposition", s"attachment; filename=${filename}") + contentType = "application/octet-stream" + response.setBufferSize(1024 * 1024); + using(new ZipArchiveOutputStream(response.getOutputStream)) { zip => + archive(".zip", zip) { (path, size, mode) => + val entry = new ZipArchiveEntry(path) + entry.setSize(size) + entry.setUnixMode(mode) + entry + } + } + case tarRe(compressor) => + response.setHeader("Content-Disposition", s"attachment; filename=${filename}") + contentType = "application/octet-stream" + response.setBufferSize(1024 * 1024) + using(compressor match { + case "gz" => new GzipCompressorOutputStream(response.getOutputStream) + case "bz2" => new BZip2CompressorOutputStream(response.getOutputStream) + case "xz" => new XZCompressorOutputStream(response.getOutputStream) + }) { compressorOutputStream => + using(new TarArchiveOutputStream(compressorOutputStream)) { tar => + tar.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_STAR) + tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU) + tar.setAddPaxHeadersForNonAsciiNames(true) + archive(".tar.gz", tar) { (path, size, mode) => + val entry = new TarArchiveEntry(path) + entry.setSize(size) + entry.setMode(mode) + entry + } + } + } + case _ => + BadRequest() } } diff --git a/src/main/scala/gitbucket/core/util/JGitUtil.scala b/src/main/scala/gitbucket/core/util/JGitUtil.scala index b6c8273..811302d 100644 --- a/src/main/scala/gitbucket/core/util/JGitUtil.scala +++ b/src/main/scala/gitbucket/core/util/JGitUtil.scala @@ -1,6 +1,6 @@ package gitbucket.core.util -import java.io.ByteArrayOutputStream +import java.io.{ByteArrayOutputStream, File, FileInputStream, InputStream} import gitbucket.core.service.RepositoryService import org.eclipse.jgit.api.Git @@ -1220,4 +1220,60 @@ Option(git.getRepository.resolve(revstr)).map(ObjectId.toString(_)) } } + + def getFileSize(git: Git, repository: RepositoryService.RepositoryInfo, treeWalk: TreeWalk): Long = { + val attrs = treeWalk.getAttributes + val loader = git.getRepository.open(treeWalk.getObjectId(0)) + if (attrs.containsKey("filter") && attrs.get("filter").getValue == "lfs") { + val lfsAttrs = getLfsAttributes(loader) + lfsAttrs.get("size").map(_.toLong).get + } else { + loader.getSize + } + } + + def getFileSize(git: Git, repository: RepositoryService.RepositoryInfo, tree: RevTree, path: String): Long = { + using(TreeWalk.forPath(git.getRepository, path, tree)) { treeWalk => + getFileSize(git, repository, treeWalk) + } + } + + def openFile[T](git: Git, repository: RepositoryService.RepositoryInfo, treeWalk: TreeWalk)( + f: InputStream => T + ): T = { + val attrs = treeWalk.getAttributes + val loader = git.getRepository.open(treeWalk.getObjectId(0)) + if (attrs.containsKey("filter") && attrs.get("filter").getValue == "lfs") { + val lfsAttrs = getLfsAttributes(loader) + if (lfsAttrs.nonEmpty) { + val oid = lfsAttrs("oid").split(":")(1) + + val file = new File(FileUtil.getLfsFilePath(repository.owner, repository.name, oid)) + using(new FileInputStream(FileUtil.getLfsFilePath(repository.owner, repository.name, oid))) { in => + f(in) + } + } else { + throw new RuntimeException + } + } else { + using(loader.openStream()) { in => + f(in) + } + } + } + + def openFile[T](git: Git, repository: RepositoryService.RepositoryInfo, tree: RevTree, path: String)( + f: InputStream => T + ): T = { + using(TreeWalk.forPath(git.getRepository, path, tree)) { treeWalk => + openFile(git, repository, treeWalk)(f) + } + } + + private def getLfsAttributes(loader: ObjectLoader): Map[String, String] = { + val bytes = loader.getCachedBytes + val text = new String(bytes, "UTF-8") + + JGitUtil.getLfsObjects(text) + } } diff --git a/src/main/twirl/gitbucket/core/repo/files.scala.html b/src/main/twirl/gitbucket/core/repo/files.scala.html index c17996c..0c8d5f4 100644 --- a/src/main/twirl/gitbucket/core/repo/files.scala.html +++ b/src/main/twirl/gitbucket/core/repo/files.scala.html @@ -33,6 +33,7 @@
+ Download ZIP @if(commitCount > 10000){10000+} else {@commitCount} @helpers.plural(commitCount, "commit")
@@ -44,7 +45,6 @@ @if(context.platform != "linux" && context.platform != null){ } - Download ZIP