Skip to content

Commit

Permalink
git: Support source code download as zip file
Browse files Browse the repository at this point in the history
  • Loading branch information
doortts committed Dec 16, 2016
1 parent f324068 commit 8afa58b
Show file tree
Hide file tree
Showing 10 changed files with 202 additions and 7 deletions.
43 changes: 41 additions & 2 deletions app/controllers/CodeApp.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,24 @@
package controllers;

import actions.DefaultProjectCheckAction;
import com.fasterxml.jackson.databind.node.ObjectNode;
import controllers.annotation.AnonymousCheck;
import controllers.annotation.IsAllowed;
import models.Project;
import models.enumeration.Operation;
import org.apache.commons.io.FilenameUtils;
import org.apache.tika.Tika;
import org.apache.tika.mime.MediaType;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.eclipse.jgit.api.ArchiveCommand;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.archive.ZipFormat;
import org.tmatesoft.svn.core.SVNException;
import play.mvc.*;
import play.mvc.Controller;
import play.mvc.Http;
import play.mvc.Result;
import play.mvc.With;
import playRepository.GitRepository;
import playRepository.PlayRepository;
import playRepository.RepositoryService;
import utils.ErrorViews;
Expand Down Expand Up @@ -118,6 +125,38 @@ public static Result ajaxRequest(String userName, String projectName, String pat
}
}

@With(DefaultProjectCheckAction.class)
public static Result download(String userName, String projectName, String branch, String path)
throws UnsupportedOperationException, IOException, SVNException, GitAPIException, ServletException {
Project project = Project.findByOwnerAndProjectName(userName, projectName);

if (!RepositoryService.VCS_GIT.equals(project.vcs) && !RepositoryService.VCS_SUBVERSION.equals(project.vcs)) {
return status(Http.Status.NOT_IMPLEMENTED, project.vcs + " is not supported!");
}

final String targetBranch = HttpUtil.decodePathSegment(branch);
final String targetPath = HttpUtil.decodePathSegment(path);

PlayRepository repository = RepositoryService.getRepository(project);
List<ObjectNode> recursiveData = RepositoryService.getMetaDataFromAncestorDirectories(
repository, targetBranch, targetPath);

if (recursiveData == null) {
return notFound(ErrorViews.NotFound.render());
}

// Prepare a chunked text stream
Chunks<byte[]> chunks = new ByteChunks() {
// Called when the stream is ready
public void onReady(Chunks.Out<byte[]> out) {
repository.getArchive(out, targetBranch);
}
};

response().setHeader("Content-Disposition", "attachment; filename=" + projectName + "-" + branch + ".zip");
return ok(chunks);
}

@With(DefaultProjectCheckAction.class)
public static Result ajaxRequestWithBranch(String userName, String projectName, String branch, String path)
throws UnsupportedOperationException, IOException, SVNException, GitAPIException, ServletException{
Expand Down
31 changes: 27 additions & 4 deletions app/playRepository/GitRepository.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
*/
package playRepository;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import controllers.UserApp;
import controllers.routes;
import models.Project;
Expand All @@ -33,19 +35,20 @@
import org.apache.commons.lang3.StringUtils;
import org.apache.tika.Tika;
import org.apache.tika.metadata.Metadata;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.eclipse.jgit.api.ArchiveCommand;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.LogCommand;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.archive.ZipFormat;
import org.eclipse.jgit.attributes.AttributesNode;
import org.eclipse.jgit.attributes.AttributesNodeProvider;
import org.eclipse.jgit.attributes.AttributesRule;
import org.eclipse.jgit.blame.BlameResult;
import org.eclipse.jgit.diff.*;
import org.eclipse.jgit.diff.Edit.Type;
import org.eclipse.jgit.errors.AmbiguousObjectException;
import org.eclipse.jgit.errors.IncorrectObjectTypeException;
import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.internal.storage.dfs.DfsRepository;
import org.eclipse.jgit.lib.*;
import org.eclipse.jgit.lib.RefUpdate.Result;
import org.eclipse.jgit.revwalk.RevCommit;
Expand All @@ -63,8 +66,9 @@
import org.eclipse.jgit.util.io.NullOutputStream;
import org.tmatesoft.svn.core.SVNException;
import play.Logger;
import play.api.Play;
import play.libs.Json;
import play.mvc.Results.Chunks;
import utils.ChunkedOutputStream;
import utils.FileUtil;
import utils.GravatarUtil;

Expand Down Expand Up @@ -1940,4 +1944,23 @@ public File getDirectory() {
public Repository getRepository() {
return repository;
}

public void getArchive(Chunks.Out<byte[]> out, String branchName){
Git git = new Git(getRepository());
ArchiveCommand.registerFormat("zip", new ZipFormat());

This comment has been minimized.

Copy link
@benelog

benelog Dec 22, 2016

제가 JGit을 잘 몰라서 드리는 질문인데, Multi thread 문제는 없을까요? 즉 아래와 같은순서로 실행되어도 상관이 없는지 궁금합니다.

  1. B사용자 ArchiveCommand.registerFormat
  2. B사용자 git.archive()
  3. A사용자 ArchiveCommand.registerFormat
  4. B사용자 ArchiveCommand.unregisterFormat
  5. A사용자 git.archive()

This comment has been minimized.

Copy link
@doortts

doortts Dec 23, 2016

Author Collaborator

우선 모처럼의 코드를 봐주시는 분이시라 너무 반갑고 좋네요! :D 고맙습니다!

이야기 하신 내용은 테스트를 해보는게 좋겠다는 생각이들었습니다.

우선 stream chunk write 할때는 synchronized 하긴 했는데 실제 Chunked Stream 만들때부터 synchronized 하거나 다른 방법을 찾아봐야 하는 상황이 발생할 수도 있을 것 같고요.

먼저 테스트를 조금 해보고 어느쪽이든 공유드릴게요! : )

This comment has been minimized.

Copy link
@benelog

benelog Dec 23, 2016

일단 제가 문의드린건 ArchiveCommand.registerFormat 메서드가 전역적인 상태를 지정하는것 같은 느낌이 들었서였었습니다. 만약 ArchiveCommand.unregisterFormat(".zip")이 된 상태에서 git.archive() 을 호출헀을때 의도하신 동작이 수행되지 않을수도 있다면 Lock을 아래와 같이 굉장히 넓게 잡아야할수도 있을것 같습니다.

synchronized(ArchiveCommand.class) {
    ArchiveCommand.registerFormat("zip", new ZipFormat());
    try {
        ...
    } catch(...){
        ArchiveCommand.unregisterFormat("zip");
    }
}

위와 같이 된다면 해당 영역은 하나의 Yobi 인스턴스에서 한 사용자만 접근할수 있게 되기는합니다.

말씀하신 ChunkedOutputStream는 조금 다른 측면인데요, ChunkedOutputStream 는 synchronized가 메서드에 걸려있습니다. 이것은 synchronized(this)와 똑같으므로 ChunkedOutputStream 객체마다 다른 lock을 가지게 됩니다. ChunkedOutputStream는 getArchive 메서드 안에서 호출되므로 사용자마다 다른 lock을 가집니다. 그래서 ChunkedOutputStream.write()앞에 synchronized를 거는것만으로는 여러 사용자가 동시에 ChunkedOutputStream.write()에 접근하는것을 막을수는 없을것 같습니다. (의도하신 바를 제가 잘 못 이해했을수도 있을것 같네요.)

This comment has been minimized.

Copy link
@benelog

benelog Dec 23, 2016

제가 Play(Akka)류의 특성을 잘 모르기 때문에 lock에 대해서 한 가정 중 틀린 부분이 있을수도 있습니다.

This comment has been minimized.

Copy link
@doortts

doortts Dec 28, 2016

Author Collaborator

이야기주신 전역상태 지정에 대해 제가 오해하고 있었던것 같습니다. zip 포맷만 지원할거라 굳이 이렇게 register와 unregister를 반복할 필요가 없을 것 같습다. 전역으로 한 번만 호출하고 unregister를 호출하지 않는걸로 변경해서 테스트해봐야 할 것 같아요.

try {
ChunkedOutputStream cos = new ChunkedOutputStream(out, 16384);
git.archive()
.setTree(getRepository().resolve(branchName))
.setFormat("zip")
.setOutputStream(cos)
.call();
} catch (IncorrectObjectTypeException | AmbiguousObjectException | GitAPIException e) {
play.Logger.error(e.getMessage());
} catch (IOException e){
play.Logger.error(e.getMessage());
} finally{
ArchiveCommand.unregisterFormat("zip");
}
}
}
3 changes: 3 additions & 0 deletions app/playRepository/PlayRepository.java
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import models.resource.Resource;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.tmatesoft.svn.core.SVNException;
import play.mvc.Results.Chunks;

import java.io.File;
import java.io.IOException;
Expand Down Expand Up @@ -79,4 +80,6 @@ public interface PlayRepository {
boolean move(String srcProjectOwner, String srcProjectName, String desrProjectOwner, String destProjectName);

public File getDirectory();

public void getArchive(Chunks.Out<byte[]> out, String branchName);
}
6 changes: 6 additions & 0 deletions app/playRepository/SVNRepository.java
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import org.tmatesoft.svn.core.wc.SVNDiffClient;
import org.tmatesoft.svn.core.wc.SVNRevision;
import play.libs.Json;
import play.mvc.Results;
import utils.Config;
import utils.FileUtil;
import utils.GravatarUtil;
Expand Down Expand Up @@ -413,6 +414,11 @@ public File getDirectory() {
return new File(getRootDirectory(), ownerName + "/" + projectName);
}

@Override
public void getArchive(Results.Chunks.Out<byte[]> out, String branchName) {

}

public static File getRootDirectory() {
return new File(Config.getYobiHome(), getRepoPrefix());
}
Expand Down
114 changes: 114 additions & 0 deletions app/utils/ChunkedOutputStream.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
/**
* Yona, 21st Century Project Hosting SW
* <p>
* Copyright Yona & Yobi Authors & NAVER Corp.
* https://yona.io
**/

package utils;

import play.mvc.Results.Chunks;

import java.io.IOException;
import java.io.OutputStream;

//
// ChunkedOutputStream is made by referring to BufferedOutputStream.java
//
public class ChunkedOutputStream extends OutputStream {

Chunks.Out<byte[]> out;
/**
* The internal buffer where data is stored.
*/
protected byte buf[];

/**
* The number of valid bytes in the buffer. This value is always
* in the range <tt>0</tt> through <tt>buf.length</tt>; elements
* <tt>buf[0]</tt> through <tt>buf[count-1]</tt> contain valid
* byte data.
*/
protected int count;

public ChunkedOutputStream(Chunks.Out<byte[]> out, int size) {
if (size <= 0) {
buf = new byte[16384];
} else {
buf = new byte[size];
}
this.out = out;
}

/**
* Writes the specified byte to this buffered output stream.
*
* @param b the byte to be written.
* @exception IOException if an I/O error occurs.
*/
@Override
public synchronized void write(int b) throws IOException {
if (count >= buf.length) {
flushBuffer();
}
buf[count++] = (byte)b;
}

public void write(byte b[]) throws IOException {
throw new UnsupportedOperationException("write(byte b[])");
}

/**
* Writes <code>len</code> bytes from the specified byte array
* starting at offset <code>off</code> to this buffered output stream.
*
* <p> Ordinarily this method stores bytes from the given array into this
* stream's buffer, flushing the buffer to the underlying output stream as
* needed. If the requested length is at least as large as this stream's
* buffer, however, then this method will flush the buffer and write the
* bytes directly to the underlying output stream. Thus redundant
* <code>BufferedOutputStream</code>s will not copy data unnecessarily.
*
* @param b the data.
* @param off the start offset in the data.
* @param len the number of bytes to write.
* @exception IOException if an I/O error occurs.
*/
@Override
public synchronized void write(byte b[], int off, int len) throws IOException {
if (len >= buf.length) {
/* If the request length exceeds the size of the output buffer,
flush the output buffer and then write the data directly.
In this way buffered streams will cascade harmlessly. */
flushBuffer();
write(b, off, len);
return;
}
if (len > buf.length - count) {
flushBuffer();
}
System.arraycopy(b, off, buf, count, len);
count += len;
}

private void flushBuffer() throws IOException {
if (count > 0) {
chunkOut();
}
}

@Override
public void close() throws IOException {
if (count > 0) {
chunkOut();
}
out.close();
}

private void chunkOut() {
byte remainBuf[] = new byte[count];
System.arraycopy(buf, 0, remainBuf,0, count);
out.write(remainBuf);
count = 0;
}
}
6 changes: 6 additions & 0 deletions app/views/code/view.scala.html
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,12 @@
<a href="@routes.CodeApp.codeBrowserWithBranch(project.owner, project.name, URLEncoder.encode(branch, "UTF-8"), "")">@project.name</a>
@makeBreadCrumbs(path)
</div>
@if(project.isGit) {
<div class="pull-right">
<a href="@routes.CodeApp.download(project.owner, project.name, URLEncoder.encode(branch, "UTF-8"))" class="ybtn">
@Messages("code.download")</a>
</div>
}
</div>

<div class="code-viewer-wrap">
Expand Down
3 changes: 2 additions & 1 deletion build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ libraryDependencies ++= Seq(
"org.eclipse.jgit" % "org.eclipse.jgit.http.server" % "4.5.0.201609210915-r",
// JGit Large File Storage
"org.eclipse.jgit" % "org.eclipse.jgit.lfs" % "4.5.0.201609210915-r",
// JGit Archive Formats
"org.eclipse.jgit" % "org.eclipse.jgit.archive" % "4.5.0.201609210915-r",
// svnkit
"org.tmatesoft.svnkit" % "svnkit" % "1.8.12",
// svnkit-dav
Expand Down Expand Up @@ -68,7 +70,6 @@ val projectSettings = Seq(
includeFilter in (Assets, LessKeys.less) := "*.less",
excludeFilter in (Assets, LessKeys.less) := "_*.less",
javaOptions in test ++= Seq("-Xmx2g", "-Xms1g", "-XX:MaxPermSize=1g", "-Dfile.encoding=UTF-8"),
javacOptions ++= Seq("-Xlint:all", "-Xlint:-path"),
scalacOptions ++= Seq("-feature")
)

Expand Down
1 change: 1 addition & 0 deletions conf/messages
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ code.copyCommitId.copied = Commit ID is copied
code.copyUrl = Copy URL
code.copyUrl.copied = URL is copied
code.deletedPath = {0} (deleted)
code.download = Download as .zip file
code.eolMissing = No newline at end of file
code.fileDiffLimitExceeded = Up to {0} files will be displayed.
code.fileModeChanged = File mode has changed
Expand Down
1 change: 1 addition & 0 deletions conf/messages.ko-KR
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ code.copyCommitId.copied = 커밋 ID가 복사되었습니다
code.copyUrl = 주소 복사
code.copyUrl.copied = 주소가 복사 되었습니다.
code.deletedPath = {0} (삭제됨)
code.download = 압축 파일로 내려받기
code.eolMissing = 파일 끝에 줄바꿈 문자 없음
code.fileDiffLimitExceeded = 최대 {0}개의 파일까지만 보여드립니다.
code.fileModeChanged = 파일 모드 변경됨
Expand Down
1 change: 1 addition & 0 deletions conf/routes
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,7 @@ GET /:user/:project/code/:branch/!/*path
#for normal
GET /:user/:project/code controllers.CodeApp.codeBrowser(user, project)
GET /:user/:project/code/:branch controllers.CodeApp.codeBrowserWithBranch(user, project, branch:String, path="")
GET /:user/:project/code/:branch/download controllers.CodeApp.download(user, project, branch:String, path="")
GET /:user/:project/code/:branch/*path controllers.CodeApp.codeBrowserWithBranch(user, project, branch:String, path:String)
GET /:user/:project/rawcode/:rev/*path controllers.CodeApp.showRawFile(user, project, rev:String, path:String)
GET /:user/:project/files/:rev/*path controllers.CodeApp.openFile(user, project, rev:String, path:String)
Expand Down

0 comments on commit 8afa58b

Please sign in to comment.