Merge branch 'globs'

.gitignore

@@ -1,3 +1,4 @@
+*.orig
 *.class
 *.local
 *.mapdb*

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changes
 
+## 0.2.4
+
+ * Fix non-suffix pattern matching (#20, thanks to @wesyq).
+
 ## 0.2.3
 
  * Allow load globs from file (#19, thanks to @leth).

build.gradle

@@ -23,16 +23,17 @@ apply plugin: "com.github.ben-manes.versions"
 
 dependencies {
   compile "org.jetbrains:annotations:15.0"
-  compile "org.eclipse.jgit:org.eclipse.jgit:4.1.1.201511131810-r"
-  compile "org.mapdb:mapdb:1.0.8"
-  compile "org.slf4j:slf4j-simple:1.7.13"
-  compile "org.jgrapht:jgrapht-core:0.9.1"
-  compile "com.beust:jcommander:1.48"
+  compile "org.eclipse.jgit:org.eclipse.jgit:4.5.0.201609210915-r"
+  compile "org.mapdb:mapdb:3.0.2"
+  compile "org.slf4j:slf4j-simple:1.7.21"
+  compile "org.jgrapht:jgrapht-core:1.0.0"
+  compile "com.beust:jcommander:1.58"
 
   compile "ru.bozaro.gitlfs:gitlfs-pointer:0.10.0"
   compile "ru.bozaro.gitlfs:gitlfs-client:0.10.0"
 
-  testCompile "org.testng:testng:6.9.10"
+  testCompile "com.google.jimfs:jimfs:1.1"
+  testCompile "org.testng:testng:6.9.13.6"
 }
 
 sourceCompatibility = JavaVersion.VERSION_1_8

src/main/java/git/lfs/migrate/GitConverter.java

@@ -1,5 +1,7 @@
 package git.lfs.migrate;
 
+import git.path.PathMatcher;
+import git.path.WildcardHelper;
 import org.apache.commons.codec.binary.Hex;
 import org.eclipse.jgit.errors.InvalidPatternException;
 import org.eclipse.jgit.fnmatch.FileNameMatcher;
@@ -9,15 +11,19 @@
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.mapdb.DB;
-import org.mapdb.DBMaker;
 import org.mapdb.HTreeMap;
+import org.mapdb.Serializer;
+import org.mapdb.serializer.SerializerJava;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import ru.bozaro.gitlfs.common.data.Meta;
 import ru.bozaro.gitlfs.pointer.Pointer;
 
 import java.io.*;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.*;
@@ -27,45 +33,42 @@
  * Converter for git objects.
  * Created by bozaro on 09.06.15.
  */
-public class GitConverter implements AutoCloseable {
+public class GitConverter {
   @NotNull
   private static final Logger log = LoggerFactory.getLogger(GitConverter.class);
   @NotNull
   private static final String GIT_ATTRIBUTES = ".gitattributes";
   @NotNull
   private final String[] globs;
   @NotNull
-  private final File basePath;
-  @NotNull
-  private final File tempPath;
+  private final PathMatcher[] matchers;
   @NotNull
   private final DB cache;
   @NotNull
+  private final Path basePath;
+  @NotNull
+  private final Path tempPath;
+  @NotNull
   private final HTreeMap<String, MetaData> cacheMeta;
 
-  public GitConverter(@NotNull File cachePath, @NotNull File basePath, @NotNull String[] globs) throws IOException, InvalidPatternException {
+  public GitConverter(@NotNull DB cache, @NotNull Path basePath, @NotNull String[] globs) throws IOException, InvalidPatternException {
     this.basePath = basePath;
+    this.cache = cache;
     this.globs = globs.clone();
+    this.matchers = convertGlobs(globs);
     Arrays.sort(globs);
 
     for (String glob : globs) {
       new FileNameMatcher(glob, '/');
     }
 
-    tempPath = new File(basePath, "lfs/tmp");
-    makeParentDirs(tempPath);
-    makeParentDirs(cachePath);
-    cache = DBMaker.newFileDB(new File(cachePath, "git-lfs-migrate.mapdb"))
-        .asyncWriteEnable()
-        .mmapFileEnable()
-        .cacheSoftRefEnable()
-        .make();
-    cacheMeta = cache.getHashMap("meta");
-  }
-
-  @Override
-  public void close() throws IOException {
-    cache.close();
+    tempPath = basePath.resolve("lfs/tmp");
+    Files.createDirectories(tempPath);
+    //noinspection unchecked
+    cacheMeta = cache.<String, MetaData>hashMap("meta")
+        .keySerializer(Serializer.STRING)
+        .valueSerializer(new SerializerJava())
+        .createOrOpen();
   }
 
   @NotNull
@@ -231,22 +234,29 @@ public ObjectId convert(@NotNull ObjectInserter inserter, @NotNull ConvertResolv
     };
   }
 
-  private boolean matchFilename(@NotNull String fileName) {
+  @NotNull
+  private static PathMatcher[] convertGlobs(String[] globs) throws InvalidPatternException {
+    final PathMatcher[] matchers = new PathMatcher[globs.length];
+    for (int i = 0; i < globs.length; ++i) {
+      String glob = globs[i];
+      if (!glob.contains("/")) {
+        glob = "**/" + glob;
+      }
+      matchers[i] = WildcardHelper.createMatcher(glob, true);
+    }
+    return matchers;
+  }
+
+  public boolean matchFilename(@NotNull String fileName) {
     if (!fileName.startsWith("/")) {
       throw new IllegalStateException("Unexpected file name: " + fileName);
     }
-    try {
-      for (String glob : globs) {
-        final FileNameMatcher matcher = new FileNameMatcher(glob, null);
-        matcher.append(fileName.substring(1));
-        if (matcher.isMatch()) {
-          return true;
-        }
+    for (PathMatcher matcher : matchers) {
+      if (WildcardHelper.isMatch(matcher, fileName)) {
+        return true;
       }
-      return false;
-    } catch (InvalidPatternException e) {
-      throw new IllegalArgumentException(e);
     }
+    return false;
   }
 
   @NotNull
@@ -320,11 +330,11 @@ private String createRemoteFile(@NotNull ObjectId id, @NotNull ObjectLoader load
   @NotNull
   private String createLocalFile(@NotNull ObjectId id, @NotNull ObjectLoader loader) throws IOException {
     // Create LFS stream.
-    final File tmpFile = new File(tempPath, UUID.randomUUID().toString());
+    final Path tmpFile = tempPath.resolve(UUID.randomUUID().toString());
     final MessageDigest md = createSha256();
     int size = 0;
     try (InputStream istream = loader.openStream();
-         OutputStream ostream = new FileOutputStream(tmpFile)) {
+         OutputStream ostream = Files.newOutputStream(tmpFile)) {
       byte[] buffer = new byte[0x10000];
       while (true) {
         int read = istream.read(buffer);
@@ -338,24 +348,20 @@ private String createLocalFile(@NotNull ObjectId id, @NotNull ObjectLoader loade
     cacheMeta.putIfAbsent(id.name(), new MetaData(hash, size));
     cache.commit();
     // Rename file.
-    final File lfsFile = new File(basePath, "lfs/objects/" + hash.substring(0, 2) + "/" + hash.substring(2, 4) + "/" + hash);
-    makeParentDirs(lfsFile.getParentFile());
-    if (lfsFile.exists()) {
-      if (!tmpFile.delete()) {
-        log.warn("Can't delete temporary file: {}", lfsFile.getAbsolutePath());
+    final Path lfsFile = basePath.resolve("lfs/objects/" + hash.substring(0, 2) + "/" + hash.substring(2, 4) + "/" + hash);
+    Files.createDirectories(lfsFile.getParent());
+    if (Files.exists(lfsFile)) {
+      try {
+        Files.delete(tmpFile);
+      } catch (IOException e) {
+        log.warn("Can't delete temporary file: {}", lfsFile.toAbsolutePath());
       }
-    } else if (!tmpFile.renameTo(lfsFile)) {
-      throw new IOException("Can't rename file: " + tmpFile + " -> " + lfsFile);
+    } else {
+      Files.move(tmpFile, lfsFile, StandardCopyOption.ATOMIC_MOVE);
     }
     return hash;
   }
 
-  private void makeParentDirs(@NotNull File path) throws IOException {
-    if (!path.mkdirs() && !path.exists()) {
-      throw new IOException("Can't create directory: " + path.getAbsolutePath());
-    }
-  }
-
   @NotNull
   private static MessageDigest createSha256() {
     // Prepare for hash calculation

src/main/java/git/lfs/migrate/Main.java

@@ -13,6 +13,8 @@
 import org.jetbrains.annotations.Nullable;
 import org.jgrapht.graph.DefaultEdge;
 import org.jgrapht.graph.SimpleDirectedGraph;
+import org.mapdb.DB;
+import org.mapdb.DBMaker;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import ru.bozaro.gitlfs.client.AuthHelper;
@@ -25,13 +27,9 @@
 import ru.bozaro.gitlfs.common.data.*;
 import ru.bozaro.gitlfs.common.data.Error;
 
-import java.io.File;
 import java.io.IOException;
 import java.net.URI;
-import java.nio.file.FileVisitResult;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.SimpleFileVisitor;
+import java.nio.file.*;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.security.GeneralSecurityException;
 import java.util.*;
@@ -79,7 +77,7 @@ public static void main(@NotNull String[] args) throws Exception {
     String[] globs = cmd.globs.toArray(new String[cmd.globs.size()]);
     if (cmd.globFile != null) {
       globs = Stream.concat(Arrays.stream(globs),
-          Files.lines(cmd.globFile.toPath())
+          Files.lines(cmd.globFile)
               .map(String::trim)
               .filter(s -> !s.isEmpty())
       ).toArray(String[]::new);
@@ -147,19 +145,21 @@ private static boolean checkLfsAuthenticate(@Nullable Client client) throws IOEx
     return false;
   }
 
-  public static void processRepository(@NotNull File srcPath, @NotNull File dstPath, @NotNull File cachePath, @Nullable Client client, int writeThreads, int uploadThreads, @NotNull String... globs) throws IOException, InterruptedException, ExecutionException, InvalidPatternException {
+  public static void processRepository(@NotNull Path srcPath, @NotNull Path dstPath, @NotNull Path cachePath, @Nullable Client client, int writeThreads, int uploadThreads, @NotNull String... globs) throws IOException, InterruptedException, ExecutionException, InvalidPatternException {
     removeDirectory(dstPath);
-    dstPath.mkdirs();
+    Files.createDirectories(dstPath);
 
     final Repository srcRepo = new FileRepositoryBuilder()
         .setMustExist(true)
-        .setGitDir(srcPath).build();
+        .setGitDir(srcPath.toFile()).build();
     final Repository dstRepo = new FileRepositoryBuilder()
         .setMustExist(false)
-        .setGitDir(dstPath).build();
+        .setGitDir(dstPath.toFile()).build();
 
-    final GitConverter converter = new GitConverter(cachePath, dstPath, globs);
-    try {
+    try (DB cache = DBMaker.fileDB(cachePath.resolve("git-lfs-migrate.mapdb").toFile())
+        .fileMmapEnableIfSupported()
+        .make()) {
+      final GitConverter converter = new GitConverter(cache, dstPath, globs);
       dstRepo.create(true);
       // Load all revision list.
       log.info("Reading full objects list...");
@@ -279,9 +279,9 @@ private static void processSingleThread(@NotNull GitConverter converter, @NotNul
     }
   }
 
-  private static void removeDirectory(@NotNull File path) throws IOException {
-    if (path.exists()) {
-      Files.walkFileTree(path.toPath(), new SimpleFileVisitor<Path>() {
+  private static void removeDirectory(@NotNull Path path) throws IOException {
+    if (Files.exists(path)) {
+      Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
         @Override
         public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
           Files.delete(file);
@@ -435,13 +435,13 @@ private void print(long current) {
   public static class CmdArgs {
     @Parameter(names = {"-s", "--source"}, description = "Source repository", required = true)
     @NotNull
-    private File src;
+    private Path src;
     @Parameter(names = {"-d", "--destination"}, description = "Destination repository", required = true)
     @NotNull
-    private File dst;
+    private Path dst;
     @Parameter(names = {"-c", "--cache"}, description = "Source repository", required = false)
     @NotNull
-    private File cache = new File(".");
+    private Path cache = FileSystems.getDefault().getPath(".");
     @Parameter(names = {"-g", "--git"}, description = "GIT repository url (ignored with --lfs parameter)", required = false)
     @Nullable
     private String git;
@@ -457,7 +457,7 @@ public static class CmdArgs {
     @Parameter(names = {"--no-check-certificate"}, description = "Don't check the server certificate against the available certificate authorities")
     private boolean noCheckCertificate = false;
     @Parameter(names = {"--glob-file"}, description = "File containing glob patterns")
-    private File globFile = null;
+    private Path globFile = null;
 
     @Parameter(description = "LFS file glob patterns")
     @NotNull

src/main/java/git/path/NameMatcher.java

@@ -0,0 +1,14 @@
+package git.path;
+
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Interface for matching name of path.
+ *
+ * @author Artem V. Navrotskiy <[email protected]>
+ */
+public interface NameMatcher {
+  boolean isMatch(@NotNull String name, boolean isDir);
+
+  boolean isRecursive();
+}

src/main/java/git/path/PathMatcher.java

@@ -0,0 +1,16 @@
+package git.path;
+
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Interface for path matching.
+ *
+ * @author Artem V. Navrotskiy <[email protected]>
+ */
+public interface PathMatcher {
+  @Nullable
+  PathMatcher createChild(@NotNull String name, boolean isDir);
+
+  boolean isMatch();
+}