diff --git a/sdk/java/src/main/java/io/juicefs/JuiceFileSystemImpl.java b/sdk/java/src/main/java/io/juicefs/JuiceFileSystemImpl.java index 697f697f0285..6ecd97ee3094 100644 --- a/sdk/java/src/main/java/io/juicefs/JuiceFileSystemImpl.java +++ b/sdk/java/src/main/java/io/juicefs/JuiceFileSystemImpl.java @@ -1185,13 +1185,14 @@ public FileChecksum getFileChecksum(Path f, long length) throws IOException { int bytesPerCrc = getConf().getInt("io.bytes.per.checksum", 512); DataChecksum summer = DataChecksum.newDataChecksum(ctype, bytesPerCrc); - long crcPerBlock = 0; DataOutputBuffer checksumBuf = new DataOutputBuffer(); DataOutputBuffer crcBuf = new DataOutputBuffer(); byte[] buf = new byte[bytesPerCrc]; FSDataInputStream in = open(f, 1 << 20); - while (length > 0) { - for (int i = 0; i < blocksize / bytesPerCrc && length > 0; i++) { + boolean eof = false; + long got = 0; + while (got < length && !eof) { + for (int i = 0; i < blocksize / bytesPerCrc && got < length; i++) { int n; if (length < bytesPerCrc) { n = in.read(buf, 0, (int) length); @@ -1199,20 +1200,18 @@ public FileChecksum getFileChecksum(Path f, long length) throws IOException { n = in.read(buf); } if (n <= 0) { - length = 0; // EOF + eof = true; + break; } else { summer.update(buf, 0, n); summer.writeValue(crcBuf, true); - length -= n; + got += n; } } if (crcBuf.getLength() > 0) { MD5Hash blockMd5 = MD5Hash.digest(crcBuf.getData(), 0, crcBuf.getLength()); blockMd5.write(checksumBuf); crcBuf.reset(); - if (length > 0) { // more than one block - crcPerBlock = blocksize / bytesPerCrc; - } } } in.close(); @@ -1220,6 +1219,10 @@ public FileChecksum getFileChecksum(Path f, long length) throws IOException { return new MD5MD5CRC32GzipFileChecksum(0, 0, MD5Hash.digest(new byte[32])); } MD5Hash md5 = MD5Hash.digest(checksumBuf.getData()); + long crcPerBlock = 0; + if (got > blocksize) { // more than one block + crcPerBlock = blocksize / bytesPerCrc; + } if (ctype == DataChecksum.Type.CRC32C) { return new MD5MD5CRC32CastagnoliFileChecksum(bytesPerCrc, crcPerBlock, md5); } else { diff --git a/sdk/java/src/test/java/io/juicefs/JuiceFileSystemTest.java b/sdk/java/src/test/java/io/juicefs/JuiceFileSystemTest.java index 2d27b4f97f5c..cd02431eda4a 100644 --- a/sdk/java/src/test/java/io/juicefs/JuiceFileSystemTest.java +++ b/sdk/java/src/test/java/io/juicefs/JuiceFileSystemTest.java @@ -423,6 +423,16 @@ public void testChecksum() throws IOException { assertEquals(new MD5MD5CRC32CastagnoliFileChecksum(512, 0, new MD5Hash("05a157db1cc7549c82ec6f31f63fdb46")), sum); + f = new Path("/medium"); + out = fs.create(f, true); + byte[] bytes = new byte[(128 << 20) - 1]; + out.write(bytes); + out.close(); + sum = fs.getFileChecksum(f); + assertEquals( + new MD5MD5CRC32CastagnoliFileChecksum(512, 0, new MD5Hash("1cf326bae8274fd824ec69ece3e4082f")), + sum); + f = new Path("/big"); out = fs.create(f, true); byte[] zeros = new byte[1024 * 1000];