diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java
index c92b1e3..a343fc0 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java
@@ -205,6 +205,14 @@ private int buildMatrix(ProgressMonitor pm) throws IOException {
 		//
 		matrix = new long[srcs.size() * dsts.size()];
 
+		long[] srcSizes = new long[srcs.size()];
+		long[] dstSizes = new long[dsts.size()];
+
+		// Init the size arrays to some value that indicates that we haven't
+		// calculated the size yet. Since sizes cannot be negative, -1 will work
+		Arrays.fill(srcSizes, -1);
+		Arrays.fill(dstSizes, -1);
+
 		// Consider each pair of files, if the score is above the minimum
 		// threshold we need record that scoring in the matrix so we can
 		// later find the best matches.
@@ -231,6 +239,26 @@ private int buildMatrix(ProgressMonitor pm) throws IOException {
 					continue;
 				}
 
+				long srcSize = srcSizes[srcIdx];
+				if (srcSize < 0) {
+					srcSize = size(srcEnt.oldId.toObjectId());
+					srcSizes[srcIdx] = srcSize;
+				}
+
+				long dstSize = dstSizes[dstIdx];
+				if (dstSize < 0) {
+					dstSize = size(dstEnt.newId.toObjectId());
+					dstSizes[dstIdx] = dstSize;
+				}
+
+				long max = Math.max(srcSize, dstSize);
+				long min = Math.min(srcSize, dstSize);
+				if (min * 100 / max < renameScore) {
+					// Cannot possibly match, as the file sizes are so different
+					pm.update(1);
+					continue;
+				}
+
 				SimilarityIndex d = hash(dstEnt.newId.toObjectId());
 				int score = s.score(d);
 
@@ -259,6 +287,10 @@ private SimilarityIndex hash(ObjectId objectId) throws IOException {
 		return r;
 	}
 
+	private long size(ObjectId objectId) throws IOException {
+		return repo.openObject(objectId).getSize();
+	}
+
 	private static int score(long value) {
 		return (int) (value >>> SCORE_SHIFT);
 	}