1
2
3
4
5
6
7
8
9
10
11
12
13
14 package org.eclipse.jgit.transport;
15
16 import static org.junit.Assert.assertEquals;
17 import static org.junit.Assert.assertTrue;
18 import static org.junit.Assert.fail;
19
20 import java.io.ByteArrayInputStream;
21 import java.io.File;
22 import java.io.FileInputStream;
23 import java.io.IOException;
24 import java.io.InputStream;
25 import java.security.MessageDigest;
26 import java.text.MessageFormat;
27 import java.util.zip.Deflater;
28
29 import org.eclipse.jgit.errors.TooLargeObjectInPackException;
30 import org.eclipse.jgit.internal.JGitText;
31 import org.eclipse.jgit.internal.storage.file.ObjectDirectoryPackParser;
32 import org.eclipse.jgit.internal.storage.file.Pack;
33 import org.eclipse.jgit.junit.JGitTestUtil;
34 import org.eclipse.jgit.junit.RepositoryTestCase;
35 import org.eclipse.jgit.junit.TestRepository;
36 import org.eclipse.jgit.lib.Constants;
37 import org.eclipse.jgit.lib.NullProgressMonitor;
38 import org.eclipse.jgit.lib.ObjectId;
39 import org.eclipse.jgit.lib.ObjectInserter;
40 import org.eclipse.jgit.lib.Repository;
41 import org.eclipse.jgit.revwalk.RevBlob;
42 import org.eclipse.jgit.util.NB;
43 import org.eclipse.jgit.util.TemporaryBuffer;
44 import org.eclipse.jgit.util.io.UnionInputStream;
45 import org.junit.After;
46 import org.junit.Test;
47
48
49
50
51
52
53
54 public class PackParserTest extends RepositoryTestCase {
55
56
57
58
59
60 @Test
61 public void test1() throws IOException {
62 File packFile = JGitTestUtil.getTestResourceFile("pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.pack");
63 try (InputStream is = new FileInputStream(packFile)) {
64 ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(is);
65 p.parse(NullProgressMonitor.INSTANCE);
66 Pack pack = p.getPack();
67
68 assertTrue(pack.hasObject(ObjectId.fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904")));
69 assertTrue(pack.hasObject(ObjectId.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab")));
70 assertTrue(pack.hasObject(ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259")));
71 assertTrue(pack.hasObject(ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3")));
72 assertTrue(pack.hasObject(ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7")));
73 assertTrue(pack.hasObject(ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327")));
74 assertTrue(pack.hasObject(ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035")));
75 assertTrue(pack.hasObject(ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799")));
76 }
77 }
78
79
80
81
82
83
84
85 @Test
86 public void test2() throws IOException {
87 File packFile = JGitTestUtil.getTestResourceFile("pack-df2982f284bbabb6bdb59ee3fcc6eb0983e20371.pack");
88 try (InputStream is = new FileInputStream(packFile)) {
89 ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(is);
90 p.parse(NullProgressMonitor.INSTANCE);
91 Pack pack = p.getPack();
92
93 assertTrue(pack.hasObject(ObjectId.fromString("02ba32d3649e510002c21651936b7077aa75ffa9")));
94 assertTrue(pack.hasObject(ObjectId.fromString("0966a434eb1a025db6b71485ab63a3bfbea520b6")));
95 assertTrue(pack.hasObject(ObjectId.fromString("09efc7e59a839528ac7bda9fa020dc9101278680")));
96 assertTrue(pack.hasObject(ObjectId.fromString("0a3d7772488b6b106fb62813c4d6d627918d9181")));
97 assertTrue(pack.hasObject(ObjectId.fromString("1004d0d7ac26fbf63050a234c9b88a46075719d3")));
98 assertTrue(pack.hasObject(ObjectId.fromString("10da5895682013006950e7da534b705252b03be6")));
99 assertTrue(pack.hasObject(ObjectId.fromString("1203b03dc816ccbb67773f28b3c19318654b0bc8")));
100 assertTrue(pack.hasObject(ObjectId.fromString("15fae9e651043de0fd1deef588aa3fbf5a7a41c6")));
101 assertTrue(pack.hasObject(ObjectId.fromString("16f9ec009e5568c435f473ba3a1df732d49ce8c3")));
102 assertTrue(pack.hasObject(ObjectId.fromString("1fd7d579fb6ae3fe942dc09c2c783443d04cf21e")));
103 assertTrue(pack.hasObject(ObjectId.fromString("20a8ade77639491ea0bd667bf95de8abf3a434c8")));
104 assertTrue(pack.hasObject(ObjectId.fromString("2675188fd86978d5bc4d7211698b2118ae3bf658")));
105
106 }
107 }
108
109 @Test
110 public void testTinyThinPack() throws Exception {
111 RevBlob a;
112 try (TestRepository d = new TestRepository<Repository>(db)) {
113 db.incrementOpen();
114 a = d.blob("a");
115 }
116
117 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
118
119 packHeader(pack, 1);
120
121 pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
122 a.copyRawTo(pack);
123 deflate(pack, new byte[] { 0x1, 0x1, 0x1, 'b' });
124
125 digest(pack);
126
127 PackParser p = index(new ByteArrayInputStream(pack.toByteArray()));
128 p.setAllowThin(true);
129 p.parse(NullProgressMonitor.INSTANCE);
130 }
131
132 @Test
133 public void testPackWithDuplicateBlob() throws Exception {
134 final byte[] data = Constants.encode("0123456789abcdefg");
135 try (TestRepository<Repository> d = new TestRepository<>(db)) {
136 db.incrementOpen();
137 assertTrue(db.getObjectDatabase().has(d.blob(data)));
138 }
139
140 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
141 packHeader(pack, 1);
142 pack.write((Constants.OBJ_BLOB) << 4 | 0x80 | 1);
143 pack.write(1);
144 deflate(pack, data);
145 digest(pack);
146
147 PackParser p = index(new ByteArrayInputStream(pack.toByteArray()));
148 p.setAllowThin(false);
149 p.parse(NullProgressMonitor.INSTANCE);
150 }
151
152 @Test
153 public void testPackWithTrailingGarbage() throws Exception {
154 RevBlob a;
155 try (TestRepository d = new TestRepository<Repository>(db)) {
156 db.incrementOpen();
157 a = d.blob("a");
158 }
159
160 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
161 packHeader(pack, 1);
162 pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
163 a.copyRawTo(pack);
164 deflate(pack, new byte[] { 0x1, 0x1, 0x1, 'b' });
165 digest(pack);
166
167 PackParser p = index(new UnionInputStream(
168 new ByteArrayInputStream(pack.toByteArray()),
169 new ByteArrayInputStream(new byte[] { 0x7e })));
170 p.setAllowThin(true);
171 p.setCheckEofAfterPackFooter(true);
172 try {
173 p.parse(NullProgressMonitor.INSTANCE);
174 fail("Pack with trailing garbage was accepted");
175 } catch (IOException err) {
176 assertEquals(
177 MessageFormat.format(JGitText.get().expectedEOFReceived, "\\x7e"),
178 err.getMessage());
179 }
180 }
181
182 @Test
183 public void testMaxObjectSizeFullBlob() throws Exception {
184 final byte[] data = Constants.encode("0123456789");
185 try (TestRepository d = new TestRepository<Repository>(db)) {
186 db.incrementOpen();
187 d.blob(data);
188 }
189
190 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
191
192 packHeader(pack, 1);
193 pack.write((Constants.OBJ_BLOB) << 4 | 10);
194 deflate(pack, data);
195 digest(pack);
196
197 PackParser p = index(new ByteArrayInputStream(pack.toByteArray()));
198 p.setMaxObjectSizeLimit(11);
199 p.parse(NullProgressMonitor.INSTANCE);
200
201 p = index(new ByteArrayInputStream(pack.toByteArray()));
202 p.setMaxObjectSizeLimit(10);
203 p.parse(NullProgressMonitor.INSTANCE);
204
205 p = index(new ByteArrayInputStream(pack.toByteArray()));
206 p.setMaxObjectSizeLimit(9);
207 try {
208 p.parse(NullProgressMonitor.INSTANCE);
209 fail("PackParser should have failed");
210 } catch (TooLargeObjectInPackException e) {
211 assertTrue(e.getMessage().contains("10"));
212 assertTrue(e.getMessage().contains("9"));
213 }
214 }
215
216 @Test
217 public void testMaxObjectSizeDeltaBlock() throws Exception {
218 RevBlob a;
219 try (TestRepository d = new TestRepository<Repository>(db)) {
220 db.incrementOpen();
221 a = d.blob("a");
222 }
223
224 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
225
226 packHeader(pack, 1);
227 pack.write((Constants.OBJ_REF_DELTA) << 4 | 14);
228 a.copyRawTo(pack);
229 deflate(pack, new byte[] { 1, 11, 11, 'a', '0', '1', '2', '3', '4',
230 '5', '6', '7', '8', '9' });
231 digest(pack);
232
233 PackParser p = index(new ByteArrayInputStream(pack.toByteArray()));
234 p.setAllowThin(true);
235 p.setMaxObjectSizeLimit(14);
236 p.parse(NullProgressMonitor.INSTANCE);
237
238 p = index(new ByteArrayInputStream(pack.toByteArray()));
239 p.setAllowThin(true);
240 p.setMaxObjectSizeLimit(13);
241 try {
242 p.parse(NullProgressMonitor.INSTANCE);
243 fail("PackParser should have failed");
244 } catch (TooLargeObjectInPackException e) {
245 assertTrue(e.getMessage().contains("13"));
246 assertTrue(e.getMessage().contains("14"));
247 }
248 }
249
250 @Test
251 public void testMaxObjectSizeDeltaResultSize() throws Exception {
252 RevBlob a;
253 try (TestRepository d = new TestRepository<Repository>(db)) {
254 db.incrementOpen();
255 a = d.blob("0123456789");
256 }
257
258 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
259
260 packHeader(pack, 1);
261 pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
262 a.copyRawTo(pack);
263 deflate(pack, new byte[] { 10, 11, 1, 'a' });
264 digest(pack);
265
266 PackParser p = index(new ByteArrayInputStream(pack.toByteArray()));
267 p.setAllowThin(true);
268 p.setMaxObjectSizeLimit(11);
269 p.parse(NullProgressMonitor.INSTANCE);
270
271 p = index(new ByteArrayInputStream(pack.toByteArray()));
272 p.setAllowThin(true);
273 p.setMaxObjectSizeLimit(10);
274 try {
275 p.parse(NullProgressMonitor.INSTANCE);
276 fail("PackParser should have failed");
277 } catch (TooLargeObjectInPackException e) {
278 assertTrue(e.getMessage().contains("11"));
279 assertTrue(e.getMessage().contains("10"));
280 }
281 }
282
283 @Test
284 public void testNonMarkingInputStream() throws Exception {
285 RevBlob a;
286 try (TestRepository d = new TestRepository<Repository>(db)) {
287 db.incrementOpen();
288 a = d.blob("a");
289 }
290
291 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
292 packHeader(pack, 1);
293 pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
294 a.copyRawTo(pack);
295 deflate(pack, new byte[] { 0x1, 0x1, 0x1, 'b' });
296 digest(pack);
297
298 InputStream in = new ByteArrayInputStream(pack.toByteArray()) {
299 @Override
300 public boolean markSupported() {
301 return false;
302 }
303
304 @Override
305 public void mark(int maxlength) {
306 fail("Mark should not be called");
307 }
308 };
309
310 PackParser p = index(in);
311 p.setAllowThin(true);
312 p.setCheckEofAfterPackFooter(false);
313 p.setExpectDataAfterPackFooter(true);
314
315 try {
316 p.parse(NullProgressMonitor.INSTANCE);
317 fail("PackParser should have failed");
318 } catch (IOException e) {
319 assertEquals(e.getMessage(),
320 JGitText.get().inputStreamMustSupportMark);
321 }
322 }
323
324 @Test
325 public void testDataAfterPackFooterSingleRead() throws Exception {
326 RevBlob a;
327 try (TestRepository d = new TestRepository<Repository>(db)) {
328 db.incrementOpen();
329 a = d.blob("a");
330 }
331
332 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(32*1024);
333 packHeader(pack, 1);
334 pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
335 a.copyRawTo(pack);
336 deflate(pack, new byte[] { 0x1, 0x1, 0x1, 'b' });
337 digest(pack);
338
339 byte packData[] = pack.toByteArray();
340 byte streamData[] = new byte[packData.length + 1];
341 System.arraycopy(packData, 0, streamData, 0, packData.length);
342 streamData[packData.length] = 0x7e;
343
344 InputStream in = new ByteArrayInputStream(streamData);
345 PackParser p = index(in);
346 p.setAllowThin(true);
347 p.setCheckEofAfterPackFooter(false);
348 p.setExpectDataAfterPackFooter(true);
349
350 p.parse(NullProgressMonitor.INSTANCE);
351
352 assertEquals(0x7e, in.read());
353 }
354
355 @Test
356 public void testDataAfterPackFooterSplitObjectRead() throws Exception {
357 final byte[] data = Constants.encode("0123456789");
358
359
360 int objects = 900;
361 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(32 * 1024);
362 packHeader(pack, objects);
363
364 for (int i = 0; i < objects; i++) {
365 pack.write((Constants.OBJ_BLOB) << 4 | 10);
366 deflate(pack, data);
367 }
368 digest(pack);
369
370 byte packData[] = pack.toByteArray();
371 byte streamData[] = new byte[packData.length + 1];
372 System.arraycopy(packData, 0, streamData, 0, packData.length);
373 streamData[packData.length] = 0x7e;
374 InputStream in = new ByteArrayInputStream(streamData);
375 PackParser p = index(in);
376 p.setAllowThin(true);
377 p.setCheckEofAfterPackFooter(false);
378 p.setExpectDataAfterPackFooter(true);
379
380 p.parse(NullProgressMonitor.INSTANCE);
381
382 assertEquals(0x7e, in.read());
383 }
384
385 @Test
386 public void testDataAfterPackFooterSplitHeaderRead() throws Exception {
387 final byte[] data = Constants.encode("a");
388 RevBlob b;
389 try (TestRepository d = new TestRepository<Repository>(db)) {
390 db.incrementOpen();
391 b = d.blob(data);
392 }
393
394 int objects = 248;
395 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(32 * 1024);
396 packHeader(pack, objects + 1);
397 int offset = 13;
398 StringBuilder sb = new StringBuilder();
399 for (int i = 0; i < offset; i++)
400 sb.append(i);
401 offset = sb.toString().length();
402 int lenByte = (Constants.OBJ_BLOB) << 4 | (offset & 0x0F);
403 offset >>= 4;
404 if (offset > 0)
405 lenByte |= 1 << 7;
406 pack.write(lenByte);
407 while (offset > 0) {
408 lenByte = offset & 0x7F;
409 offset >>= 6;
410 if (offset > 0)
411 lenByte |= 1 << 7;
412 pack.write(lenByte);
413 }
414 deflate(pack, Constants.encode(sb.toString()));
415
416 for (int i = 0; i < objects; i++) {
417
418
419 pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
420 b.copyRawTo(pack);
421 deflate(pack, new byte[] { 0x1, 0x1, 0x1, 'b' });
422 }
423 digest(pack);
424
425 byte packData[] = pack.toByteArray();
426 byte streamData[] = new byte[packData.length + 1];
427 System.arraycopy(packData, 0, streamData, 0, packData.length);
428 streamData[packData.length] = 0x7e;
429 InputStream in = new ByteArrayInputStream(streamData);
430 PackParser p = index(in);
431 p.setAllowThin(true);
432 p.setCheckEofAfterPackFooter(false);
433 p.setExpectDataAfterPackFooter(true);
434
435 p.parse(NullProgressMonitor.INSTANCE);
436
437 assertEquals(0x7e, in.read());
438 }
439
440 private static void packHeader(TemporaryBuffer.Heap tinyPack, int cnt)
441 throws IOException {
442 final byte[] hdr = new byte[8];
443 NB.encodeInt32(hdr, 0, 2);
444 NB.encodeInt32(hdr, 4, cnt);
445
446 tinyPack.write(Constants.PACK_SIGNATURE);
447 tinyPack.write(hdr, 0, 8);
448 }
449
450 private static void deflate(TemporaryBuffer.Heap tinyPack,
451 final byte[] content)
452 throws IOException {
453 final Deflater deflater = new Deflater();
454 final byte[] buf = new byte[128];
455 deflater.setInput(content, 0, content.length);
456 deflater.finish();
457 do {
458 final int n = deflater.deflate(buf, 0, buf.length);
459 if (n > 0)
460 tinyPack.write(buf, 0, n);
461 } while (!deflater.finished());
462 }
463
464 private static void digest(TemporaryBuffer.Heap buf) throws IOException {
465 MessageDigest md = Constants.newMessageDigest();
466 md.update(buf.toByteArray());
467 buf.write(md.digest());
468 }
469
470 private ObjectInserter inserter;
471
472 @After
473 public void release() {
474 if (inserter != null) {
475 inserter.close();
476 }
477 }
478
479 private PackParser index(InputStream in) throws IOException {
480 if (inserter == null)
481 inserter = db.newObjectInserter();
482 return inserter.newPackParser(in);
483 }
484 }