| /* |
| * Copyright (C) 2011, Google Inc. |
| * and other copyright owners as documented in the project's IP log. |
| * |
| * This program and the accompanying materials are made available |
| * under the terms of the Eclipse Distribution License v1.0 which |
| * accompanies this distribution, is reproduced below, and is |
| * available at http://www.eclipse.org/org/documents/edl-v10.php |
| * |
| * All rights reserved. |
| * |
| * Redistribution and use in source and binary forms, with or |
| * without modification, are permitted provided that the following |
| * conditions are met: |
| * |
| * - Redistributions of source code must retain the above copyright |
| * notice, this list of conditions and the following disclaimer. |
| * |
| * - Redistributions in binary form must reproduce the above |
| * copyright notice, this list of conditions and the following |
| * disclaimer in the documentation and/or other materials provided |
| * with the distribution. |
| * |
| * - Neither the name of the Eclipse Foundation, Inc. nor the |
| * names of its contributors may be used to endorse or promote |
| * products derived from this software without specific prior |
| * written permission. |
| * |
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
| * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
| * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
| * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
| * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
| * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
| * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
| * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
| * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| */ |
| |
| package org.eclipse.jgit.storage.dfs; |
| |
| import java.io.EOFException; |
| import java.io.IOException; |
| import java.io.InputStream; |
| import java.io.OutputStream; |
| import java.security.MessageDigest; |
| import java.util.Collections; |
| import java.util.List; |
| import java.util.zip.CRC32; |
| import java.util.zip.Deflater; |
| import java.util.zip.DeflaterOutputStream; |
| |
| import org.eclipse.jgit.lib.Constants; |
| import org.eclipse.jgit.lib.ObjectId; |
| import org.eclipse.jgit.lib.ObjectIdOwnerMap; |
| import org.eclipse.jgit.lib.ObjectInserter; |
| import org.eclipse.jgit.storage.file.PackIndex; |
| import org.eclipse.jgit.storage.file.PackIndexWriter; |
| import org.eclipse.jgit.transport.PackedObjectInfo; |
| import org.eclipse.jgit.util.BlockList; |
| import org.eclipse.jgit.util.IO; |
| import org.eclipse.jgit.util.NB; |
| import org.eclipse.jgit.util.TemporaryBuffer; |
| import org.eclipse.jgit.util.io.CountingOutputStream; |
| |
| /** Inserts objects into the DFS. */ |
| public class DfsInserter extends ObjectInserter { |
| /** Always produce version 2 indexes, to get CRC data. */ |
| private static final int INDEX_VERSION = 2; |
| |
| private final DfsObjDatabase db; |
| |
| private List<PackedObjectInfo> objectList; |
| private ObjectIdOwnerMap<PackedObjectInfo> objectMap; |
| |
| private DfsBlockCache cache; |
| private DfsPackKey packKey; |
| private DfsPackDescription packDsc; |
| private PackStream packOut; |
| private boolean rollback; |
| |
| /** |
| * Initialize a new inserter. |
| * |
| * @param db |
| * database the inserter writes to. |
| */ |
| protected DfsInserter(DfsObjDatabase db) { |
| this.db = db; |
| } |
| |
| @Override |
| public DfsPackParser newPackParser(InputStream in) throws IOException { |
| return new DfsPackParser(db, this, in); |
| } |
| |
| @Override |
| public ObjectId insert(int type, byte[] data, int off, int len) |
| throws IOException { |
| ObjectId id = idFor(type, data, off, len); |
| if (objectMap != null && objectMap.contains(id)) |
| return id; |
| if (db.has(id)) |
| return id; |
| |
| long offset = beginObject(type, len); |
| packOut.compress.write(data, off, len); |
| packOut.compress.finish(); |
| return endObject(id, offset); |
| } |
| |
| @Override |
| public ObjectId insert(int type, long len, InputStream in) |
| throws IOException { |
| byte[] buf = buffer(); |
| if (len <= buf.length) { |
| IO.readFully(in, buf, 0, (int) len); |
| return insert(type, buf, 0, (int) len); |
| } |
| |
| long offset = beginObject(type, len); |
| MessageDigest md = digest(); |
| md.update(Constants.encodedTypeString(type)); |
| md.update((byte) ' '); |
| md.update(Constants.encodeASCII(len)); |
| md.update((byte) 0); |
| |
| while (0 < len) { |
| int n = in.read(buf, 0, (int) Math.min(buf.length, len)); |
| if (n <= 0) |
| throw new EOFException(); |
| md.update(buf, 0, n); |
| packOut.compress.write(buf, 0, n); |
| len -= n; |
| } |
| packOut.compress.finish(); |
| return endObject(ObjectId.fromRaw(md.digest()), offset); |
| } |
| |
| @Override |
| public void flush() throws IOException { |
| if (packDsc == null) |
| return; |
| |
| if (packOut == null) |
| throw new IOException(); |
| |
| byte[] packHash = packOut.writePackFooter(); |
| packDsc.setPackSize(packOut.getCount()); |
| packOut.close(); |
| packOut = null; |
| |
| sortObjectsById(); |
| |
| PackIndex index = writePackIndex(packDsc, packHash, objectList); |
| db.commitPack(Collections.singletonList(packDsc), null); |
| rollback = false; |
| |
| DfsPackFile p = cache.getOrCreate(packDsc, packKey); |
| if (index != null) |
| p.setPackIndex(index); |
| db.addPack(p); |
| clear(); |
| } |
| |
| @Override |
| public void release() { |
| if (packOut != null) { |
| try { |
| packOut.close(); |
| } catch (IOException err) { |
| // Ignore a close failure, the pack should be removed. |
| } finally { |
| packOut = null; |
| } |
| } |
| if (rollback && packDsc != null) { |
| try { |
| db.rollbackPack(Collections.singletonList(packDsc)); |
| } finally { |
| packDsc = null; |
| rollback = false; |
| } |
| } |
| clear(); |
| } |
| |
| private void clear() { |
| objectList = null; |
| objectMap = null; |
| packKey = null; |
| packDsc = null; |
| } |
| |
| private long beginObject(int type, long len) throws IOException { |
| if (packOut == null) |
| beginPack(); |
| long offset = packOut.getCount(); |
| packOut.beginObject(type, len); |
| return offset; |
| } |
| |
| private ObjectId endObject(ObjectId id, long offset) { |
| PackedObjectInfo obj = new PackedObjectInfo(id); |
| obj.setOffset(offset); |
| obj.setCRC((int) packOut.crc32.getValue()); |
| objectList.add(obj); |
| objectMap.addIfAbsent(obj); |
| return id; |
| } |
| |
| private void beginPack() throws IOException { |
| objectList = new BlockList<PackedObjectInfo>(); |
| objectMap = new ObjectIdOwnerMap<PackedObjectInfo>(); |
| cache = DfsBlockCache.getInstance(); |
| |
| rollback = true; |
| packDsc = db.newPack(DfsObjDatabase.PackSource.INSERT); |
| packOut = new PackStream(db.writePackFile(packDsc)); |
| packKey = new DfsPackKey(); |
| |
| // Write the header as though it were a single object pack. |
| byte[] buf = packOut.hdrBuf; |
| System.arraycopy(Constants.PACK_SIGNATURE, 0, buf, 0, 4); |
| NB.encodeInt32(buf, 4, 2); // Always use pack version 2. |
| NB.encodeInt32(buf, 8, 1); // Always assume 1 object. |
| packOut.write(buf, 0, 12); |
| } |
| |
| private void sortObjectsById() { |
| Collections.sort(objectList); |
| } |
| |
| PackIndex writePackIndex(DfsPackDescription pack, byte[] packHash, |
| List<PackedObjectInfo> list) throws IOException { |
| pack.setObjectCount(list.size()); |
| |
| // If there are less than 58,000 objects, the entire index fits in under |
| // 2 MiB. Callers will probably need the index immediately, so buffer |
| // the index in process and load from the buffer. |
| TemporaryBuffer.Heap buf = null; |
| PackIndex packIndex = null; |
| if (list.size() <= 58000) { |
| buf = new TemporaryBuffer.Heap(2 << 20); |
| index(buf, packHash, list); |
| packIndex = PackIndex.read(buf.openInputStream()); |
| } |
| |
| DfsOutputStream os = db.writePackIndex(pack); |
| try { |
| CountingOutputStream cnt = new CountingOutputStream(os); |
| if (buf != null) |
| buf.writeTo(cnt, null); |
| else |
| index(cnt, packHash, list); |
| pack.setIndexSize(cnt.getCount()); |
| } finally { |
| os.close(); |
| } |
| return packIndex; |
| } |
| |
| private static void index(OutputStream out, byte[] packHash, |
| List<PackedObjectInfo> list) throws IOException { |
| PackIndexWriter.createVersion(out, INDEX_VERSION).write(list, packHash); |
| } |
| |
| private class PackStream extends OutputStream { |
| private final DfsOutputStream out; |
| private final MessageDigest md; |
| private final byte[] hdrBuf; |
| private final Deflater deflater; |
| private final int blockSize; |
| |
| private long currPos; // Position of currBuf[0] in the output stream. |
| private int currPtr; // Number of bytes in currBuf. |
| private byte[] currBuf; |
| |
| final CRC32 crc32; |
| final DeflaterOutputStream compress; |
| |
| PackStream(DfsOutputStream out) { |
| this.out = out; |
| |
| hdrBuf = new byte[32]; |
| md = Constants.newMessageDigest(); |
| crc32 = new CRC32(); |
| deflater = new Deflater(Deflater.BEST_COMPRESSION); |
| compress = new DeflaterOutputStream(this, deflater, 8192); |
| |
| int size = out.blockSize(); |
| if (size <= 0) |
| size = cache.getBlockSize(); |
| else if (size < cache.getBlockSize()) |
| size = (cache.getBlockSize() / size) * size; |
| blockSize = size; |
| currBuf = new byte[blockSize]; |
| } |
| |
| long getCount() { |
| return currPos + currPtr; |
| } |
| |
| void beginObject(int objectType, long length) throws IOException { |
| crc32.reset(); |
| deflater.reset(); |
| write(hdrBuf, 0, encodeTypeSize(objectType, length)); |
| } |
| |
| private int encodeTypeSize(int type, long rawLength) { |
| long nextLength = rawLength >>> 4; |
| hdrBuf[0] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (type << 4) | (rawLength & 0x0F)); |
| rawLength = nextLength; |
| int n = 1; |
| while (rawLength > 0) { |
| nextLength >>>= 7; |
| hdrBuf[n++] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (rawLength & 0x7F)); |
| rawLength = nextLength; |
| } |
| return n; |
| } |
| |
| @Override |
| public void write(final int b) throws IOException { |
| hdrBuf[0] = (byte) b; |
| write(hdrBuf, 0, 1); |
| } |
| |
| @Override |
| public void write(byte[] data, int off, int len) throws IOException { |
| crc32.update(data, off, len); |
| md.update(data, off, len); |
| writeNoHash(data, off, len); |
| } |
| |
| private void writeNoHash(byte[] data, int off, int len) |
| throws IOException { |
| while (0 < len) { |
| int n = Math.min(len, currBuf.length - currPtr); |
| if (n == 0) { |
| flushBlock(); |
| currBuf = new byte[blockSize]; |
| continue; |
| } |
| |
| System.arraycopy(data, off, currBuf, currPtr, n); |
| off += n; |
| len -= n; |
| currPtr += n; |
| } |
| } |
| |
| private void flushBlock() throws IOException { |
| out.write(currBuf, 0, currPtr); |
| |
| byte[] buf; |
| if (currPtr == currBuf.length) |
| buf = currBuf; |
| else |
| buf = copyOf(currBuf, 0, currPtr); |
| cache.put(new DfsBlock(packKey, currPos, buf)); |
| |
| currPos += currPtr; |
| currPtr = 0; |
| currBuf = null; |
| } |
| |
| private byte[] copyOf(byte[] src, int ptr, int cnt) { |
| byte[] dst = new byte[cnt]; |
| System.arraycopy(src, ptr, dst, 0, cnt); |
| return dst; |
| } |
| |
| byte[] writePackFooter() throws IOException { |
| byte[] packHash = md.digest(); |
| writeNoHash(packHash, 0, packHash.length); |
| if (currPtr != 0) |
| flushBlock(); |
| return packHash; |
| } |
| |
| @Override |
| public void close() throws IOException { |
| deflater.end(); |
| out.close(); |
| } |
| } |
| } |