| /* |
| * Copyright (C) 2008-2010, Google Inc. |
| * Copyright (C) 2008, Marek Zawirski <marek.zawirski@gmail.com> |
| * and other copyright owners as documented in the project's IP log. |
| * |
| * This program and the accompanying materials are made available |
| * under the terms of the Eclipse Distribution License v1.0 which |
| * accompanies this distribution, is reproduced below, and is |
| * available at http://www.eclipse.org/org/documents/edl-v10.php |
| * |
| * All rights reserved. |
| * |
| * Redistribution and use in source and binary forms, with or |
| * without modification, are permitted provided that the following |
| * conditions are met: |
| * |
| * - Redistributions of source code must retain the above copyright |
| * notice, this list of conditions and the following disclaimer. |
| * |
| * - Redistributions in binary form must reproduce the above |
| * copyright notice, this list of conditions and the following |
| * disclaimer in the documentation and/or other materials provided |
| * with the distribution. |
| * |
| * - Neither the name of the Eclipse Foundation, Inc. nor the |
| * names of its contributors may be used to endorse or promote |
| * products derived from this software without specific prior |
| * written permission. |
| * |
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
| * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
| * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
| * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
| * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
| * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
| * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
| * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
| * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| */ |
| |
| package org.eclipse.jgit.internal.storage.pack; |
| |
| import static java.util.Objects.requireNonNull; |
| import static org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation.PACK_DELTA; |
| import static org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation.PACK_WHOLE; |
| import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH; |
| import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; |
| import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT; |
| import static org.eclipse.jgit.lib.Constants.OBJ_TAG; |
| import static org.eclipse.jgit.lib.Constants.OBJ_TREE; |
| |
| import java.io.IOException; |
| import java.io.OutputStream; |
| import java.lang.ref.WeakReference; |
| import java.security.MessageDigest; |
| import java.text.MessageFormat; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.Collection; |
| import java.util.Collections; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.Iterator; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.NoSuchElementException; |
| import java.util.Set; |
| import java.util.concurrent.ConcurrentHashMap; |
| import java.util.concurrent.ExecutionException; |
| import java.util.concurrent.Executor; |
| import java.util.concurrent.ExecutorService; |
| import java.util.concurrent.Executors; |
| import java.util.concurrent.Future; |
| import java.util.concurrent.TimeUnit; |
| import java.util.zip.CRC32; |
| import java.util.zip.CheckedOutputStream; |
| import java.util.zip.Deflater; |
| import java.util.zip.DeflaterOutputStream; |
| |
| import org.eclipse.jgit.annotations.NonNull; |
| import org.eclipse.jgit.annotations.Nullable; |
| import org.eclipse.jgit.errors.CorruptObjectException; |
| import org.eclipse.jgit.errors.IncorrectObjectTypeException; |
| import org.eclipse.jgit.errors.LargeObjectException; |
| import org.eclipse.jgit.errors.MissingObjectException; |
| import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException; |
| import org.eclipse.jgit.internal.JGitText; |
| import org.eclipse.jgit.internal.storage.file.PackBitmapIndexBuilder; |
| import org.eclipse.jgit.internal.storage.file.PackBitmapIndexWriterV1; |
| import org.eclipse.jgit.internal.storage.file.PackIndexWriter; |
| import org.eclipse.jgit.lib.AnyObjectId; |
| import org.eclipse.jgit.lib.AsyncObjectSizeQueue; |
| import org.eclipse.jgit.lib.BatchingProgressMonitor; |
| import org.eclipse.jgit.lib.BitmapIndex; |
| import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder; |
| import org.eclipse.jgit.lib.BitmapObject; |
| import org.eclipse.jgit.lib.Constants; |
| import org.eclipse.jgit.lib.NullProgressMonitor; |
| import org.eclipse.jgit.lib.ObjectId; |
| import org.eclipse.jgit.lib.ObjectIdOwnerMap; |
| import org.eclipse.jgit.lib.ObjectIdSet; |
| import org.eclipse.jgit.lib.ObjectLoader; |
| import org.eclipse.jgit.lib.ObjectReader; |
| import org.eclipse.jgit.lib.ProgressMonitor; |
| import org.eclipse.jgit.lib.Repository; |
| import org.eclipse.jgit.lib.ThreadSafeProgressMonitor; |
| import org.eclipse.jgit.revwalk.AsyncRevObjectQueue; |
| import org.eclipse.jgit.revwalk.BitmapWalker; |
| import org.eclipse.jgit.revwalk.DepthWalk; |
| import org.eclipse.jgit.revwalk.ObjectWalk; |
| import org.eclipse.jgit.revwalk.RevCommit; |
| import org.eclipse.jgit.revwalk.RevFlag; |
| import org.eclipse.jgit.revwalk.RevObject; |
| import org.eclipse.jgit.revwalk.RevSort; |
| import org.eclipse.jgit.revwalk.RevTag; |
| import org.eclipse.jgit.revwalk.RevTree; |
| import org.eclipse.jgit.storage.pack.PackConfig; |
| import org.eclipse.jgit.storage.pack.PackStatistics; |
| import org.eclipse.jgit.transport.FilterSpec; |
| import org.eclipse.jgit.transport.ObjectCountCallback; |
| import org.eclipse.jgit.transport.PacketLineOut; |
| import org.eclipse.jgit.transport.WriteAbortedException; |
| import org.eclipse.jgit.util.BlockList; |
| import org.eclipse.jgit.util.TemporaryBuffer; |
| |
| /** |
| * <p> |
| * PackWriter class is responsible for generating pack files from specified set |
| * of objects from repository. This implementation produce pack files in format |
| * version 2. |
| * </p> |
| * <p> |
| * Source of objects may be specified in two ways: |
| * <ul> |
| * <li>(usually) by providing sets of interesting and uninteresting objects in |
| * repository - all interesting objects and their ancestors except uninteresting |
| * objects and their ancestors will be included in pack, or</li> |
| * <li>by providing iterator of {@link org.eclipse.jgit.revwalk.RevObject} |
| * specifying exact list and order of objects in pack</li> |
| * </ul> |
| * <p> |
| * Typical usage consists of creating an instance, configuring options, |
| * preparing the list of objects by calling {@link #preparePack(Iterator)} or |
| * {@link #preparePack(ProgressMonitor, Set, Set)}, and streaming with |
| * {@link #writePack(ProgressMonitor, ProgressMonitor, OutputStream)}. If the |
| * pack is being stored as a file the matching index can be written out after |
| * writing the pack by {@link #writeIndex(OutputStream)}. An optional bitmap |
| * index can be made by calling {@link #prepareBitmapIndex(ProgressMonitor)} |
| * followed by {@link #writeBitmapIndex(OutputStream)}. |
| * </p> |
| * <p> |
| * Class provide set of configurable options and |
| * {@link org.eclipse.jgit.lib.ProgressMonitor} support, as operations may take |
| * a long time for big repositories. Deltas searching algorithm is <b>NOT |
| * IMPLEMENTED</b> yet - this implementation relies only on deltas and objects |
| * reuse. |
| * </p> |
| * <p> |
| * This class is not thread safe. It is intended to be used in one thread as a |
| * single pass to produce one pack. Invoking methods multiple times or out of |
| * order is not supported as internal data structures are destroyed during |
| * certain phases to save memory when packing large repositories. |
| * </p> |
| */ |
| public class PackWriter implements AutoCloseable { |
| private static final int PACK_VERSION_GENERATED = 2; |
| |
| /** Empty set of objects for {@code preparePack()}. */ |
| public static final Set<ObjectId> NONE = Collections.emptySet(); |
| |
| private static final Map<WeakReference<PackWriter>, Boolean> instances = |
| new ConcurrentHashMap<>(); |
| |
| private static final Iterable<PackWriter> instancesIterable = new Iterable<PackWriter>() { |
| @Override |
| public Iterator<PackWriter> iterator() { |
| return new Iterator<PackWriter>() { |
| private final Iterator<WeakReference<PackWriter>> it = |
| instances.keySet().iterator(); |
| private PackWriter next; |
| |
| @Override |
| public boolean hasNext() { |
| if (next != null) |
| return true; |
| while (it.hasNext()) { |
| WeakReference<PackWriter> ref = it.next(); |
| next = ref.get(); |
| if (next != null) |
| return true; |
| it.remove(); |
| } |
| return false; |
| } |
| |
| @Override |
| public PackWriter next() { |
| if (hasNext()) { |
| PackWriter result = next; |
| next = null; |
| return result; |
| } |
| throw new NoSuchElementException(); |
| } |
| |
| @Override |
| public void remove() { |
| throw new UnsupportedOperationException(); |
| } |
| }; |
| } |
| }; |
| |
| /** |
| * Get all allocated, non-released PackWriters instances. |
| * |
| * @return all allocated, non-released PackWriters instances. |
| */ |
| public static Iterable<PackWriter> getInstances() { |
| return instancesIterable; |
| } |
| |
| @SuppressWarnings("unchecked") |
| BlockList<ObjectToPack>[] objectsLists = new BlockList[OBJ_TAG + 1]; |
| { |
| objectsLists[OBJ_COMMIT] = new BlockList<>(); |
| objectsLists[OBJ_TREE] = new BlockList<>(); |
| objectsLists[OBJ_BLOB] = new BlockList<>(); |
| objectsLists[OBJ_TAG] = new BlockList<>(); |
| } |
| |
| private ObjectIdOwnerMap<ObjectToPack> objectsMap = new ObjectIdOwnerMap<>(); |
| |
| // edge objects for thin packs |
| private List<ObjectToPack> edgeObjects = new BlockList<>(); |
| |
| // Objects the client is known to have already. |
| private BitmapBuilder haveObjects; |
| |
| private List<CachedPack> cachedPacks = new ArrayList<>(2); |
| |
| private Set<ObjectId> tagTargets = NONE; |
| |
| private Set<? extends ObjectId> excludeFromBitmapSelection = NONE; |
| |
| private ObjectIdSet[] excludeInPacks; |
| |
| private ObjectIdSet excludeInPackLast; |
| |
| private Deflater myDeflater; |
| |
| private final ObjectReader reader; |
| |
| /** {@link #reader} recast to the reuse interface, if it supports it. */ |
| private final ObjectReuseAsIs reuseSupport; |
| |
| final PackConfig config; |
| |
| private final PackStatistics.Accumulator stats; |
| |
| private final MutableState state; |
| |
| private final WeakReference<PackWriter> selfRef; |
| |
| private PackStatistics.ObjectType.Accumulator typeStats; |
| |
| private List<ObjectToPack> sortedByName; |
| |
| private byte[] packcsum; |
| |
| private boolean deltaBaseAsOffset; |
| |
| private boolean reuseDeltas; |
| |
| private boolean reuseDeltaCommits; |
| |
| private boolean reuseValidate; |
| |
| private boolean thin; |
| |
| private boolean useCachedPacks; |
| |
| private boolean useBitmaps; |
| |
| private boolean ignoreMissingUninteresting = true; |
| |
| private boolean pruneCurrentObjectList; |
| |
| private boolean shallowPack; |
| |
| private boolean canBuildBitmaps; |
| |
| private boolean indexDisabled; |
| |
| private int depth; |
| |
| private Collection<? extends ObjectId> unshallowObjects; |
| |
| private PackBitmapIndexBuilder writeBitmaps; |
| |
| private CRC32 crc32; |
| |
| private ObjectCountCallback callback; |
| |
| private FilterSpec filterSpec = FilterSpec.NO_FILTER; |
| |
| private PackfileUriConfig packfileUriConfig; |
| |
| /** |
| * Create writer for specified repository. |
| * <p> |
| * Objects for packing are specified in {@link #preparePack(Iterator)} or |
| * {@link #preparePack(ProgressMonitor, Set, Set)}. |
| * |
| * @param repo |
| * repository where objects are stored. |
| */ |
| public PackWriter(Repository repo) { |
| this(repo, repo.newObjectReader()); |
| } |
| |
| /** |
| * Create a writer to load objects from the specified reader. |
| * <p> |
| * Objects for packing are specified in {@link #preparePack(Iterator)} or |
| * {@link #preparePack(ProgressMonitor, Set, Set)}. |
| * |
| * @param reader |
| * reader to read from the repository with. |
| */ |
| public PackWriter(ObjectReader reader) { |
| this(new PackConfig(), reader); |
| } |
| |
| /** |
| * Create writer for specified repository. |
| * <p> |
| * Objects for packing are specified in {@link #preparePack(Iterator)} or |
| * {@link #preparePack(ProgressMonitor, Set, Set)}. |
| * |
| * @param repo |
| * repository where objects are stored. |
| * @param reader |
| * reader to read from the repository with. |
| */ |
| public PackWriter(Repository repo, ObjectReader reader) { |
| this(new PackConfig(repo), reader); |
| } |
| |
| /** |
| * Create writer with a specified configuration. |
| * <p> |
| * Objects for packing are specified in {@link #preparePack(Iterator)} or |
| * {@link #preparePack(ProgressMonitor, Set, Set)}. |
| * |
| * @param config |
| * configuration for the pack writer. |
| * @param reader |
| * reader to read from the repository with. |
| */ |
| public PackWriter(PackConfig config, ObjectReader reader) { |
| this(config, reader, null); |
| } |
| |
| /** |
| * Create writer with a specified configuration. |
| * <p> |
| * Objects for packing are specified in {@link #preparePack(Iterator)} or |
| * {@link #preparePack(ProgressMonitor, Set, Set)}. |
| * |
| * @param config |
| * configuration for the pack writer. |
| * @param reader |
| * reader to read from the repository with. |
| * @param statsAccumulator |
| * accumulator for statics |
| */ |
| public PackWriter(PackConfig config, final ObjectReader reader, |
| @Nullable PackStatistics.Accumulator statsAccumulator) { |
| this.config = config; |
| this.reader = reader; |
| if (reader instanceof ObjectReuseAsIs) |
| reuseSupport = ((ObjectReuseAsIs) reader); |
| else |
| reuseSupport = null; |
| |
| deltaBaseAsOffset = config.isDeltaBaseAsOffset(); |
| reuseDeltas = config.isReuseDeltas(); |
| reuseValidate = true; // be paranoid by default |
| stats = statsAccumulator != null ? statsAccumulator |
| : new PackStatistics.Accumulator(); |
| state = new MutableState(); |
| selfRef = new WeakReference<>(this); |
| instances.put(selfRef, Boolean.TRUE); |
| } |
| |
| /** |
| * Set the {@code ObjectCountCallback}. |
| * <p> |
| * It should be set before calling |
| * {@link #writePack(ProgressMonitor, ProgressMonitor, OutputStream)}. |
| * |
| * @param callback |
| * the callback to set |
| * @return this object for chaining. |
| */ |
| public PackWriter setObjectCountCallback(ObjectCountCallback callback) { |
| this.callback = callback; |
| return this; |
| } |
| |
| /** |
| * Records the set of shallow commits in the client. |
| * |
| * @param clientShallowCommits |
| * the shallow commits in the client |
| */ |
| public void setClientShallowCommits(Set<ObjectId> clientShallowCommits) { |
| stats.clientShallowCommits = Collections |
| .unmodifiableSet(new HashSet<>(clientShallowCommits)); |
| } |
| |
| /** |
| * Check whether writer can store delta base as an offset (new style |
| * reducing pack size) or should store it as an object id (legacy style, |
| * compatible with old readers). |
| * |
| * Default setting: {@value PackConfig#DEFAULT_DELTA_BASE_AS_OFFSET} |
| * |
| * @return true if delta base is stored as an offset; false if it is stored |
| * as an object id. |
| */ |
| public boolean isDeltaBaseAsOffset() { |
| return deltaBaseAsOffset; |
| } |
| |
| /** |
| * Set writer delta base format. Delta base can be written as an offset in a |
| * pack file (new approach reducing file size) or as an object id (legacy |
| * approach, compatible with old readers). |
| * |
| * Default setting: {@value PackConfig#DEFAULT_DELTA_BASE_AS_OFFSET} |
| * |
| * @param deltaBaseAsOffset |
| * boolean indicating whether delta base can be stored as an |
| * offset. |
| */ |
| public void setDeltaBaseAsOffset(boolean deltaBaseAsOffset) { |
| this.deltaBaseAsOffset = deltaBaseAsOffset; |
| } |
| |
| /** |
| * Check if the writer will reuse commits that are already stored as deltas. |
| * |
| * @return true if the writer would reuse commits stored as deltas, assuming |
| * delta reuse is already enabled. |
| */ |
| public boolean isReuseDeltaCommits() { |
| return reuseDeltaCommits; |
| } |
| |
| /** |
| * Set the writer to reuse existing delta versions of commits. |
| * |
| * @param reuse |
| * if true, the writer will reuse any commits stored as deltas. |
| * By default the writer does not reuse delta commits. |
| */ |
| public void setReuseDeltaCommits(boolean reuse) { |
| reuseDeltaCommits = reuse; |
| } |
| |
| /** |
| * Check if the writer validates objects before copying them. |
| * |
| * @return true if validation is enabled; false if the reader will handle |
| * object validation as a side-effect of it consuming the output. |
| */ |
| public boolean isReuseValidatingObjects() { |
| return reuseValidate; |
| } |
| |
| /** |
| * Enable (or disable) object validation during packing. |
| * |
| * @param validate |
| * if true the pack writer will validate an object before it is |
| * put into the output. This additional validation work may be |
| * necessary to avoid propagating corruption from one local pack |
| * file to another local pack file. |
| */ |
| public void setReuseValidatingObjects(boolean validate) { |
| reuseValidate = validate; |
| } |
| |
| /** |
| * Whether this writer is producing a thin pack. |
| * |
| * @return true if this writer is producing a thin pack. |
| */ |
| public boolean isThin() { |
| return thin; |
| } |
| |
| /** |
| * Whether writer may pack objects with delta base object not within set of |
| * objects to pack |
| * |
| * @param packthin |
| * a boolean indicating whether writer may pack objects with |
| * delta base object not within set of objects to pack, but |
| * belonging to party repository (uninteresting/boundary) as |
| * determined by set; this kind of pack is used only for |
| * transport; true - to produce thin pack, false - otherwise. |
| */ |
| public void setThin(boolean packthin) { |
| thin = packthin; |
| } |
| |
| /** |
| * Whether to reuse cached packs. |
| * |
| * @return {@code true} to reuse cached packs. If true index creation isn't |
| * available. |
| */ |
| public boolean isUseCachedPacks() { |
| return useCachedPacks; |
| } |
| |
| /** |
| * Whether to use cached packs |
| * |
| * @param useCached |
| * if set to {@code true} and a cached pack is present, it will |
| * be appended onto the end of a thin-pack, reducing the amount |
| * of working set space and CPU used by PackWriter. Enabling this |
| * feature prevents PackWriter from creating an index for the |
| * newly created pack, so its only suitable for writing to a |
| * network client, where the client will make the index. |
| */ |
| public void setUseCachedPacks(boolean useCached) { |
| useCachedPacks = useCached; |
| } |
| |
| /** |
| * Whether to use bitmaps |
| * |
| * @return {@code true} to use bitmaps for ObjectWalks, if available. |
| */ |
| public boolean isUseBitmaps() { |
| return useBitmaps; |
| } |
| |
| /** |
| * Whether to use bitmaps |
| * |
| * @param useBitmaps |
| * if set to true, bitmaps will be used when preparing a pack. |
| */ |
| public void setUseBitmaps(boolean useBitmaps) { |
| this.useBitmaps = useBitmaps; |
| } |
| |
| /** |
| * Whether the index file cannot be created by this PackWriter. |
| * |
| * @return {@code true} if the index file cannot be created by this |
| * PackWriter. |
| */ |
| public boolean isIndexDisabled() { |
| return indexDisabled || !cachedPacks.isEmpty(); |
| } |
| |
| /** |
| * Whether to disable creation of the index file. |
| * |
| * @param noIndex |
| * {@code true} to disable creation of the index file. |
| */ |
| public void setIndexDisabled(boolean noIndex) { |
| this.indexDisabled = noIndex; |
| } |
| |
| /** |
| * Whether to ignore missing uninteresting objects |
| * |
| * @return {@code true} to ignore objects that are uninteresting and also |
| * not found on local disk; false to throw a |
| * {@link org.eclipse.jgit.errors.MissingObjectException} out of |
| * {@link #preparePack(ProgressMonitor, Set, Set)} if an |
| * uninteresting object is not in the source repository. By default, |
| * true, permitting gracefully ignoring of uninteresting objects. |
| */ |
| public boolean isIgnoreMissingUninteresting() { |
| return ignoreMissingUninteresting; |
| } |
| |
| /** |
| * Whether writer should ignore non existing uninteresting objects |
| * |
| * @param ignore |
| * {@code true} if writer should ignore non existing |
| * uninteresting objects during construction set of objects to |
| * pack; false otherwise - non existing uninteresting objects may |
| * cause {@link org.eclipse.jgit.errors.MissingObjectException} |
| */ |
| public void setIgnoreMissingUninteresting(boolean ignore) { |
| ignoreMissingUninteresting = ignore; |
| } |
| |
| /** |
| * Set the tag targets that should be hoisted earlier during packing. |
| * <p> |
| * Callers may put objects into this set before invoking any of the |
| * preparePack methods to influence where an annotated tag's target is |
| * stored within the resulting pack. Typically these will be clustered |
| * together, and hoisted earlier in the file even if they are ancient |
| * revisions, allowing readers to find tag targets with better locality. |
| * |
| * @param objects |
| * objects that annotated tags point at. |
| */ |
| public void setTagTargets(Set<ObjectId> objects) { |
| tagTargets = objects; |
| } |
| |
| /** |
| * Configure this pack for a shallow clone. |
| * |
| * @param depth |
| * maximum depth of history to return. 1 means return only the |
| * "wants". |
| * @param unshallow |
| * objects which used to be shallow on the client, but are being |
| * extended as part of this fetch |
| */ |
| public void setShallowPack(int depth, |
| Collection<? extends ObjectId> unshallow) { |
| this.shallowPack = true; |
| this.depth = depth; |
| this.unshallowObjects = unshallow; |
| } |
| |
| /** |
| * @param filter the filter which indicates what and what not this writer |
| * should include |
| */ |
| public void setFilterSpec(@NonNull FilterSpec filter) { |
| filterSpec = requireNonNull(filter); |
| } |
| |
| /** |
| * @param config configuration related to packfile URIs |
| * @since 5.5 |
| */ |
| public void setPackfileUriConfig(PackfileUriConfig config) { |
| packfileUriConfig = config; |
| } |
| |
| /** |
| * Returns objects number in a pack file that was created by this writer. |
| * |
| * @return number of objects in pack. |
| * @throws java.io.IOException |
| * a cached pack cannot supply its object count. |
| */ |
| public long getObjectCount() throws IOException { |
| if (stats.totalObjects == 0) { |
| long objCnt = 0; |
| |
| objCnt += objectsLists[OBJ_COMMIT].size(); |
| objCnt += objectsLists[OBJ_TREE].size(); |
| objCnt += objectsLists[OBJ_BLOB].size(); |
| objCnt += objectsLists[OBJ_TAG].size(); |
| |
| for (CachedPack pack : cachedPacks) |
| objCnt += pack.getObjectCount(); |
| return objCnt; |
| } |
| return stats.totalObjects; |
| } |
| |
| private long getUnoffloadedObjectCount() throws IOException { |
| long objCnt = 0; |
| |
| objCnt += objectsLists[OBJ_COMMIT].size(); |
| objCnt += objectsLists[OBJ_TREE].size(); |
| objCnt += objectsLists[OBJ_BLOB].size(); |
| objCnt += objectsLists[OBJ_TAG].size(); |
| |
| for (CachedPack pack : cachedPacks) { |
| CachedPackUriProvider.PackInfo packInfo = |
| packfileUriConfig.cachedPackUriProvider.getInfo( |
| pack, packfileUriConfig.protocolsSupported); |
| if (packInfo == null) { |
| objCnt += pack.getObjectCount(); |
| } |
| } |
| |
| return objCnt; |
| } |
| |
| /** |
| * Returns the object ids in the pack file that was created by this writer. |
| * <p> |
| * This method can only be invoked after |
| * {@link #writePack(ProgressMonitor, ProgressMonitor, OutputStream)} has |
| * been invoked and completed successfully. |
| * |
| * @return set of objects in pack. |
| * @throws java.io.IOException |
| * a cached pack cannot supply its object ids. |
| */ |
| public ObjectIdOwnerMap<ObjectIdOwnerMap.Entry> getObjectSet() |
| throws IOException { |
| if (!cachedPacks.isEmpty()) |
| throw new IOException( |
| JGitText.get().cachedPacksPreventsListingObjects); |
| |
| if (writeBitmaps != null) { |
| return writeBitmaps.getObjectSet(); |
| } |
| |
| ObjectIdOwnerMap<ObjectIdOwnerMap.Entry> r = new ObjectIdOwnerMap<>(); |
| for (BlockList<ObjectToPack> objList : objectsLists) { |
| if (objList != null) { |
| for (ObjectToPack otp : objList) |
| r.add(new ObjectIdOwnerMap.Entry(otp) { |
| // A new entry that copies the ObjectId |
| }); |
| } |
| } |
| return r; |
| } |
| |
| /** |
| * Add a pack index whose contents should be excluded from the result. |
| * |
| * @param idx |
| * objects in this index will not be in the output pack. |
| */ |
| public void excludeObjects(ObjectIdSet idx) { |
| if (excludeInPacks == null) { |
| excludeInPacks = new ObjectIdSet[] { idx }; |
| excludeInPackLast = idx; |
| } else { |
| int cnt = excludeInPacks.length; |
| ObjectIdSet[] newList = new ObjectIdSet[cnt + 1]; |
| System.arraycopy(excludeInPacks, 0, newList, 0, cnt); |
| newList[cnt] = idx; |
| excludeInPacks = newList; |
| } |
| } |
| |
| /** |
| * Prepare the list of objects to be written to the pack stream. |
| * <p> |
| * Iterator <b>exactly</b> determines which objects are included in a pack |
| * and order they appear in pack (except that objects order by type is not |
| * needed at input). This order should conform general rules of ordering |
| * objects in git - by recency and path (type and delta-base first is |
| * internally secured) and responsibility for guaranteeing this order is on |
| * a caller side. Iterator must return each id of object to write exactly |
| * once. |
| * </p> |
| * |
| * @param objectsSource |
| * iterator of object to store in a pack; order of objects within |
| * each type is important, ordering by type is not needed; |
| * allowed types for objects are |
| * {@link org.eclipse.jgit.lib.Constants#OBJ_COMMIT}, |
| * {@link org.eclipse.jgit.lib.Constants#OBJ_TREE}, |
| * {@link org.eclipse.jgit.lib.Constants#OBJ_BLOB} and |
| * {@link org.eclipse.jgit.lib.Constants#OBJ_TAG}; objects |
| * returned by iterator may be later reused by caller as object |
| * id and type are internally copied in each iteration. |
| * @throws java.io.IOException |
| * when some I/O problem occur during reading objects. |
| */ |
| public void preparePack(@NonNull Iterator<RevObject> objectsSource) |
| throws IOException { |
| while (objectsSource.hasNext()) { |
| addObject(objectsSource.next()); |
| } |
| } |
| |
| /** |
| * Prepare the list of objects to be written to the pack stream. |
| * <p> |
| * Basing on these 2 sets, another set of objects to put in a pack file is |
| * created: this set consists of all objects reachable (ancestors) from |
| * interesting objects, except uninteresting objects and their ancestors. |
| * This method uses class {@link org.eclipse.jgit.revwalk.ObjectWalk} |
| * extensively to find out that appropriate set of output objects and their |
| * optimal order in output pack. Order is consistent with general git |
| * in-pack rules: sort by object type, recency, path and delta-base first. |
| * </p> |
| * |
| * @param countingMonitor |
| * progress during object enumeration. |
| * @param want |
| * collection of objects to be marked as interesting (start |
| * points of graph traversal). Must not be {@code null}. |
| * @param have |
| * collection of objects to be marked as uninteresting (end |
| * points of graph traversal). Pass {@link #NONE} if all objects |
| * reachable from {@code want} are desired, such as when serving |
| * a clone. |
| * @throws java.io.IOException |
| * when some I/O problem occur during reading objects. |
| */ |
| public void preparePack(ProgressMonitor countingMonitor, |
| @NonNull Set<? extends ObjectId> want, |
| @NonNull Set<? extends ObjectId> have) throws IOException { |
| preparePack(countingMonitor, want, have, NONE, NONE); |
| } |
| |
| /** |
| * Prepare the list of objects to be written to the pack stream. |
| * <p> |
| * Like {@link #preparePack(ProgressMonitor, Set, Set)} but also allows |
| * specifying commits that should not be walked past ("shallow" commits). |
| * The caller is responsible for filtering out commits that should not be |
| * shallow any more ("unshallow" commits as in {@link #setShallowPack}) from |
| * the shallow set. |
| * |
| * @param countingMonitor |
| * progress during object enumeration. |
| * @param want |
| * objects of interest, ancestors of which will be included in |
| * the pack. Must not be {@code null}. |
| * @param have |
| * objects whose ancestors (up to and including {@code shallow} |
| * commits) do not need to be included in the pack because they |
| * are already available from elsewhere. Must not be |
| * {@code null}. |
| * @param shallow |
| * commits indicating the boundary of the history marked with |
| * {@code have}. Shallow commits have parents but those parents |
| * are considered not to be already available. Parents of |
| * {@code shallow} commits and earlier generations will be |
| * included in the pack if requested by {@code want}. Must not be |
| * {@code null}. |
| * @throws java.io.IOException |
| * an I/O problem occurred while reading objects. |
| */ |
| public void preparePack(ProgressMonitor countingMonitor, |
| @NonNull Set<? extends ObjectId> want, |
| @NonNull Set<? extends ObjectId> have, |
| @NonNull Set<? extends ObjectId> shallow) throws IOException { |
| preparePack(countingMonitor, want, have, shallow, NONE); |
| } |
| |
| /** |
| * Prepare the list of objects to be written to the pack stream. |
| * <p> |
| * Like {@link #preparePack(ProgressMonitor, Set, Set)} but also allows |
| * specifying commits that should not be walked past ("shallow" commits). |
| * The caller is responsible for filtering out commits that should not be |
| * shallow any more ("unshallow" commits as in {@link #setShallowPack}) from |
| * the shallow set. |
| * |
| * @param countingMonitor |
| * progress during object enumeration. |
| * @param want |
| * objects of interest, ancestors of which will be included in |
| * the pack. Must not be {@code null}. |
| * @param have |
| * objects whose ancestors (up to and including {@code shallow} |
| * commits) do not need to be included in the pack because they |
| * are already available from elsewhere. Must not be |
| * {@code null}. |
| * @param shallow |
| * commits indicating the boundary of the history marked with |
| * {@code have}. Shallow commits have parents but those parents |
| * are considered not to be already available. Parents of |
| * {@code shallow} commits and earlier generations will be |
| * included in the pack if requested by {@code want}. Must not be |
| * {@code null}. |
| * @param noBitmaps |
| * collection of objects to be excluded from bitmap commit |
| * selection. |
| * @throws java.io.IOException |
| * an I/O problem occurred while reading objects. |
| */ |
| public void preparePack(ProgressMonitor countingMonitor, |
| @NonNull Set<? extends ObjectId> want, |
| @NonNull Set<? extends ObjectId> have, |
| @NonNull Set<? extends ObjectId> shallow, |
| @NonNull Set<? extends ObjectId> noBitmaps) throws IOException { |
| try (ObjectWalk ow = getObjectWalk()) { |
| ow.assumeShallow(shallow); |
| preparePack(countingMonitor, ow, want, have, noBitmaps); |
| } |
| } |
| |
| private ObjectWalk getObjectWalk() { |
| return shallowPack ? new DepthWalk.ObjectWalk(reader, depth - 1) |
| : new ObjectWalk(reader); |
| } |
| |
| /** |
| * A visitation policy which uses the depth at which the object is seen to |
| * decide if re-traversal is necessary. In particular, if the object has |
| * already been visited at this depth or shallower, it is not necessary to |
| * re-visit at this depth. |
| */ |
| private static class DepthAwareVisitationPolicy |
| implements ObjectWalk.VisitationPolicy { |
| private final Map<ObjectId, Integer> lowestDepthVisited = new HashMap<>(); |
| |
| private final ObjectWalk walk; |
| |
| DepthAwareVisitationPolicy(ObjectWalk walk) { |
| this.walk = requireNonNull(walk); |
| } |
| |
| @Override |
| public boolean shouldVisit(RevObject o) { |
| Integer lastDepth = lowestDepthVisited.get(o); |
| if (lastDepth == null) { |
| return true; |
| } |
| return walk.getTreeDepth() < lastDepth.intValue(); |
| } |
| |
| @Override |
| public void visited(RevObject o) { |
| lowestDepthVisited.put(o, Integer.valueOf(walk.getTreeDepth())); |
| } |
| } |
| |
| /** |
| * Prepare the list of objects to be written to the pack stream. |
| * <p> |
| * Basing on these 2 sets, another set of objects to put in a pack file is |
| * created: this set consists of all objects reachable (ancestors) from |
| * interesting objects, except uninteresting objects and their ancestors. |
| * This method uses class {@link org.eclipse.jgit.revwalk.ObjectWalk} |
| * extensively to find out that appropriate set of output objects and their |
| * optimal order in output pack. Order is consistent with general git |
| * in-pack rules: sort by object type, recency, path and delta-base first. |
| * </p> |
| * |
| * @param countingMonitor |
| * progress during object enumeration. |
| * @param walk |
| * ObjectWalk to perform enumeration. |
| * @param interestingObjects |
| * collection of objects to be marked as interesting (start |
| * points of graph traversal). Must not be {@code null}. |
| * @param uninterestingObjects |
| * collection of objects to be marked as uninteresting (end |
| * points of graph traversal). Pass {@link #NONE} if all objects |
| * reachable from {@code want} are desired, such as when serving |
| * a clone. |
| * @param noBitmaps |
| * collection of objects to be excluded from bitmap commit |
| * selection. |
| * @throws java.io.IOException |
| * when some I/O problem occur during reading objects. |
| */ |
| public void preparePack(ProgressMonitor countingMonitor, |
| @NonNull ObjectWalk walk, |
| @NonNull Set<? extends ObjectId> interestingObjects, |
| @NonNull Set<? extends ObjectId> uninterestingObjects, |
| @NonNull Set<? extends ObjectId> noBitmaps) |
| throws IOException { |
| if (countingMonitor == null) |
| countingMonitor = NullProgressMonitor.INSTANCE; |
| if (shallowPack && !(walk instanceof DepthWalk.ObjectWalk)) |
| throw new IllegalArgumentException( |
| JGitText.get().shallowPacksRequireDepthWalk); |
| if (filterSpec.getTreeDepthLimit() >= 0) { |
| walk.setVisitationPolicy(new DepthAwareVisitationPolicy(walk)); |
| } |
| findObjectsToPack(countingMonitor, walk, interestingObjects, |
| uninterestingObjects, noBitmaps); |
| } |
| |
| /** |
| * Determine if the pack file will contain the requested object. |
| * |
| * @param id |
| * the object to test the existence of. |
| * @return true if the object will appear in the output pack file. |
| * @throws java.io.IOException |
| * a cached pack cannot be examined. |
| */ |
| public boolean willInclude(AnyObjectId id) throws IOException { |
| ObjectToPack obj = objectsMap.get(id); |
| return obj != null && !obj.isEdge(); |
| } |
| |
| /** |
| * Lookup the ObjectToPack object for a given ObjectId. |
| * |
| * @param id |
| * the object to find in the pack. |
| * @return the object we are packing, or null. |
| */ |
| public ObjectToPack get(AnyObjectId id) { |
| ObjectToPack obj = objectsMap.get(id); |
| return obj != null && !obj.isEdge() ? obj : null; |
| } |
| |
| /** |
| * Computes SHA-1 of lexicographically sorted objects ids written in this |
| * pack, as used to name a pack file in repository. |
| * |
| * @return ObjectId representing SHA-1 name of a pack that was created. |
| */ |
| public ObjectId computeName() { |
| final byte[] buf = new byte[OBJECT_ID_LENGTH]; |
| final MessageDigest md = Constants.newMessageDigest(); |
| for (ObjectToPack otp : sortByName()) { |
| otp.copyRawTo(buf, 0); |
| md.update(buf, 0, OBJECT_ID_LENGTH); |
| } |
| return ObjectId.fromRaw(md.digest()); |
| } |
| |
| /** |
| * Returns the index format version that will be written. |
| * <p> |
| * This method can only be invoked after |
| * {@link #writePack(ProgressMonitor, ProgressMonitor, OutputStream)} has |
| * been invoked and completed successfully. |
| * |
| * @return the index format version. |
| */ |
| public int getIndexVersion() { |
| int indexVersion = config.getIndexVersion(); |
| if (indexVersion <= 0) { |
| for (BlockList<ObjectToPack> objs : objectsLists) |
| indexVersion = Math.max(indexVersion, |
| PackIndexWriter.oldestPossibleFormat(objs)); |
| } |
| return indexVersion; |
| } |
| |
| /** |
| * Create an index file to match the pack file just written. |
| * <p> |
| * Called after |
| * {@link #writePack(ProgressMonitor, ProgressMonitor, OutputStream)}. |
| * <p> |
| * Writing an index is only required for local pack storage. Packs sent on |
| * the network do not need to create an index. |
| * |
| * @param indexStream |
| * output for the index data. Caller is responsible for closing |
| * this stream. |
| * @throws java.io.IOException |
| * the index data could not be written to the supplied stream. |
| */ |
| public void writeIndex(OutputStream indexStream) throws IOException { |
| if (isIndexDisabled()) |
| throw new IOException(JGitText.get().cachedPacksPreventsIndexCreation); |
| |
| long writeStart = System.currentTimeMillis(); |
| final PackIndexWriter iw = PackIndexWriter.createVersion( |
| indexStream, getIndexVersion()); |
| iw.write(sortByName(), packcsum); |
| stats.timeWriting += System.currentTimeMillis() - writeStart; |
| } |
| |
| /** |
| * Create a bitmap index file to match the pack file just written. |
| * <p> |
| * Called after {@link #prepareBitmapIndex(ProgressMonitor)}. |
| * |
| * @param bitmapIndexStream |
| * output for the bitmap index data. Caller is responsible for |
| * closing this stream. |
| * @throws java.io.IOException |
| * the index data could not be written to the supplied stream. |
| */ |
| public void writeBitmapIndex(OutputStream bitmapIndexStream) |
| throws IOException { |
| if (writeBitmaps == null) |
| throw new IOException(JGitText.get().bitmapsMustBePrepared); |
| |
| long writeStart = System.currentTimeMillis(); |
| final PackBitmapIndexWriterV1 iw = new PackBitmapIndexWriterV1(bitmapIndexStream); |
| iw.write(writeBitmaps, packcsum); |
| stats.timeWriting += System.currentTimeMillis() - writeStart; |
| } |
| |
| private List<ObjectToPack> sortByName() { |
| if (sortedByName == null) { |
| int cnt = 0; |
| cnt += objectsLists[OBJ_COMMIT].size(); |
| cnt += objectsLists[OBJ_TREE].size(); |
| cnt += objectsLists[OBJ_BLOB].size(); |
| cnt += objectsLists[OBJ_TAG].size(); |
| |
| sortedByName = new BlockList<>(cnt); |
| sortedByName.addAll(objectsLists[OBJ_COMMIT]); |
| sortedByName.addAll(objectsLists[OBJ_TREE]); |
| sortedByName.addAll(objectsLists[OBJ_BLOB]); |
| sortedByName.addAll(objectsLists[OBJ_TAG]); |
| Collections.sort(sortedByName); |
| } |
| return sortedByName; |
| } |
| |
| private void beginPhase(PackingPhase phase, ProgressMonitor monitor, |
| long cnt) { |
| state.phase = phase; |
| String task; |
| switch (phase) { |
| case COUNTING: |
| task = JGitText.get().countingObjects; |
| break; |
| case GETTING_SIZES: |
| task = JGitText.get().searchForSizes; |
| break; |
| case FINDING_SOURCES: |
| task = JGitText.get().searchForReuse; |
| break; |
| case COMPRESSING: |
| task = JGitText.get().compressingObjects; |
| break; |
| case WRITING: |
| task = JGitText.get().writingObjects; |
| break; |
| case BUILDING_BITMAPS: |
| task = JGitText.get().buildingBitmaps; |
| break; |
| default: |
| throw new IllegalArgumentException( |
| MessageFormat.format(JGitText.get().illegalPackingPhase, phase)); |
| } |
| monitor.beginTask(task, (int) cnt); |
| } |
| |
| private void endPhase(ProgressMonitor monitor) { |
| monitor.endTask(); |
| } |
| |
| /** |
| * Write the prepared pack to the supplied stream. |
| * <p> |
| * Called after |
| * {@link #preparePack(ProgressMonitor, ObjectWalk, Set, Set, Set)} or |
| * {@link #preparePack(ProgressMonitor, Set, Set)}. |
| * <p> |
| * Performs delta search if enabled and writes the pack stream. |
| * <p> |
| * All reused objects data checksum (Adler32/CRC32) is computed and |
| * validated against existing checksum. |
| * |
| * @param compressMonitor |
| * progress monitor to report object compression work. |
| * @param writeMonitor |
| * progress monitor to report the number of objects written. |
| * @param packStream |
| * output stream of pack data. The stream should be buffered by |
| * the caller. The caller is responsible for closing the stream. |
| * @throws java.io.IOException |
| * an error occurred reading a local object's data to include in |
| * the pack, or writing compressed object data to the output |
| * stream. |
| * @throws WriteAbortedException |
| * the write operation is aborted by |
| * {@link org.eclipse.jgit.transport.ObjectCountCallback} . |
| */ |
| public void writePack(ProgressMonitor compressMonitor, |
| ProgressMonitor writeMonitor, OutputStream packStream) |
| throws IOException { |
| if (compressMonitor == null) |
| compressMonitor = NullProgressMonitor.INSTANCE; |
| if (writeMonitor == null) |
| writeMonitor = NullProgressMonitor.INSTANCE; |
| |
| excludeInPacks = null; |
| excludeInPackLast = null; |
| |
| boolean needSearchForReuse = reuseSupport != null && ( |
| reuseDeltas |
| || config.isReuseObjects() |
| || !cachedPacks.isEmpty()); |
| |
| if (compressMonitor instanceof BatchingProgressMonitor) { |
| long delay = 1000; |
| if (needSearchForReuse && config.isDeltaCompress()) |
| delay = 500; |
| ((BatchingProgressMonitor) compressMonitor).setDelayStart( |
| delay, |
| TimeUnit.MILLISECONDS); |
| } |
| |
| if (needSearchForReuse) |
| searchForReuse(compressMonitor); |
| if (config.isDeltaCompress()) |
| searchForDeltas(compressMonitor); |
| |
| crc32 = new CRC32(); |
| final PackOutputStream out = new PackOutputStream( |
| writeMonitor, |
| isIndexDisabled() |
| ? packStream |
| : new CheckedOutputStream(packStream, crc32), |
| this); |
| |
| long objCnt = packfileUriConfig == null ? getObjectCount() : |
| getUnoffloadedObjectCount(); |
| stats.totalObjects = objCnt; |
| if (callback != null) |
| callback.setObjectCount(objCnt); |
| beginPhase(PackingPhase.WRITING, writeMonitor, objCnt); |
| long writeStart = System.currentTimeMillis(); |
| try { |
| List<CachedPack> unwrittenCachedPacks; |
| |
| if (packfileUriConfig != null) { |
| unwrittenCachedPacks = new ArrayList<>(); |
| CachedPackUriProvider p = packfileUriConfig.cachedPackUriProvider; |
| PacketLineOut o = packfileUriConfig.pckOut; |
| |
| o.writeString("packfile-uris\n"); //$NON-NLS-1$ |
| for (CachedPack pack : cachedPacks) { |
| CachedPackUriProvider.PackInfo packInfo = p.getInfo( |
| pack, packfileUriConfig.protocolsSupported); |
| if (packInfo != null) { |
| o.writeString(packInfo.getHash() + ' ' + |
| packInfo.getUri() + '\n'); |
| stats.offloadedPackfiles += 1; |
| stats.offloadedPackfileSize += packInfo.getSize(); |
| } else { |
| unwrittenCachedPacks.add(pack); |
| } |
| } |
| packfileUriConfig.pckOut.writeDelim(); |
| packfileUriConfig.pckOut.writeString("packfile\n"); //$NON-NLS-1$ |
| } else { |
| unwrittenCachedPacks = cachedPacks; |
| } |
| |
| out.writeFileHeader(PACK_VERSION_GENERATED, objCnt); |
| out.flush(); |
| |
| writeObjects(out); |
| if (!edgeObjects.isEmpty() || !cachedPacks.isEmpty()) { |
| for (PackStatistics.ObjectType.Accumulator typeStat : stats.objectTypes) { |
| if (typeStat == null) |
| continue; |
| stats.thinPackBytes += typeStat.bytes; |
| } |
| } |
| |
| stats.reusedPacks = Collections.unmodifiableList(cachedPacks); |
| for (CachedPack pack : unwrittenCachedPacks) { |
| long deltaCnt = pack.getDeltaCount(); |
| stats.reusedObjects += pack.getObjectCount(); |
| stats.reusedDeltas += deltaCnt; |
| stats.totalDeltas += deltaCnt; |
| reuseSupport.copyPackAsIs(out, pack); |
| } |
| writeChecksum(out); |
| out.flush(); |
| } finally { |
| stats.timeWriting = System.currentTimeMillis() - writeStart; |
| stats.depth = depth; |
| |
| for (PackStatistics.ObjectType.Accumulator typeStat : stats.objectTypes) { |
| if (typeStat == null) |
| continue; |
| typeStat.cntDeltas += typeStat.reusedDeltas; |
| stats.reusedObjects += typeStat.reusedObjects; |
| stats.reusedDeltas += typeStat.reusedDeltas; |
| stats.totalDeltas += typeStat.cntDeltas; |
| } |
| } |
| |
| stats.totalBytes = out.length(); |
| reader.close(); |
| endPhase(writeMonitor); |
| } |
| |
| /** |
| * Get statistics of what this PackWriter did in order to create the final |
| * pack stream. |
| * |
| * @return description of what this PackWriter did in order to create the |
| * final pack stream. This should only be invoked after the calls to |
| * create the pack/index/bitmap have completed. |
| */ |
| public PackStatistics getStatistics() { |
| return new PackStatistics(stats); |
| } |
| |
| /** |
| * Get snapshot of the current state of this PackWriter. |
| * |
| * @return snapshot of the current state of this PackWriter. |
| */ |
| public State getState() { |
| return state.snapshot(); |
| } |
| |
| /** |
| * {@inheritDoc} |
| * <p> |
| * Release all resources used by this writer. |
| */ |
| @Override |
| public void close() { |
| reader.close(); |
| if (myDeflater != null) { |
| myDeflater.end(); |
| myDeflater = null; |
| } |
| instances.remove(selfRef); |
| } |
| |
| private void searchForReuse(ProgressMonitor monitor) throws IOException { |
| long cnt = 0; |
| cnt += objectsLists[OBJ_COMMIT].size(); |
| cnt += objectsLists[OBJ_TREE].size(); |
| cnt += objectsLists[OBJ_BLOB].size(); |
| cnt += objectsLists[OBJ_TAG].size(); |
| |
| long start = System.currentTimeMillis(); |
| beginPhase(PackingPhase.FINDING_SOURCES, monitor, cnt); |
| if (cnt <= 4096) { |
| // For small object counts, do everything as one list. |
| BlockList<ObjectToPack> tmp = new BlockList<>((int) cnt); |
| tmp.addAll(objectsLists[OBJ_TAG]); |
| tmp.addAll(objectsLists[OBJ_COMMIT]); |
| tmp.addAll(objectsLists[OBJ_TREE]); |
| tmp.addAll(objectsLists[OBJ_BLOB]); |
| searchForReuse(monitor, tmp); |
| if (pruneCurrentObjectList) { |
| // If the list was pruned, we need to re-prune the main lists. |
| pruneEdgesFromObjectList(objectsLists[OBJ_COMMIT]); |
| pruneEdgesFromObjectList(objectsLists[OBJ_TREE]); |
| pruneEdgesFromObjectList(objectsLists[OBJ_BLOB]); |
| pruneEdgesFromObjectList(objectsLists[OBJ_TAG]); |
| } |
| } else { |
| searchForReuse(monitor, objectsLists[OBJ_TAG]); |
| searchForReuse(monitor, objectsLists[OBJ_COMMIT]); |
| searchForReuse(monitor, objectsLists[OBJ_TREE]); |
| searchForReuse(monitor, objectsLists[OBJ_BLOB]); |
| } |
| endPhase(monitor); |
| stats.timeSearchingForReuse = System.currentTimeMillis() - start; |
| |
| if (config.isReuseDeltas() && config.getCutDeltaChains()) { |
| cutDeltaChains(objectsLists[OBJ_TREE]); |
| cutDeltaChains(objectsLists[OBJ_BLOB]); |
| } |
| } |
| |
| private void searchForReuse(ProgressMonitor monitor, List<ObjectToPack> list) |
| throws IOException, MissingObjectException { |
| pruneCurrentObjectList = false; |
| reuseSupport.selectObjectRepresentation(this, monitor, list); |
| if (pruneCurrentObjectList) |
| pruneEdgesFromObjectList(list); |
| } |
| |
| private void cutDeltaChains(BlockList<ObjectToPack> list) |
| throws IOException { |
| int max = config.getMaxDeltaDepth(); |
| for (int idx = list.size() - 1; idx >= 0; idx--) { |
| int d = 0; |
| ObjectToPack b = list.get(idx).getDeltaBase(); |
| while (b != null) { |
| if (d < b.getChainLength()) |
| break; |
| b.setChainLength(++d); |
| if (d >= max && b.isDeltaRepresentation()) { |
| reselectNonDelta(b); |
| break; |
| } |
| b = b.getDeltaBase(); |
| } |
| } |
| if (config.isDeltaCompress()) { |
| for (ObjectToPack otp : list) |
| otp.clearChainLength(); |
| } |
| } |
| |
| private void searchForDeltas(ProgressMonitor monitor) |
| throws MissingObjectException, IncorrectObjectTypeException, |
| IOException { |
| // Commits and annotated tags tend to have too many differences to |
| // really benefit from delta compression. Consequently just don't |
| // bother examining those types here. |
| // |
| ObjectToPack[] list = new ObjectToPack[ |
| objectsLists[OBJ_TREE].size() |
| + objectsLists[OBJ_BLOB].size() |
| + edgeObjects.size()]; |
| int cnt = 0; |
| cnt = findObjectsNeedingDelta(list, cnt, OBJ_TREE); |
| cnt = findObjectsNeedingDelta(list, cnt, OBJ_BLOB); |
| if (cnt == 0) |
| return; |
| int nonEdgeCnt = cnt; |
| |
| // Queue up any edge objects that we might delta against. We won't |
| // be sending these as we assume the other side has them, but we need |
| // them in the search phase below. |
| // |
| for (ObjectToPack eo : edgeObjects) { |
| eo.setWeight(0); |
| list[cnt++] = eo; |
| } |
| |
| // Compute the sizes of the objects so we can do a proper sort. |
| // We let the reader skip missing objects if it chooses. For |
| // some readers this can be a huge win. We detect missing objects |
| // by having set the weights above to 0 and allowing the delta |
| // search code to discover the missing object and skip over it, or |
| // abort with an exception if we actually had to have it. |
| // |
| final long sizingStart = System.currentTimeMillis(); |
| beginPhase(PackingPhase.GETTING_SIZES, monitor, cnt); |
| AsyncObjectSizeQueue<ObjectToPack> sizeQueue = reader.getObjectSize( |
| Arrays.<ObjectToPack> asList(list).subList(0, cnt), false); |
| try { |
| final long limit = Math.min( |
| config.getBigFileThreshold(), |
| Integer.MAX_VALUE); |
| for (;;) { |
| try { |
| if (!sizeQueue.next()) |
| break; |
| } catch (MissingObjectException notFound) { |
| monitor.update(1); |
| if (ignoreMissingUninteresting) { |
| ObjectToPack otp = sizeQueue.getCurrent(); |
| if (otp != null && otp.isEdge()) { |
| otp.setDoNotDelta(); |
| continue; |
| } |
| |
| otp = objectsMap.get(notFound.getObjectId()); |
| if (otp != null && otp.isEdge()) { |
| otp.setDoNotDelta(); |
| continue; |
| } |
| } |
| throw notFound; |
| } |
| |
| ObjectToPack otp = sizeQueue.getCurrent(); |
| if (otp == null) |
| otp = objectsMap.get(sizeQueue.getObjectId()); |
| |
| long sz = sizeQueue.getSize(); |
| if (DeltaIndex.BLKSZ < sz && sz < limit) |
| otp.setWeight((int) sz); |
| else |
| otp.setDoNotDelta(); // too small, or too big |
| monitor.update(1); |
| } |
| } finally { |
| sizeQueue.release(); |
| } |
| endPhase(monitor); |
| stats.timeSearchingForSizes = System.currentTimeMillis() - sizingStart; |
| |
| // Sort the objects by path hash so like files are near each other, |
| // and then by size descending so that bigger files are first. This |
| // applies "Linus' Law" which states that newer files tend to be the |
| // bigger ones, because source files grow and hardly ever shrink. |
| // |
| Arrays.sort(list, 0, cnt, (ObjectToPack a, ObjectToPack b) -> { |
| int cmp = (a.isDoNotDelta() ? 1 : 0) - (b.isDoNotDelta() ? 1 : 0); |
| if (cmp != 0) { |
| return cmp; |
| } |
| |
| cmp = a.getType() - b.getType(); |
| if (cmp != 0) { |
| return cmp; |
| } |
| |
| cmp = (a.getPathHash() >>> 1) - (b.getPathHash() >>> 1); |
| if (cmp != 0) { |
| return cmp; |
| } |
| |
| cmp = (a.getPathHash() & 1) - (b.getPathHash() & 1); |
| if (cmp != 0) { |
| return cmp; |
| } |
| |
| cmp = (a.isEdge() ? 0 : 1) - (b.isEdge() ? 0 : 1); |
| if (cmp != 0) { |
| return cmp; |
| } |
| |
| return b.getWeight() - a.getWeight(); |
| }); |
| |
| // Above we stored the objects we cannot delta onto the end. |
| // Remove them from the list so we don't waste time on them. |
| while (0 < cnt && list[cnt - 1].isDoNotDelta()) { |
| if (!list[cnt - 1].isEdge()) |
| nonEdgeCnt--; |
| cnt--; |
| } |
| if (cnt == 0) |
| return; |
| |
| final long searchStart = System.currentTimeMillis(); |
| searchForDeltas(monitor, list, cnt); |
| stats.deltaSearchNonEdgeObjects = nonEdgeCnt; |
| stats.timeCompressing = System.currentTimeMillis() - searchStart; |
| |
| for (int i = 0; i < cnt; i++) |
| if (!list[i].isEdge() && list[i].isDeltaRepresentation()) |
| stats.deltasFound++; |
| } |
| |
| private int findObjectsNeedingDelta(ObjectToPack[] list, int cnt, int type) { |
| for (ObjectToPack otp : objectsLists[type]) { |
| if (otp.isDoNotDelta()) // delta is disabled for this path |
| continue; |
| if (otp.isDeltaRepresentation()) // already reusing a delta |
| continue; |
| otp.setWeight(0); |
| list[cnt++] = otp; |
| } |
| return cnt; |
| } |
| |
| private void reselectNonDelta(ObjectToPack otp) throws IOException { |
| otp.clearDeltaBase(); |
| otp.clearReuseAsIs(); |
| boolean old = reuseDeltas; |
| reuseDeltas = false; |
| reuseSupport.selectObjectRepresentation(this, |
| NullProgressMonitor.INSTANCE, |
| Collections.singleton(otp)); |
| reuseDeltas = old; |
| } |
| |
| private void searchForDeltas(final ProgressMonitor monitor, |
| final ObjectToPack[] list, final int cnt) |
| throws MissingObjectException, IncorrectObjectTypeException, |
| LargeObjectException, IOException { |
| int threads = config.getThreads(); |
| if (threads == 0) |
| threads = Runtime.getRuntime().availableProcessors(); |
| if (threads <= 1 || cnt <= config.getDeltaSearchWindowSize()) |
| singleThreadDeltaSearch(monitor, list, cnt); |
| else |
| parallelDeltaSearch(monitor, list, cnt, threads); |
| } |
| |
| private void singleThreadDeltaSearch(ProgressMonitor monitor, |
| ObjectToPack[] list, int cnt) throws IOException { |
| long totalWeight = 0; |
| for (int i = 0; i < cnt; i++) { |
| ObjectToPack o = list[i]; |
| totalWeight += DeltaTask.getAdjustedWeight(o); |
| } |
| |
| long bytesPerUnit = 1; |
| while (DeltaTask.MAX_METER <= (totalWeight / bytesPerUnit)) |
| bytesPerUnit <<= 10; |
| int cost = (int) (totalWeight / bytesPerUnit); |
| if (totalWeight % bytesPerUnit != 0) |
| cost++; |
| |
| beginPhase(PackingPhase.COMPRESSING, monitor, cost); |
| new DeltaWindow(config, new DeltaCache(config), reader, |
| monitor, bytesPerUnit, |
| list, 0, cnt).search(); |
| endPhase(monitor); |
| } |
| |
| private void parallelDeltaSearch(ProgressMonitor monitor, |
| ObjectToPack[] list, int cnt, int threads) throws IOException { |
| DeltaCache dc = new ThreadSafeDeltaCache(config); |
| ThreadSafeProgressMonitor pm = new ThreadSafeProgressMonitor(monitor); |
| DeltaTask.Block taskBlock = new DeltaTask.Block(threads, config, |
| reader, dc, pm, |
| list, 0, cnt); |
| taskBlock.partitionTasks(); |
| beginPhase(PackingPhase.COMPRESSING, monitor, taskBlock.cost()); |
| pm.startWorkers(taskBlock.tasks.size()); |
| |
| Executor executor = config.getExecutor(); |
| final List<Throwable> errors = |
| Collections.synchronizedList(new ArrayList<>(threads)); |
| if (executor instanceof ExecutorService) { |
| // Caller supplied us a service, use it directly. |
| runTasks((ExecutorService) executor, pm, taskBlock, errors); |
| } else if (executor == null) { |
| // Caller didn't give us a way to run the tasks, spawn up a |
| // temporary thread pool and make sure it tears down cleanly. |
| ExecutorService pool = Executors.newFixedThreadPool(threads); |
| try { |
| runTasks(pool, pm, taskBlock, errors); |
| } finally { |
| pool.shutdown(); |
| for (;;) { |
| try { |
| if (pool.awaitTermination(60, TimeUnit.SECONDS)) |
| break; |
| } catch (InterruptedException e) { |
| throw new IOException( |
| JGitText.get().packingCancelledDuringObjectsWriting); |
| } |
| } |
| } |
| } else { |
| // The caller gave us an executor, but it might not do |
| // asynchronous execution. Wrap everything and hope it |
| // can schedule these for us. |
| for (DeltaTask task : taskBlock.tasks) { |
| executor.execute(() -> { |
| try { |
| task.call(); |
| } catch (Throwable failure) { |
| errors.add(failure); |
| } |
| }); |
| } |
| try { |
| pm.waitForCompletion(); |
| } catch (InterruptedException ie) { |
| // We can't abort the other tasks as we have no handle. |
| // Cross our fingers and just break out anyway. |
| // |
| throw new IOException( |
| JGitText.get().packingCancelledDuringObjectsWriting); |
| } |
| } |
| |
| // If any task threw an error, try to report it back as |
| // though we weren't using a threaded search algorithm. |
| // |
| if (!errors.isEmpty()) { |
| Throwable err = errors.get(0); |
| if (err instanceof Error) |
| throw (Error) err; |
| if (err instanceof RuntimeException) |
| throw (RuntimeException) err; |
| if (err instanceof IOException) |
| throw (IOException) err; |
| |
| throw new IOException(err.getMessage(), err); |
| } |
| endPhase(monitor); |
| } |
| |
| private static void runTasks(ExecutorService pool, |
| ThreadSafeProgressMonitor pm, |
| DeltaTask.Block tb, List<Throwable> errors) throws IOException { |
| List<Future<?>> futures = new ArrayList<>(tb.tasks.size()); |
| for (DeltaTask task : tb.tasks) |
| futures.add(pool.submit(task)); |
| |
| try { |
| pm.waitForCompletion(); |
| for (Future<?> f : futures) { |
| try { |
| f.get(); |
| } catch (ExecutionException failed) { |
| errors.add(failed.getCause()); |
| } |
| } |
| } catch (InterruptedException ie) { |
| for (Future<?> f : futures) |
| f.cancel(true); |
| throw new IOException( |
| JGitText.get().packingCancelledDuringObjectsWriting); |
| } |
| } |
| |
| private void writeObjects(PackOutputStream out) throws IOException { |
| writeObjects(out, objectsLists[OBJ_COMMIT]); |
| writeObjects(out, objectsLists[OBJ_TAG]); |
| writeObjects(out, objectsLists[OBJ_TREE]); |
| writeObjects(out, objectsLists[OBJ_BLOB]); |
| } |
| |
| private void writeObjects(PackOutputStream out, List<ObjectToPack> list) |
| throws IOException { |
| if (list.isEmpty()) |
| return; |
| |
| typeStats = stats.objectTypes[list.get(0).getType()]; |
| long beginOffset = out.length(); |
| |
| if (reuseSupport != null) { |
| reuseSupport.writeObjects(out, list); |
| } else { |
| for (ObjectToPack otp : list) |
| out.writeObject(otp); |
| } |
| |
| typeStats.bytes += out.length() - beginOffset; |
| typeStats.cntObjects = list.size(); |
| } |
| |
| void writeObject(PackOutputStream out, ObjectToPack otp) throws IOException { |
| if (!otp.isWritten()) |
| writeObjectImpl(out, otp); |
| } |
| |
| private void writeObjectImpl(PackOutputStream out, ObjectToPack otp) |
| throws IOException { |
| if (otp.wantWrite()) { |
| // A cycle exists in this delta chain. This should only occur if a |
| // selected object representation disappeared during writing |
| // (for example due to a concurrent repack) and a different base |
| // was chosen, forcing a cycle. Select something other than a |
| // delta, and write this object. |
| reselectNonDelta(otp); |
| } |
| otp.markWantWrite(); |
| |
| while (otp.isReuseAsIs()) { |
| writeBase(out, otp.getDeltaBase()); |
| if (otp.isWritten()) |
| return; // Delta chain cycle caused this to write already. |
| |
| crc32.reset(); |
| otp.setOffset(out.length()); |
| try { |
| reuseSupport.copyObjectAsIs(out, otp, reuseValidate); |
| out.endObject(); |
| otp.setCRC((int) crc32.getValue()); |
| typeStats.reusedObjects++; |
| if (otp.isDeltaRepresentation()) { |
| typeStats.reusedDeltas++; |
| typeStats.deltaBytes += out.length() - otp.getOffset(); |
| } |
| return; |
| } catch (StoredObjectRepresentationNotAvailableException gone) { |
| if (otp.getOffset() == out.length()) { |
| otp.setOffset(0); |
| otp.clearDeltaBase(); |
| otp.clearReuseAsIs(); |
| reuseSupport.selectObjectRepresentation(this, |
| NullProgressMonitor.INSTANCE, |
| Collections.singleton(otp)); |
| continue; |
| } |
| // Object writing already started, we cannot recover. |
| // |
| CorruptObjectException coe; |
| coe = new CorruptObjectException(otp, ""); //$NON-NLS-1$ |
| coe.initCause(gone); |
| throw coe; |
| } |
| } |
| |
| // If we reached here, reuse wasn't possible. |
| // |
| if (otp.isDeltaRepresentation()) { |
| writeDeltaObjectDeflate(out, otp); |
| } else { |
| writeWholeObjectDeflate(out, otp); |
| } |
| out.endObject(); |
| otp.setCRC((int) crc32.getValue()); |
| } |
| |
| private void writeBase(PackOutputStream out, ObjectToPack base) |
| throws IOException { |
| if (base != null && !base.isWritten() && !base.isEdge()) |
| writeObjectImpl(out, base); |
| } |
| |
| private void writeWholeObjectDeflate(PackOutputStream out, |
| final ObjectToPack otp) throws IOException { |
| final Deflater deflater = deflater(); |
| final ObjectLoader ldr = reader.open(otp, otp.getType()); |
| |
| crc32.reset(); |
| otp.setOffset(out.length()); |
| out.writeHeader(otp, ldr.getSize()); |
| |
| deflater.reset(); |
| DeflaterOutputStream dst = new DeflaterOutputStream(out, deflater); |
| ldr.copyTo(dst); |
| dst.finish(); |
| } |
| |
| private void writeDeltaObjectDeflate(PackOutputStream out, |
| final ObjectToPack otp) throws IOException { |
| writeBase(out, otp.getDeltaBase()); |
| |
| crc32.reset(); |
| otp.setOffset(out.length()); |
| |
| DeltaCache.Ref ref = otp.popCachedDelta(); |
| if (ref != null) { |
| byte[] zbuf = ref.get(); |
| if (zbuf != null) { |
| out.writeHeader(otp, otp.getCachedSize()); |
| out.write(zbuf); |
| typeStats.cntDeltas++; |
| typeStats.deltaBytes += out.length() - otp.getOffset(); |
| return; |
| } |
| } |
| |
| try (TemporaryBuffer.Heap delta = delta(otp)) { |
| out.writeHeader(otp, delta.length()); |
| |
| Deflater deflater = deflater(); |
| deflater.reset(); |
| DeflaterOutputStream dst = new DeflaterOutputStream(out, deflater); |
| delta.writeTo(dst, null); |
| dst.finish(); |
| } |
| typeStats.cntDeltas++; |
| typeStats.deltaBytes += out.length() - otp.getOffset(); |
| } |
| |
| private TemporaryBuffer.Heap delta(ObjectToPack otp) |
| throws IOException { |
| DeltaIndex index = new DeltaIndex(buffer(otp.getDeltaBaseId())); |
| byte[] res = buffer(otp); |
| |
| // We never would have proposed this pair if the delta would be |
| // larger than the unpacked version of the object. So using it |
| // as our buffer limit is valid: we will never reach it. |
| // |
| TemporaryBuffer.Heap delta = new TemporaryBuffer.Heap(res.length); |
| index.encode(delta, res); |
| return delta; |
| } |
| |
| private byte[] buffer(AnyObjectId objId) throws IOException { |
| return buffer(config, reader, objId); |
| } |
| |
| static byte[] buffer(PackConfig config, ObjectReader or, AnyObjectId objId) |
| throws IOException { |
| // PackWriter should have already pruned objects that |
| // are above the big file threshold, so our chances of |
| // the object being below it are very good. We really |
| // shouldn't be here, unless the implementation is odd. |
| |
| return or.open(objId).getCachedBytes(config.getBigFileThreshold()); |
| } |
| |
| private Deflater deflater() { |
| if (myDeflater == null) |
| myDeflater = new Deflater(config.getCompressionLevel()); |
| return myDeflater; |
| } |
| |
| private void writeChecksum(PackOutputStream out) throws IOException { |
| packcsum = out.getDigest(); |
| out.write(packcsum); |
| } |
| |
| private void findObjectsToPack(@NonNull ProgressMonitor countingMonitor, |
| @NonNull ObjectWalk walker, @NonNull Set<? extends ObjectId> want, |
| @NonNull Set<? extends ObjectId> have, |
| @NonNull Set<? extends ObjectId> noBitmaps) throws IOException { |
| final long countingStart = System.currentTimeMillis(); |
| beginPhase(PackingPhase.COUNTING, countingMonitor, ProgressMonitor.UNKNOWN); |
| |
| stats.interestingObjects = Collections.unmodifiableSet(new HashSet<ObjectId>(want)); |
| stats.uninterestingObjects = Collections.unmodifiableSet(new HashSet<ObjectId>(have)); |
| excludeFromBitmapSelection = noBitmaps; |
| |
| canBuildBitmaps = config.isBuildBitmaps() |
| && !shallowPack |
| && have.isEmpty() |
| && (excludeInPacks == null || excludeInPacks.length == 0); |
| if (!shallowPack && useBitmaps) { |
| BitmapIndex bitmapIndex = reader.getBitmapIndex(); |
| if (bitmapIndex != null) { |
| BitmapWalker bitmapWalker = new BitmapWalker( |
| walker, bitmapIndex, countingMonitor); |
| findObjectsToPackUsingBitmaps(bitmapWalker, want, have); |
| endPhase(countingMonitor); |
| stats.timeCounting = System.currentTimeMillis() - countingStart; |
| stats.bitmapIndexMisses = bitmapWalker.getCountOfBitmapIndexMisses(); |
| return; |
| } |
| } |
| |
| List<ObjectId> all = new ArrayList<>(want.size() + have.size()); |
| all.addAll(want); |
| all.addAll(have); |
| |
| final RevFlag include = walker.newFlag("include"); //$NON-NLS-1$ |
| final RevFlag added = walker.newFlag("added"); //$NON-NLS-1$ |
| |
| walker.carry(include); |
| |
| int haveEst = have.size(); |
| if (have.isEmpty()) { |
| walker.sort(RevSort.COMMIT_TIME_DESC); |
| } else { |
| walker.sort(RevSort.TOPO); |
| if (thin) |
| walker.sort(RevSort.BOUNDARY, true); |
| } |
| |
| List<RevObject> wantObjs = new ArrayList<>(want.size()); |
| List<RevObject> haveObjs = new ArrayList<>(haveEst); |
| List<RevTag> wantTags = new ArrayList<>(want.size()); |
| |
| // Retrieve the RevWalk's versions of "want" and "have" objects to |
| // maintain any state previously set in the RevWalk. |
| AsyncRevObjectQueue q = walker.parseAny(all, true); |
| try { |
| for (;;) { |
| try { |
| RevObject o = q.next(); |
| if (o == null) |
| break; |
| if (have.contains(o)) |
| haveObjs.add(o); |
| if (want.contains(o)) { |
| o.add(include); |
| wantObjs.add(o); |
| if (o instanceof RevTag) |
| wantTags.add((RevTag) o); |
| } |
| } catch (MissingObjectException e) { |
| if (ignoreMissingUninteresting |
| && have.contains(e.getObjectId())) |
| continue; |
| throw e; |
| } |
| } |
| } finally { |
| q.release(); |
| } |
| |
| if (!wantTags.isEmpty()) { |
| all = new ArrayList<>(wantTags.size()); |
| for (RevTag tag : wantTags) |
| all.add(tag.getObject()); |
| q = walker.parseAny(all, true); |
| try { |
| while (q.next() != null) { |
| // Just need to pop the queue item to parse the object. |
| } |
| } finally { |
| q.release(); |
| } |
| } |
| |
| if (walker instanceof DepthWalk.ObjectWalk) { |
| DepthWalk.ObjectWalk depthWalk = (DepthWalk.ObjectWalk) walker; |
| for (RevObject obj : wantObjs) { |
| depthWalk.markRoot(obj); |
| } |
| // Mark the tree objects associated with "have" commits as |
| // uninteresting to avoid writing redundant blobs. A normal RevWalk |
| // lazily propagates the "uninteresting" state from a commit to its |
| // tree during the walk, but DepthWalks can terminate early so |
| // preemptively propagate that state here. |
| for (RevObject obj : haveObjs) { |
| if (obj instanceof RevCommit) { |
| RevTree t = ((RevCommit) obj).getTree(); |
| depthWalk.markUninteresting(t); |
| } |
| } |
| |
| if (unshallowObjects != null) { |
| for (ObjectId id : unshallowObjects) { |
| depthWalk.markUnshallow(walker.parseAny(id)); |
| } |
| } |
| } else { |
| for (RevObject obj : wantObjs) |
| walker.markStart(obj); |
| } |
| for (RevObject obj : haveObjs) |
| walker.markUninteresting(obj); |
| |
| final int maxBases = config.getDeltaSearchWindowSize(); |
| Set<RevTree> baseTrees = new HashSet<>(); |
| BlockList<RevCommit> commits = new BlockList<>(); |
| Set<ObjectId> roots = new HashSet<>(); |
| RevCommit c; |
| while ((c = walker.next()) != null) { |
| if (exclude(c)) |
| continue; |
| if (c.has(RevFlag.UNINTERESTING)) { |
| if (baseTrees.size() <= maxBases) |
| baseTrees.add(c.getTree()); |
| continue; |
| } |
| |
| commits.add(c); |
| if (c.getParentCount() == 0) { |
| roots.add(c.copy()); |
| } |
| countingMonitor.update(1); |
| } |
| stats.rootCommits = Collections.unmodifiableSet(roots); |
| |
| if (shallowPack) { |
| for (RevCommit cmit : commits) { |
| addObject(cmit, 0); |
| } |
| } else { |
| int commitCnt = 0; |
| boolean putTagTargets = false; |
| for (RevCommit cmit : commits) { |
| if (!cmit.has(added)) { |
| cmit.add(added); |
| addObject(cmit, 0); |
| commitCnt++; |
| } |
| |
| for (int i = 0; i < cmit.getParentCount(); i++) { |
| RevCommit p = cmit.getParent(i); |
| if (!p.has(added) && !p.has(RevFlag.UNINTERESTING) |
| && !exclude(p)) { |
| p.add(added); |
| addObject(p, 0); |
| commitCnt++; |
| } |
| } |
| |
| if (!putTagTargets && 4096 < commitCnt) { |
| for (ObjectId id : tagTargets) { |
| RevObject obj = walker.lookupOrNull(id); |
| if (obj instanceof RevCommit |
| && obj.has(include) |
| && !obj.has(RevFlag.UNINTERESTING) |
| && !obj.has(added)) { |
| obj.add(added); |
| addObject(obj, 0); |
| } |
| } |
| putTagTargets = true; |
| } |
| } |
| } |
| commits = null; |
| |
| if (thin && !baseTrees.isEmpty()) { |
| BaseSearch bases = new BaseSearch(countingMonitor, baseTrees, // |
| objectsMap, edgeObjects, reader); |
| RevObject o; |
| while ((o = walker.nextObject()) != null) { |
| if (o.has(RevFlag.UNINTERESTING)) |
| continue; |
| if (exclude(o)) |
| continue; |
| |
| int pathHash = walker.getPathHashCode(); |
| byte[] pathBuf = walker.getPathBuffer(); |
| int pathLen = walker.getPathLength(); |
| bases.addBase(o.getType(), pathBuf, pathLen, pathHash); |
| if (!depthSkip(o, walker)) { |
| filterAndAddObject(o, o.getType(), pathHash, want); |
| } |
| countingMonitor.update(1); |
| } |
| } else { |
| RevObject o; |
| while ((o = walker.nextObject()) != null) { |
| if (o.has(RevFlag.UNINTERESTING)) |
| continue; |
| if (exclude(o)) |
| continue; |
| if (!depthSkip(o, walker)) { |
| filterAndAddObject(o, o.getType(), walker.getPathHashCode(), |
| want); |
| } |
| countingMonitor.update(1); |
| } |
| } |
| |
| for (CachedPack pack : cachedPacks) |
| countingMonitor.update((int) pack.getObjectCount()); |
| endPhase(countingMonitor); |
| stats.timeCounting = System.currentTimeMillis() - countingStart; |
| stats.bitmapIndexMisses = -1; |
| } |
| |
| private void findObjectsToPackUsingBitmaps( |
| BitmapWalker bitmapWalker, Set<? extends ObjectId> want, |
| Set<? extends ObjectId> have) |
| throws MissingObjectException, IncorrectObjectTypeException, |
| IOException { |
| BitmapBuilder haveBitmap = bitmapWalker.findObjects(have, null, true); |
| BitmapBuilder wantBitmap = bitmapWalker.findObjects(want, haveBitmap, |
| false); |
| BitmapBuilder needBitmap = wantBitmap.andNot(haveBitmap); |
| |
| if (useCachedPacks && reuseSupport != null && !reuseValidate |
| && (excludeInPacks == null || excludeInPacks.length == 0)) |
| cachedPacks.addAll( |
| reuseSupport.getCachedPacksAndUpdate(needBitmap)); |
| |
| for (BitmapObject obj : needBitmap) { |
| ObjectId objectId = obj.getObjectId(); |
| if (exclude(objectId)) { |
| needBitmap.remove(objectId); |
| continue; |
| } |
| filterAndAddObject(objectId, obj.getType(), 0, want); |
| } |
| |
| if (thin) |
| haveObjects = haveBitmap; |
| } |
| |
| private static void pruneEdgesFromObjectList(List<ObjectToPack> list) { |
| final int size = list.size(); |
| int src = 0; |
| int dst = 0; |
| |
| for (; src < size; src++) { |
| ObjectToPack obj = list.get(src); |
| if (obj.isEdge()) |
| continue; |
| if (dst != src) |
| list.set(dst, obj); |
| dst++; |
| } |
| |
| while (dst < list.size()) |
| list.remove(list.size() - 1); |
| } |
| |
| /** |
| * Include one object to the output file. |
| * <p> |
| * Objects are written in the order they are added. If the same object is |
| * added twice, it may be written twice, creating a larger than necessary |
| * file. |
| * |
| * @param object |
| * the object to add. |
| * @throws org.eclipse.jgit.errors.IncorrectObjectTypeException |
| * the object is an unsupported type. |
| */ |
| public void addObject(RevObject object) |
| throws IncorrectObjectTypeException { |
| if (!exclude(object)) |
| addObject(object, 0); |
| } |
| |
| private void addObject(RevObject object, int pathHashCode) { |
| addObject(object, object.getType(), pathHashCode); |
| } |
| |
| private void addObject( |
| final AnyObjectId src, final int type, final int pathHashCode) { |
| final ObjectToPack otp; |
| if (reuseSupport != null) |
| otp = reuseSupport.newObjectToPack(src, type); |
| else |
| otp = new ObjectToPack(src, type); |
| otp.setPathHash(pathHashCode); |
| objectsLists[type].add(otp); |
| objectsMap.add(otp); |
| } |
| |
| /** |
| * Determines if the object should be omitted from the pack as a result of |
| * its depth (probably because of the tree:<depth> filter). |
| * <p> |
| * Causes {@code walker} to skip traversing the current tree, which ought to |
| * have just started traversal, assuming this method is called as soon as a |
| * new depth is reached. |
| * <p> |
| * This method increments the {@code treesTraversed} statistic. |
| * |
| * @param obj |
| * the object to check whether it should be omitted. |
| * @param walker |
| * the walker being used for traveresal. |
| * @return whether the given object should be skipped. |
| */ |
| private boolean depthSkip(@NonNull RevObject obj, ObjectWalk walker) { |
| long treeDepth = walker.getTreeDepth(); |
| |
| // Check if this object needs to be rejected because it is a tree or |
| // blob that is too deep from the root tree. |
| |
| // A blob is considered one level deeper than the tree that contains it. |
| if (obj.getType() == OBJ_BLOB) { |
| treeDepth++; |
| } else { |
| stats.treesTraversed++; |
| } |
| |
| if (filterSpec.getTreeDepthLimit() < 0 || |
| treeDepth <= filterSpec.getTreeDepthLimit()) { |
| return false; |
| } |
| |
| walker.skipTree(); |
| return true; |
| } |
| |
| // Adds the given object as an object to be packed, first performing |
| // filtering on blobs at or exceeding a given size. |
| private void filterAndAddObject(@NonNull AnyObjectId src, int type, |
| int pathHashCode, @NonNull Set<? extends AnyObjectId> want) |
| throws IOException { |
| |
| // Check if this object needs to be rejected, doing the cheaper |
| // checks first. |
| boolean reject = filterSpec.getBlobLimit() >= 0 && |
| type == OBJ_BLOB && |
| !want.contains(src) && |
| reader.getObjectSize(src, OBJ_BLOB) > filterSpec.getBlobLimit(); |
| if (!reject) { |
| addObject(src, type, pathHashCode); |
| } |
| } |
| |
| private boolean exclude(AnyObjectId objectId) { |
| if (excludeInPacks == null) |
| return false; |
| if (excludeInPackLast.contains(objectId)) |
| return true; |
| for (ObjectIdSet idx : excludeInPacks) { |
| if (idx.contains(objectId)) { |
| excludeInPackLast = idx; |
| return true; |
| } |
| } |
| return false; |
| } |
| |
| /** |
| * Select an object representation for this writer. |
| * <p> |
| * An {@link org.eclipse.jgit.lib.ObjectReader} implementation should invoke |
| * this method once for each representation available for an object, to |
| * allow the writer to find the most suitable one for the output. |
| * |
| * @param otp |
| * the object being packed. |
| * @param next |
| * the next available representation from the repository. |
| */ |
| public void select(ObjectToPack otp, StoredObjectRepresentation next) { |
| int nFmt = next.getFormat(); |
| |
| if (!cachedPacks.isEmpty()) { |
| if (otp.isEdge()) |
| return; |
| if (nFmt == PACK_WHOLE || nFmt == PACK_DELTA) { |
| for (CachedPack pack : cachedPacks) { |
| if (pack.hasObject(otp, next)) { |
| otp.setEdge(); |
| otp.clearDeltaBase(); |
| otp.clearReuseAsIs(); |
| pruneCurrentObjectList = true; |
| return; |
| } |
| } |
| } |
| } |
| |
| if (nFmt == PACK_DELTA && reuseDeltas && reuseDeltaFor(otp)) { |
| ObjectId baseId = next.getDeltaBase(); |
| ObjectToPack ptr = objectsMap.get(baseId); |
| if (ptr != null && !ptr.isEdge()) { |
| otp.setDeltaBase(ptr); |
| otp.setReuseAsIs(); |
| } else if (thin && have(ptr, baseId)) { |
| otp.setDeltaBase(baseId); |
| otp.setReuseAsIs(); |
| } else { |
| otp.clearDeltaBase(); |
| otp.clearReuseAsIs(); |
| } |
| } else if (nFmt == PACK_WHOLE && config.isReuseObjects()) { |
| int nWeight = next.getWeight(); |
| if (otp.isReuseAsIs() && !otp.isDeltaRepresentation()) { |
| // We've chosen another PACK_WHOLE format for this object, |
| // choose the one that has the smaller compressed size. |
| // |
| if (otp.getWeight() <= nWeight) |
| return; |
| } |
| otp.clearDeltaBase(); |
| otp.setReuseAsIs(); |
| otp.setWeight(nWeight); |
| } else { |
| otp.clearDeltaBase(); |
| otp.clearReuseAsIs(); |
| } |
| |
| otp.setDeltaAttempted(reuseDeltas && next.wasDeltaAttempted()); |
| otp.select(next); |
| } |
| |
| private final boolean have(ObjectToPack ptr, AnyObjectId objectId) { |
| return (ptr != null && ptr.isEdge()) |
| || (haveObjects != null && haveObjects.contains(objectId)); |
| } |
| |
| /** |
| * Prepares the bitmaps to be written to the bitmap index file. |
| * <p> |
| * Bitmaps can be used to speed up fetches and clones by storing the entire |
| * object graph at selected commits. Writing a bitmap index is an optional |
| * feature that not all pack users may require. |
| * <p> |
| * Called after {@link #writeIndex(OutputStream)}. |
| * <p> |
| * To reduce memory internal state is cleared during this method, rendering |
| * the PackWriter instance useless for anything further than a call to write |
| * out the new bitmaps with {@link #writeBitmapIndex(OutputStream)}. |
| * |
| * @param pm |
| * progress monitor to report bitmap building work. |
| * @return whether a bitmap index may be written. |
| * @throws java.io.IOException |
| * when some I/O problem occur during reading objects. |
| */ |
| public boolean prepareBitmapIndex(ProgressMonitor pm) throws IOException { |
| if (!canBuildBitmaps || getObjectCount() > Integer.MAX_VALUE |
| || !cachedPacks.isEmpty()) |
| return false; |
| |
| if (pm == null) |
| pm = NullProgressMonitor.INSTANCE; |
| |
| int numCommits = objectsLists[OBJ_COMMIT].size(); |
| List<ObjectToPack> byName = sortByName(); |
| sortedByName = null; |
| objectsLists = null; |
| objectsMap = null; |
| writeBitmaps = new PackBitmapIndexBuilder(byName); |
| byName = null; |
| |
| PackWriterBitmapPreparer bitmapPreparer = new PackWriterBitmapPreparer( |
| reader, writeBitmaps, pm, stats.interestingObjects, config); |
| |
| Collection<PackWriterBitmapPreparer.BitmapCommit> selectedCommits = bitmapPreparer |
| .selectCommits(numCommits, excludeFromBitmapSelection); |
| |
| beginPhase(PackingPhase.BUILDING_BITMAPS, pm, selectedCommits.size()); |
| |
| BitmapWalker walker = bitmapPreparer.newBitmapWalker(); |
| AnyObjectId last = null; |
| for (PackWriterBitmapPreparer.BitmapCommit cmit : selectedCommits) { |
| if (!cmit.isReuseWalker()) { |
| walker = bitmapPreparer.newBitmapWalker(); |
| } |
| BitmapBuilder bitmap = walker.findObjects( |
| Collections.singleton(cmit), null, false); |
| |
| if (last != null && cmit.isReuseWalker() && !bitmap.contains(last)) |
| throw new IllegalStateException(MessageFormat.format( |
| JGitText.get().bitmapMissingObject, cmit.name(), |
| last.name())); |
| last = cmit; |
| writeBitmaps.addBitmap(cmit, bitmap.build(), cmit.getFlags()); |
| |
| pm.update(1); |
| } |
| |
| endPhase(pm); |
| return true; |
| } |
| |
| private boolean reuseDeltaFor(ObjectToPack otp) { |
| int type = otp.getType(); |
| if ((type & 2) != 0) // OBJ_TREE(2) or OBJ_BLOB(3) |
| return true; |
| if (type == OBJ_COMMIT) |
| return reuseDeltaCommits; |
| if (type == OBJ_TAG) |
| return false; |
| return true; |
| } |
| |
| private class MutableState { |
| /** Estimated size of a single ObjectToPack instance. */ |
| // Assume 64-bit pointers, since this is just an estimate. |
| private static final long OBJECT_TO_PACK_SIZE = |
| (2 * 8) // Object header |
| + (2 * 8) + (2 * 8) // ObjectToPack fields |
| + (8 + 8) // PackedObjectInfo fields |
| + 8 // ObjectIdOwnerMap fields |
| + 40 // AnyObjectId fields |
| + 8; // Reference in BlockList |
| |
| private final long totalDeltaSearchBytes; |
| |
| private volatile PackingPhase phase; |
| |
| MutableState() { |
| phase = PackingPhase.COUNTING; |
| if (config.isDeltaCompress()) { |
| int threads = config.getThreads(); |
| if (threads <= 0) |
| threads = Runtime.getRuntime().availableProcessors(); |
| totalDeltaSearchBytes = (threads * config.getDeltaSearchMemoryLimit()) |
| + config.getBigFileThreshold(); |
| } else |
| totalDeltaSearchBytes = 0; |
| } |
| |
| State snapshot() { |
| long objCnt = 0; |
| BlockList<ObjectToPack>[] lists = objectsLists; |
| if (lists != null) { |
| objCnt += lists[OBJ_COMMIT].size(); |
| objCnt += lists[OBJ_TREE].size(); |
| objCnt += lists[OBJ_BLOB].size(); |
| objCnt += lists[OBJ_TAG].size(); |
| // Exclude CachedPacks. |
| } |
| |
| long bytesUsed = OBJECT_TO_PACK_SIZE * objCnt; |
| PackingPhase curr = phase; |
| if (curr == PackingPhase.COMPRESSING) |
| bytesUsed += totalDeltaSearchBytes; |
| return new State(curr, bytesUsed); |
| } |
| } |
| |
| /** Possible states that a PackWriter can be in. */ |
| public enum PackingPhase { |
| /** Counting objects phase. */ |
| COUNTING, |
| |
| /** Getting sizes phase. */ |
| GETTING_SIZES, |
| |
| /** Finding sources phase. */ |
| FINDING_SOURCES, |
| |
| /** Compressing objects phase. */ |
| COMPRESSING, |
| |
| /** Writing objects phase. */ |
| WRITING, |
| |
| /** Building bitmaps phase. */ |
| BUILDING_BITMAPS; |
| } |
| |
| /** Summary of the current state of a PackWriter. */ |
| public class State { |
| private final PackingPhase phase; |
| |
| private final long bytesUsed; |
| |
| State(PackingPhase phase, long bytesUsed) { |
| this.phase = phase; |
| this.bytesUsed = bytesUsed; |
| } |
| |
| /** @return the PackConfig used to build the writer. */ |
| public PackConfig getConfig() { |
| return config; |
| } |
| |
| /** @return the current phase of the writer. */ |
| public PackingPhase getPhase() { |
| return phase; |
| } |
| |
| /** @return an estimate of the total memory used by the writer. */ |
| public long estimateBytesUsed() { |
| return bytesUsed; |
| } |
| |
| @SuppressWarnings("nls") |
| @Override |
| public String toString() { |
| return "PackWriter.State[" + phase + ", memory=" + bytesUsed + "]"; |
| } |
| } |
| |
| /** |
| * Configuration related to the packfile URI feature. |
| * |
| * @since 5.5 |
| */ |
| public static class PackfileUriConfig { |
| @NonNull |
| private final PacketLineOut pckOut; |
| |
| @NonNull |
| private final Collection<String> protocolsSupported; |
| |
| @NonNull |
| private final CachedPackUriProvider cachedPackUriProvider; |
| |
| /** |
| * @param pckOut where to write "packfile-uri" lines to (should |
| * output to the same stream as the one passed to |
| * PackWriter#writePack) |
| * @param protocolsSupported list of protocols supported (e.g. "https") |
| * @param cachedPackUriProvider provider of URIs corresponding |
| * to cached packs |
| * @since 5.5 |
| */ |
| public PackfileUriConfig(@NonNull PacketLineOut pckOut, |
| @NonNull Collection<String> protocolsSupported, |
| @NonNull CachedPackUriProvider cachedPackUriProvider) { |
| this.pckOut = pckOut; |
| this.protocolsSupported = protocolsSupported; |
| this.cachedPackUriProvider = cachedPackUriProvider; |
| } |
| } |
| } |