| /* |
| * Copyright (C) 2011, Google Inc. |
| * and other copyright owners as documented in the project's IP log. |
| * |
| * This program and the accompanying materials are made available |
| * under the terms of the Eclipse Distribution License v1.0 which |
| * accompanies this distribution, is reproduced below, and is |
| * available at http://www.eclipse.org/org/documents/edl-v10.php |
| * |
| * All rights reserved. |
| * |
| * Redistribution and use in source and binary forms, with or |
| * without modification, are permitted provided that the following |
| * conditions are met: |
| * |
| * - Redistributions of source code must retain the above copyright |
| * notice, this list of conditions and the following disclaimer. |
| * |
| * - Redistributions in binary form must reproduce the above |
| * copyright notice, this list of conditions and the following |
| * disclaimer in the documentation and/or other materials provided |
| * with the distribution. |
| * |
| * - Neither the name of the Eclipse Foundation, Inc. nor the |
| * names of its contributors may be used to endorse or promote |
| * products derived from this software without specific prior |
| * written permission. |
| * |
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
| * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
| * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
| * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
| * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
| * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
| * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
| * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
| * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| */ |
| |
| package org.eclipse.jgit.storage.dht; |
| |
| import java.util.Collections; |
| import java.util.Comparator; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.LinkedHashMap; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Set; |
| import java.util.concurrent.Semaphore; |
| import java.util.concurrent.atomic.AtomicReference; |
| |
| import org.eclipse.jgit.generated.storage.dht.proto.GitStore.ChunkMeta; |
| import org.eclipse.jgit.storage.dht.spi.Context; |
| import org.eclipse.jgit.util.BlockList; |
| |
| /** |
| * Re-orders objects destined for a pack stream by chunk locality. |
| * <p> |
| * By re-ordering objects according to chunk locality, and then the original |
| * order the PackWriter intended to use, objects can be copied quickly from |
| * chunks, and each chunk is visited at most once. A {@link Prefetcher} for the |
| * {@link DhtReader} is used to fetch chunks in the order they will be used, |
| * improving throughput by reducing the number of round-trips required to the |
| * storage system. |
| */ |
| final class ObjectWriter { |
| private final DhtReader ctx; |
| |
| private final Prefetcher prefetch; |
| |
| private final int batchSize; |
| |
| private final Semaphore metaBatches; |
| |
| private final AtomicReference<DhtException> metaError; |
| |
| private final LinkedHashMap<ChunkKey, Integer> allVisits; |
| |
| private final Map<ChunkKey, ChunkMeta> allMeta; |
| |
| private final Set<ChunkKey> metaMissing; |
| |
| private Set<ChunkKey> metaToRead; |
| |
| private int curVisit; |
| |
| ObjectWriter(DhtReader ctx, Prefetcher prefetch) { |
| this.ctx = ctx; |
| this.prefetch = prefetch; |
| |
| batchSize = ctx.getOptions().getObjectIndexBatchSize(); |
| metaBatches = new Semaphore(batchSize); |
| metaError = new AtomicReference<DhtException>(); |
| |
| allVisits = new LinkedHashMap<ChunkKey, Integer>(); |
| allMeta = new HashMap<ChunkKey, ChunkMeta>(); |
| metaMissing = new HashSet<ChunkKey>(); |
| metaToRead = new HashSet<ChunkKey>(); |
| curVisit = 1; |
| } |
| |
| void plan(List<DhtObjectToPack> list) throws DhtException { |
| try { |
| for (DhtObjectToPack obj : list) |
| visit(obj); |
| |
| if (!metaToRead.isEmpty()) |
| startBatch(Context.FAST_MISSING_OK); |
| awaitPendingBatches(); |
| |
| synchronized (metaMissing) { |
| if (!metaMissing.isEmpty()) { |
| metaBatches.release(batchSize); |
| resolveMissing(); |
| awaitPendingBatches(); |
| } |
| } |
| } catch (InterruptedException err) { |
| throw new DhtTimeoutException(err); |
| } |
| |
| Iterable<ChunkKey> order; |
| synchronized (allMeta) { |
| if (allMeta.isEmpty()) { |
| order = allVisits.keySet(); |
| } else { |
| BlockList<ChunkKey> keys = new BlockList<ChunkKey>(); |
| for (ChunkKey key : allVisits.keySet()) { |
| keys.add(key); |
| |
| ChunkMeta meta = allMeta.remove(key); |
| if (meta != null) { |
| for (int i = 1; i < meta.getFragmentCount(); i++) |
| keys.add(ChunkKey.fromString(meta.getFragment(i))); |
| } |
| } |
| order = keys; |
| } |
| } |
| prefetch.push(order); |
| |
| Collections.sort(list, new Comparator<DhtObjectToPack>() { |
| public int compare(DhtObjectToPack a, DhtObjectToPack b) { |
| return a.visitOrder - b.visitOrder; |
| } |
| }); |
| } |
| |
| private void visit(DhtObjectToPack obj) throws InterruptedException, |
| DhtTimeoutException { |
| // Plan the visit to the delta base before the object. This |
| // ensures the base is in the stream first, and OFS_DELTA can |
| // be used for the delta. |
| // |
| DhtObjectToPack base = (DhtObjectToPack) obj.getDeltaBase(); |
| if (base != null && base.visitOrder == 0) { |
| // Use the current visit, even if its wrong. This will |
| // prevent infinite recursion when there is a cycle in the |
| // delta chain. Cycles are broken during writing, not in |
| // the earlier planning phases. |
| // |
| obj.visitOrder = curVisit; |
| visit(base); |
| } |
| |
| ChunkKey key = obj.chunk; |
| if (key != null) { |
| Integer i = allVisits.get(key); |
| if (i == null) { |
| i = Integer.valueOf(1 + allVisits.size()); |
| allVisits.put(key, i); |
| } |
| curVisit = i.intValue(); |
| } |
| |
| if (obj.isFragmented()) { |
| metaToRead.add(key); |
| if (metaToRead.size() == batchSize) |
| startBatch(Context.FAST_MISSING_OK); |
| } |
| obj.visitOrder = curVisit; |
| } |
| |
| private void resolveMissing() throws DhtTimeoutException, |
| InterruptedException { |
| metaToRead = new HashSet<ChunkKey>(); |
| for (ChunkKey key : metaMissing) { |
| metaToRead.add(key); |
| if (metaToRead.size() == batchSize) |
| startBatch(Context.LOCAL); |
| } |
| if (!metaToRead.isEmpty()) |
| startBatch(Context.LOCAL); |
| } |
| |
| private void startBatch(Context context) throws InterruptedException, |
| DhtTimeoutException { |
| Timeout to = ctx.getOptions().getTimeout(); |
| if (!metaBatches.tryAcquire(1, to.getTime(), to.getUnit())) |
| throw new DhtTimeoutException(DhtText.get().timeoutChunkMeta); |
| |
| Set<ChunkKey> keys = metaToRead; |
| ctx.getDatabase().chunk().getMeta( |
| context, |
| keys, |
| new MetaLoader(context, keys)); |
| metaToRead = new HashSet<ChunkKey>(); |
| } |
| |
| private void awaitPendingBatches() throws InterruptedException, |
| DhtTimeoutException, DhtException { |
| Timeout to = ctx.getOptions().getTimeout(); |
| if (!metaBatches.tryAcquire(batchSize, to.getTime(), to.getUnit())) |
| throw new DhtTimeoutException(DhtText.get().timeoutChunkMeta); |
| if (metaError.get() != null) |
| throw metaError.get(); |
| } |
| |
| private class MetaLoader implements AsyncCallback<Map<ChunkKey, ChunkMeta>> { |
| private final Context context; |
| |
| private final Set<ChunkKey> keys; |
| |
| MetaLoader(Context context, Set<ChunkKey> keys) { |
| this.context = context; |
| this.keys = keys; |
| } |
| |
| public void onSuccess(Map<ChunkKey, ChunkMeta> result) { |
| try { |
| synchronized (allMeta) { |
| allMeta.putAll(result); |
| keys.removeAll(result.keySet()); |
| } |
| if (context == Context.FAST_MISSING_OK && !keys.isEmpty()) { |
| synchronized (metaMissing) { |
| metaMissing.addAll(keys); |
| } |
| } |
| } finally { |
| metaBatches.release(1); |
| } |
| } |
| |
| public void onFailure(DhtException error) { |
| metaError.compareAndSet(null, error); |
| metaBatches.release(1); |
| } |
| } |
| } |