| /* |
| * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com> |
| * Copyright (C) 2008, 2022 Shawn O. Pearce <spearce@spearce.org> and others |
| * |
| * This program and the accompanying materials are made available under the |
| * terms of the Eclipse Distribution License v. 1.0 which is available at |
| * https://www.eclipse.org/org/documents/edl-v10.php. |
| * |
| * SPDX-License-Identifier: BSD-3-Clause |
| */ |
| |
| package org.eclipse.jgit.transport; |
| |
| import static java.nio.charset.StandardCharsets.UTF_8; |
| import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; |
| import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; |
| import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NONFASTFORWARD; |
| import static org.eclipse.jgit.transport.ReceiveCommand.Type.UPDATE_NONFASTFORWARD; |
| |
| import java.io.File; |
| import java.io.IOException; |
| import java.io.OutputStreamWriter; |
| import java.io.Writer; |
| import java.text.MessageFormat; |
| import java.util.ArrayList; |
| import java.util.Collection; |
| import java.util.Collections; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.Iterator; |
| import java.util.Map; |
| import java.util.Set; |
| import java.util.concurrent.TimeUnit; |
| import java.util.stream.Collectors; |
| |
| import org.eclipse.jgit.errors.MissingObjectException; |
| import org.eclipse.jgit.errors.NotSupportedException; |
| import org.eclipse.jgit.errors.TransportException; |
| import org.eclipse.jgit.internal.JGitText; |
| import org.eclipse.jgit.internal.storage.file.LockFile; |
| import org.eclipse.jgit.lib.BatchRefUpdate; |
| import org.eclipse.jgit.lib.BatchingProgressMonitor; |
| import org.eclipse.jgit.lib.Constants; |
| import org.eclipse.jgit.lib.ObjectId; |
| import org.eclipse.jgit.lib.ObjectIdRef; |
| import org.eclipse.jgit.lib.ProgressMonitor; |
| import org.eclipse.jgit.lib.Ref; |
| import org.eclipse.jgit.lib.RefDatabase; |
| import org.eclipse.jgit.revwalk.ObjectWalk; |
| import org.eclipse.jgit.revwalk.RevObject; |
| import org.eclipse.jgit.revwalk.RevWalk; |
| import org.eclipse.jgit.util.StringUtils; |
| |
| class FetchProcess { |
| /** Transport we will fetch over. */ |
| private final Transport transport; |
| |
| /** List of things we want to fetch from the remote repository. */ |
| private final Collection<RefSpec> toFetch; |
| |
| /** |
| * List of things we don't want to fetch from the remote repository or to |
| * the local repository. |
| */ |
| private final Collection<RefSpec> negativeRefSpecs; |
| |
| /** Set of refs we will actually wind up asking to obtain. */ |
| private final HashMap<ObjectId, Ref> askFor = new HashMap<>(); |
| |
| /** Objects we know we have locally. */ |
| private final HashSet<ObjectId> have = new HashSet<>(); |
| |
| /** Updates to local tracking branches (if any). */ |
| private final ArrayList<TrackingRefUpdate> localUpdates = new ArrayList<>(); |
| |
| /** Records to be recorded into FETCH_HEAD. */ |
| private final ArrayList<FetchHeadRecord> fetchHeadUpdates = new ArrayList<>(); |
| |
| private final ArrayList<PackLock> packLocks = new ArrayList<>(); |
| |
| private FetchConnection conn; |
| |
| private Map<String, Ref> localRefs; |
| |
| FetchProcess(Transport t, Collection<RefSpec> refSpecs) { |
| transport = t; |
| toFetch = refSpecs.stream().filter(refSpec -> !refSpec.isNegative()) |
| .collect(Collectors.toList()); |
| negativeRefSpecs = refSpecs.stream().filter(RefSpec::isNegative) |
| .collect(Collectors.toList()); |
| } |
| |
| @SuppressWarnings("Finally") |
| void execute(ProgressMonitor monitor, FetchResult result, |
| String initialBranch) |
| throws NotSupportedException, TransportException { |
| askFor.clear(); |
| localUpdates.clear(); |
| fetchHeadUpdates.clear(); |
| packLocks.clear(); |
| localRefs = null; |
| |
| Throwable e1 = null; |
| try { |
| executeImp(monitor, result, initialBranch); |
| } catch (NotSupportedException | TransportException err) { |
| e1 = err; |
| throw err; |
| } finally { |
| try { |
| for (PackLock lock : packLocks) { |
| lock.unlock(); |
| } |
| } catch (Throwable e) { |
| if (e1 != null) { |
| e.addSuppressed(e1); |
| } |
| throw new TransportException(e.getMessage(), e); |
| } |
| } |
| } |
| |
| private boolean isInitialBranchMissing(Map<String, Ref> refsMap, |
| String initialBranch) { |
| if (StringUtils.isEmptyOrNull(initialBranch) || refsMap.isEmpty()) { |
| return false; |
| } |
| if (refsMap.containsKey(initialBranch) |
| || refsMap.containsKey(Constants.R_HEADS + initialBranch) |
| || refsMap.containsKey(Constants.R_TAGS + initialBranch)) { |
| return false; |
| } |
| return true; |
| } |
| |
| private void executeImp(final ProgressMonitor monitor, |
| final FetchResult result, String initialBranch) |
| throws NotSupportedException, TransportException { |
| final TagOpt tagopt = transport.getTagOpt(); |
| String getTags = (tagopt == TagOpt.NO_TAGS) ? null : Constants.R_TAGS; |
| String getHead = null; |
| try { |
| // If we don't have a HEAD yet, we're cloning and need to get the |
| // upstream HEAD, too. |
| Ref head = transport.local.exactRef(Constants.HEAD); |
| ObjectId id = head != null ? head.getObjectId() : null; |
| if (id == null || id.equals(ObjectId.zeroId())) { |
| getHead = Constants.HEAD; |
| } |
| } catch (IOException e) { |
| // Ignore |
| } |
| conn = transport.openFetch(toFetch, getTags, getHead); |
| try { |
| Map<String, Ref> refsMap = conn.getRefsMap(); |
| if (isInitialBranchMissing(refsMap, initialBranch)) { |
| throw new TransportException(MessageFormat.format( |
| JGitText.get().remoteBranchNotFound, initialBranch)); |
| } |
| result.setAdvertisedRefs(transport.getURI(), refsMap); |
| result.peerUserAgent = conn.getPeerUserAgent(); |
| final Set<Ref> matched = new HashSet<>(); |
| for (RefSpec spec : toFetch) { |
| if (spec.getSource() == null) |
| throw new TransportException(MessageFormat.format( |
| JGitText.get().sourceRefNotSpecifiedForRefspec, spec)); |
| |
| if (spec.isWildcard()) |
| expandWildcard(spec, matched); |
| else |
| expandSingle(spec, matched); |
| } |
| |
| Collection<Ref> additionalTags = Collections.<Ref> emptyList(); |
| if (tagopt == TagOpt.AUTO_FOLLOW) |
| additionalTags = expandAutoFollowTags(); |
| else if (tagopt == TagOpt.FETCH_TAGS) |
| expandFetchTags(); |
| |
| final boolean includedTags; |
| if (!askFor.isEmpty() && !askForIsComplete()) { |
| fetchObjects(monitor); |
| includedTags = conn.didFetchIncludeTags(); |
| |
| // Connection was used for object transfer. If we |
| // do another fetch we must open a new connection. |
| // |
| closeConnection(result); |
| } else { |
| includedTags = false; |
| } |
| |
| if (tagopt == TagOpt.AUTO_FOLLOW && !additionalTags.isEmpty()) { |
| // There are more tags that we want to follow, but |
| // not all were asked for on the initial request. |
| // |
| have.addAll(askFor.keySet()); |
| askFor.clear(); |
| for (Ref r : additionalTags) { |
| ObjectId id = r.getPeeledObjectId(); |
| if (id == null) |
| id = r.getObjectId(); |
| if (localHasObject(id)) |
| wantTag(r); |
| } |
| |
| if (!askFor.isEmpty() && (!includedTags || !askForIsComplete())) { |
| reopenConnection(); |
| if (!askFor.isEmpty()) |
| fetchObjects(monitor); |
| } |
| } |
| } finally { |
| closeConnection(result); |
| } |
| |
| BatchRefUpdate batch = transport.local.getRefDatabase() |
| .newBatchUpdate() |
| .setAllowNonFastForwards(true); |
| |
| // Generate reflog only when fetching updates and not at the first clone |
| if (initialBranch == null) { |
| batch.setRefLogMessage("fetch", true); //$NON-NLS-1$ |
| } |
| |
| try (RevWalk walk = new RevWalk(transport.local)) { |
| walk.setRetainBody(false); |
| if (monitor instanceof BatchingProgressMonitor) { |
| ((BatchingProgressMonitor) monitor).setDelayStart( |
| 250, TimeUnit.MILLISECONDS); |
| } |
| if (transport.isRemoveDeletedRefs()) { |
| deleteStaleTrackingRefs(result, batch); |
| } |
| addUpdateBatchCommands(result, batch); |
| for (ReceiveCommand cmd : batch.getCommands()) { |
| cmd.updateType(walk); |
| if (cmd.getType() == UPDATE_NONFASTFORWARD |
| && cmd instanceof TrackingRefUpdate.Command |
| && !((TrackingRefUpdate.Command) cmd).canForceUpdate()) |
| cmd.setResult(REJECTED_NONFASTFORWARD); |
| } |
| if (transport.isDryRun()) { |
| for (ReceiveCommand cmd : batch.getCommands()) { |
| if (cmd.getResult() == NOT_ATTEMPTED) |
| cmd.setResult(OK); |
| } |
| } else { |
| batch.execute(walk, monitor); |
| } |
| } catch (TransportException e) { |
| throw e; |
| } catch (IOException err) { |
| throw new TransportException(MessageFormat.format( |
| JGitText.get().failureUpdatingTrackingRef, |
| getFirstFailedRefName(batch), err.getMessage()), err); |
| } |
| |
| if (!fetchHeadUpdates.isEmpty()) { |
| try { |
| updateFETCH_HEAD(result); |
| } catch (IOException err) { |
| throw new TransportException(MessageFormat.format( |
| JGitText.get().failureUpdatingFETCH_HEAD, err.getMessage()), err); |
| } |
| } |
| } |
| |
| private void addUpdateBatchCommands(FetchResult result, |
| BatchRefUpdate batch) throws TransportException { |
| Map<String, ObjectId> refs = new HashMap<>(); |
| for (TrackingRefUpdate u : localUpdates) { |
| // Try to skip duplicates if they'd update to the same object ID |
| ObjectId existing = refs.get(u.getLocalName()); |
| if (existing == null) { |
| refs.put(u.getLocalName(), u.getNewObjectId()); |
| result.add(u); |
| batch.addCommand(u.asReceiveCommand()); |
| } else if (!existing.equals(u.getNewObjectId())) { |
| throw new TransportException(MessageFormat |
| .format(JGitText.get().duplicateRef, u.getLocalName())); |
| } |
| } |
| } |
| |
| private void fetchObjects(ProgressMonitor monitor) |
| throws TransportException { |
| try { |
| conn.setPackLockMessage("jgit fetch " + transport.uri); //$NON-NLS-1$ |
| conn.fetch(monitor, askFor.values(), have); |
| } finally { |
| packLocks.addAll(conn.getPackLocks()); |
| } |
| if (transport.isCheckFetchedObjects() |
| && !conn.didFetchTestConnectivity() && !askForIsComplete()) |
| throw new TransportException(transport.getURI(), |
| JGitText.get().peerDidNotSupplyACompleteObjectGraph); |
| } |
| |
| private void closeConnection(FetchResult result) { |
| if (conn != null) { |
| conn.close(); |
| result.addMessages(conn.getMessages()); |
| conn = null; |
| } |
| } |
| |
| private void reopenConnection() throws NotSupportedException, |
| TransportException { |
| if (conn != null) |
| return; |
| |
| // Build prefixes |
| Set<String> prefixes = new HashSet<>(); |
| for (Ref toGet : askFor.values()) { |
| String src = toGet.getName(); |
| prefixes.add(src); |
| prefixes.add(Constants.R_REFS + src); |
| prefixes.add(Constants.R_HEADS + src); |
| prefixes.add(Constants.R_TAGS + src); |
| } |
| conn = transport.openFetch(Collections.emptyList(), |
| prefixes.toArray(new String[0])); |
| |
| // Since we opened a new connection we cannot be certain |
| // that the system we connected to has the same exact set |
| // of objects available (think round-robin DNS and mirrors |
| // that aren't updated at the same time). |
| // |
| // We rebuild our askFor list using only the refs that the |
| // new connection has offered to us. |
| // |
| final HashMap<ObjectId, Ref> avail = new HashMap<>(); |
| for (Ref r : conn.getRefs()) |
| avail.put(r.getObjectId(), r); |
| |
| final Collection<Ref> wants = new ArrayList<>(askFor.values()); |
| askFor.clear(); |
| for (Ref want : wants) { |
| final Ref newRef = avail.get(want.getObjectId()); |
| if (newRef != null) { |
| askFor.put(newRef.getObjectId(), newRef); |
| } else { |
| removeFetchHeadRecord(want.getObjectId()); |
| removeTrackingRefUpdate(want.getObjectId()); |
| } |
| } |
| } |
| |
| private void removeTrackingRefUpdate(ObjectId want) { |
| final Iterator<TrackingRefUpdate> i = localUpdates.iterator(); |
| while (i.hasNext()) { |
| final TrackingRefUpdate u = i.next(); |
| if (u.getNewObjectId().equals(want)) |
| i.remove(); |
| } |
| } |
| |
| private void removeFetchHeadRecord(ObjectId want) { |
| final Iterator<FetchHeadRecord> i = fetchHeadUpdates.iterator(); |
| while (i.hasNext()) { |
| final FetchHeadRecord fh = i.next(); |
| if (fh.newValue.equals(want)) |
| i.remove(); |
| } |
| } |
| |
| private void updateFETCH_HEAD(FetchResult result) throws IOException { |
| File meta = transport.local.getDirectory(); |
| if (meta == null) |
| return; |
| final LockFile lock = new LockFile(new File(meta, "FETCH_HEAD")); //$NON-NLS-1$ |
| try { |
| if (lock.lock()) { |
| try (Writer w = new OutputStreamWriter( |
| lock.getOutputStream(), UTF_8)) { |
| for (FetchHeadRecord h : fetchHeadUpdates) { |
| h.write(w); |
| result.add(h); |
| } |
| } |
| lock.commit(); |
| } |
| } finally { |
| lock.unlock(); |
| } |
| } |
| |
| private boolean askForIsComplete() throws TransportException { |
| try { |
| try (ObjectWalk ow = new ObjectWalk(transport.local)) { |
| boolean hasCommitObject = false; |
| for (ObjectId want : askFor.keySet()) { |
| RevObject obj = ow.parseAny(want); |
| ow.markStart(obj); |
| hasCommitObject |= obj.getType() == Constants.OBJ_COMMIT; |
| } |
| // Checking connectivity makes sense on commits only |
| if (hasCommitObject) { |
| for (Ref ref : localRefs().values()) { |
| ow.markUninteresting(ow.parseAny(ref.getObjectId())); |
| } |
| ow.checkConnectivity(); |
| } |
| } |
| return transport.getDepth() == null; // if depth is set we need to request objects that are already available |
| } catch (MissingObjectException e) { |
| return false; |
| } catch (IOException e) { |
| throw new TransportException(JGitText.get().unableToCheckConnectivity, e); |
| } |
| } |
| |
| private void expandWildcard(RefSpec spec, Set<Ref> matched) |
| throws TransportException { |
| for (Ref src : conn.getRefs()) { |
| if (spec.matchSource(src)) { |
| RefSpec expandedRefSpec = spec.expandFromSource(src); |
| if (!matchNegativeRefSpec(expandedRefSpec) |
| && matched.add(src)) { |
| want(src, expandedRefSpec); |
| } |
| } |
| } |
| } |
| |
| private void expandSingle(RefSpec spec, Set<Ref> matched) |
| throws TransportException { |
| String want = spec.getSource(); |
| if (ObjectId.isId(want)) { |
| want(ObjectId.fromString(want)); |
| return; |
| } |
| |
| Ref src = conn.getRef(want); |
| if (src == null) { |
| throw new TransportException(MessageFormat.format(JGitText.get().remoteDoesNotHaveSpec, want)); |
| } |
| if (!matchNegativeRefSpec(spec) && matched.add(src)) { |
| want(src, spec); |
| } |
| } |
| |
| private boolean matchNegativeRefSpec(RefSpec spec) { |
| for (RefSpec negativeRefSpec : negativeRefSpecs) { |
| if (negativeRefSpec.getSource() != null && spec.getSource() != null |
| && negativeRefSpec.matchSource(spec.getSource())) { |
| return true; |
| } |
| |
| if (negativeRefSpec.getDestination() != null |
| && spec.getDestination() != null && negativeRefSpec |
| .matchDestination(spec.getDestination())) { |
| return true; |
| } |
| } |
| return false; |
| } |
| |
| private boolean localHasObject(ObjectId id) throws TransportException { |
| try { |
| return transport.local.getObjectDatabase().has(id); |
| } catch (IOException err) { |
| throw new TransportException( |
| MessageFormat.format( |
| JGitText.get().readingObjectsFromLocalRepositoryFailed, |
| err.getMessage()), |
| err); |
| } |
| } |
| |
| private Collection<Ref> expandAutoFollowTags() throws TransportException { |
| final Collection<Ref> additionalTags = new ArrayList<>(); |
| final Map<String, Ref> haveRefs = localRefs(); |
| for (Ref r : conn.getRefs()) { |
| if (!isTag(r)) |
| continue; |
| |
| Ref local = haveRefs.get(r.getName()); |
| if (local != null) |
| // We already have a tag with this name, don't fetch it (even if |
| // the local is different). |
| continue; |
| |
| ObjectId obj = r.getPeeledObjectId(); |
| if (obj == null) |
| obj = r.getObjectId(); |
| |
| if (askFor.containsKey(obj) || localHasObject(obj)) |
| wantTag(r); |
| else |
| additionalTags.add(r); |
| } |
| return additionalTags; |
| } |
| |
| private void expandFetchTags() throws TransportException { |
| final Map<String, Ref> haveRefs = localRefs(); |
| for (Ref r : conn.getRefs()) { |
| if (!isTag(r)) { |
| continue; |
| } |
| ObjectId id = r.getObjectId(); |
| if (id == null) { |
| continue; |
| } |
| final Ref local = haveRefs.get(r.getName()); |
| if (local == null || !id.equals(local.getObjectId())) { |
| wantTag(r); |
| } |
| } |
| } |
| |
| private void wantTag(Ref r) throws TransportException { |
| want(r, new RefSpec().setSource(r.getName()) |
| .setDestination(r.getName()).setForceUpdate(true)); |
| } |
| |
| private void want(Ref src, RefSpec spec) |
| throws TransportException { |
| final ObjectId newId = src.getObjectId(); |
| if (newId == null) { |
| throw new NullPointerException(MessageFormat.format( |
| JGitText.get().transportProvidedRefWithNoObjectId, |
| src.getName())); |
| } |
| if (spec.getDestination() != null) { |
| final TrackingRefUpdate tru = createUpdate(spec, newId); |
| // if depth is set we need to update the ref |
| if (newId.equals(tru.getOldObjectId()) && transport.getDepth() == null) { |
| return; |
| } |
| localUpdates.add(tru); |
| } |
| |
| askFor.put(newId, src); |
| |
| final FetchHeadRecord fhr = new FetchHeadRecord(); |
| fhr.newValue = newId; |
| fhr.notForMerge = spec.getDestination() != null; |
| fhr.sourceName = src.getName(); |
| fhr.sourceURI = transport.getURI(); |
| fetchHeadUpdates.add(fhr); |
| } |
| |
| private void want(ObjectId id) { |
| askFor.put(id, |
| new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK, id.name(), id)); |
| } |
| |
| private TrackingRefUpdate createUpdate(RefSpec spec, ObjectId newId) |
| throws TransportException { |
| Ref ref = localRefs().get(spec.getDestination()); |
| ObjectId oldId = ref != null && ref.getObjectId() != null |
| ? ref.getObjectId() |
| : ObjectId.zeroId(); |
| return new TrackingRefUpdate( |
| spec.isForceUpdate(), |
| spec.getSource(), |
| spec.getDestination(), |
| oldId, |
| newId); |
| } |
| |
| private Map<String, Ref> localRefs() throws TransportException { |
| if (localRefs == null) { |
| try { |
| localRefs = transport.local.getRefDatabase() |
| .getRefs(RefDatabase.ALL); |
| } catch (IOException err) { |
| throw new TransportException(JGitText.get().cannotListRefs, err); |
| } |
| } |
| return localRefs; |
| } |
| |
| private void deleteStaleTrackingRefs(FetchResult result, |
| BatchRefUpdate batch) throws IOException { |
| Set<Ref> processed = new HashSet<>(); |
| for (Ref ref : localRefs().values()) { |
| if (ref.isSymbolic()) { |
| continue; |
| } |
| String refname = ref.getName(); |
| for (RefSpec spec : toFetch) { |
| if (spec.matchDestination(refname)) { |
| RefSpec s = spec.expandFromDestination(refname); |
| if (result.getAdvertisedRef(s.getSource()) == null |
| && processed.add(ref)) { |
| deleteTrackingRef(result, batch, s, ref); |
| } |
| } |
| } |
| } |
| } |
| |
| private void deleteTrackingRef(final FetchResult result, |
| final BatchRefUpdate batch, final RefSpec spec, final Ref localRef) { |
| if (localRef.getObjectId() == null) |
| return; |
| TrackingRefUpdate update = new TrackingRefUpdate( |
| true, |
| spec.getSource(), |
| localRef.getName(), |
| localRef.getObjectId(), |
| ObjectId.zeroId()); |
| result.add(update); |
| batch.addCommand(update.asReceiveCommand()); |
| } |
| |
| private static boolean isTag(Ref r) { |
| return isTag(r.getName()); |
| } |
| |
| private static boolean isTag(String name) { |
| return name.startsWith(Constants.R_TAGS); |
| } |
| |
| private static String getFirstFailedRefName(BatchRefUpdate batch) { |
| for (ReceiveCommand cmd : batch.getCommands()) { |
| if (cmd.getResult() != ReceiveCommand.Result.OK) |
| return cmd.getRefName(); |
| } |
| return ""; //$NON-NLS-1$ |
| } |
| } |