| /* |
| * Copyright 2012-present Facebook, Inc. |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); you may |
| * not use this file except in compliance with the License. You may obtain |
| * a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| * License for the specific language governing permissions and limitations |
| * under the License. |
| */ |
| |
| package com.facebook.buck.rules; |
| |
| import com.facebook.buck.event.BuckEventBus; |
| import com.facebook.buck.event.LogEvent; |
| import com.facebook.buck.step.Step; |
| import com.facebook.buck.step.StepFailedException; |
| import com.facebook.buck.step.StepRunner; |
| import com.facebook.buck.util.BuckConstant; |
| import com.facebook.buck.util.MorePaths; |
| import com.facebook.buck.util.concurrent.MoreFutures; |
| import com.facebook.buck.zip.Unzip; |
| import com.google.common.annotations.Beta; |
| import com.google.common.base.Optional; |
| import com.google.common.base.Preconditions; |
| import com.google.common.base.Supplier; |
| import com.google.common.base.Suppliers; |
| import com.google.common.base.Throwables; |
| import com.google.common.collect.Lists; |
| import com.google.common.util.concurrent.FutureCallback; |
| import com.google.common.util.concurrent.Futures; |
| import com.google.common.util.concurrent.ListenableFuture; |
| import com.google.common.util.concurrent.SettableFuture; |
| |
| import java.io.File; |
| import java.io.IOException; |
| import java.nio.file.Path; |
| import java.nio.file.Paths; |
| import java.util.List; |
| import java.util.concurrent.ExecutionException; |
| import java.util.concurrent.atomic.AtomicBoolean; |
| |
| import javax.annotation.Nullable; |
| |
| /** |
| * Abstract implementation of a {@link BuildRule} that can be cached. If its current {@link RuleKey} |
| * matches the one on disk, then it has no work to do. It should also try to fetch its output from |
| * an {@link ArtifactCache} to avoid doing any computation. |
| */ |
| @Beta |
| public abstract class AbstractCachingBuildRule extends AbstractBuildRule |
| implements BuildRule { |
| |
| private final Buildable buildable; |
| |
| /** |
| * Lock used to ensure that the logic to kick off a build is performed at most once. |
| */ |
| private final AtomicBoolean hasBuildStarted; |
| |
| /** |
| * This is the value returned by {@link #build(BuildContext)}. |
| * This is initialized by the constructor and marked as final because {@link #build(BuildContext)} |
| * must always return the same value. |
| */ |
| private final SettableFuture<BuildRuleSuccess> buildRuleResult; |
| |
| /** @see Buildable#getInputsToCompareToOutput() */ |
| private Iterable<Path> inputsToCompareToOutputs; |
| |
| protected AbstractCachingBuildRule(Buildable buildable, BuildRuleParams params) { |
| super(params); |
| this.buildable = Preconditions.checkNotNull(buildable); |
| this.hasBuildStarted = new AtomicBoolean(false); |
| this.buildRuleResult = SettableFuture.create(); |
| } |
| |
| protected AbstractCachingBuildRule(BuildRuleParams buildRuleParams) { |
| super(buildRuleParams); |
| this.hasBuildStarted = new AtomicBoolean(false); |
| this.buildRuleResult = SettableFuture.create(); |
| this.buildable = Preconditions.checkNotNull(getBuildable()); |
| } |
| |
| /** |
| * This rule is designed to be used for precondition checks in subclasses. For example, before |
| * running the tests associated with a build rule, it is reasonable to do a sanity check to |
| * ensure that the rule has been built. |
| */ |
| protected final synchronized boolean isRuleBuilt() { |
| return MoreFutures.isSuccess(buildRuleResult); |
| } |
| |
| @Override |
| public BuildRuleSuccess.Type getBuildResultType() { |
| Preconditions.checkState(isRuleBuilt()); |
| try { |
| return buildRuleResult.get().getType(); |
| } catch (InterruptedException | ExecutionException e) { |
| throw new RuntimeException(e); |
| } |
| } |
| |
| @Override |
| public Iterable<Path> getInputs() { |
| if (inputsToCompareToOutputs == null) { |
| inputsToCompareToOutputs = MorePaths.asPaths( |
| buildable.getInputsToCompareToOutput()); |
| } |
| return inputsToCompareToOutputs; |
| } |
| |
| @Override |
| public RuleKey.Builder appendToRuleKey(RuleKey.Builder builder) throws IOException { |
| // For a rule that lists its inputs via a "srcs" argument, this may seem redundant, but it is |
| // not. Here, the inputs are specified as InputRules, which means that the _contents_ of the |
| // files will be hashed. In the case of .set("srcs", srcs), the list of strings itself will be |
| // hashed. It turns out that we need both of these in order to construct a RuleKey correctly. |
| builder = super.appendToRuleKey(builder) |
| .setInputs("buck.inputs", getInputs().iterator()); |
| // TODO(simons): Rename this when no Buildables extend this class. |
| return buildable.appendDetailsToRuleKey(builder); |
| } |
| |
| @Override |
| public final ListenableFuture<BuildRuleSuccess> build(final BuildContext context) { |
| // We use hasBuildStarted as a lock so that we can minimize how much we need to synchronize. |
| synchronized(hasBuildStarted) { |
| if (hasBuildStarted.get()) { |
| return buildRuleResult; |
| } else { |
| hasBuildStarted.set(true); |
| } |
| } |
| |
| // Build all of the deps first and then schedule a callback for this rule to build itself once |
| // all of those rules are done building. |
| try { |
| // Invoke every dep's build() method and create an uber-ListenableFuture that represents the |
| // successful completion of all deps. |
| List<ListenableFuture<BuildRuleSuccess>> builtDeps = |
| Lists.newArrayListWithCapacity(getDeps().size()); |
| for (BuildRule dep : getDeps()) { |
| builtDeps.add(dep.build(context)); |
| } |
| ListenableFuture<List<BuildRuleSuccess>> allBuiltDeps = Futures.allAsList(builtDeps); |
| |
| // Schedule this rule to build itself once all of the deps are built. |
| Futures.addCallback(allBuiltDeps, |
| new FutureCallback<List<BuildRuleSuccess>>() { |
| |
| private final BuckEventBus eventBus = context.getEventBus(); |
| |
| private final OnDiskBuildInfo onDiskBuildInfo = context.createOnDiskBuildInfoFor( |
| getBuildTarget()); |
| |
| /** |
| * It is imperative that: |
| * <ol> |
| * <li>The {@link BuildInfoRecorder} is not constructed until all of the |
| * {@link Buildable}'s {@code deps} are guaranteed to be built. This ensures that |
| * the {@link RuleKey} will be available before the {@link BuildInfoRecorder} is |
| * constructed. |
| * <p> |
| * This is why a {@link Supplier} is used. |
| * <li>Only one {@link BuildInfoRecorder} is created per {@link Buildable}. This |
| * ensures that all build-related information for a {@link Buildable} goes though |
| * a single recorder, whose data will be persisted in {@link #onSuccess(List)}. |
| * <p> |
| * This is why {@link Suppliers#memoize(Supplier)} is used. |
| * </ol> |
| */ |
| private final Supplier<BuildInfoRecorder> buildInfoRecorder = Suppliers.memoize( |
| new Supplier<BuildInfoRecorder>() { |
| @Override |
| public BuildInfoRecorder get() { |
| AbstractBuildRule buildRule = AbstractCachingBuildRule.this; |
| RuleKey ruleKey; |
| RuleKey ruleKeyWithoutDeps; |
| try { |
| ruleKey = buildRule.getRuleKey(); |
| ruleKeyWithoutDeps = buildRule.getRuleKeyWithoutDeps(); |
| } catch (IOException e) { |
| throw new RuntimeException(e); |
| } |
| |
| return context.createBuildInfoRecorder( |
| buildRule.getBuildTarget(), ruleKey, ruleKeyWithoutDeps); |
| } |
| }); |
| |
| private boolean startOfBuildWasRecordedOnTheEventBus = false; |
| |
| @Override |
| public void onSuccess(List<BuildRuleSuccess> deps) { |
| // Record the start of the build. |
| eventBus.post(BuildRuleEvent.started(AbstractCachingBuildRule.this)); |
| startOfBuildWasRecordedOnTheEventBus = true; |
| |
| try { |
| BuildResult result = buildOnceDepsAreBuilt( |
| context, onDiskBuildInfo, buildInfoRecorder.get()); |
| if (result.isSuccess()) { |
| recordBuildRuleSuccess(result); |
| } else { |
| recordBuildRuleFailure(result); |
| } |
| } catch (IOException e) { |
| onFailure(e); |
| } |
| } |
| |
| private void recordBuildRuleSuccess(BuildResult result) { |
| // Make sure that all of the local files have the same values they would as if the |
| // rule had been built locally. |
| if (result.success.shouldWriteRecordedMetadataToDiskAfterBuilding()) { |
| try { |
| buildInfoRecorder.get().writeMetadataToDisk(); |
| } catch (IOException e) { |
| onFailure(e); |
| } |
| } |
| |
| // Give the rule a chance to populate its internal data structures now that all of the |
| // files should be in a valid state. |
| if (result.success.shouldInitializeFromDiskAfterBuilding()) { |
| initializeFromDisk(onDiskBuildInfo); |
| } |
| |
| // Only now that the rule should be in a completely valid state, resolve the future. |
| BuildRuleSuccess buildRuleSuccess = new BuildRuleSuccess( |
| AbstractCachingBuildRule.this, result.success); |
| buildRuleResult.set(buildRuleSuccess); |
| |
| // Do the post to the event bus immediately after the future is set so that the |
| // build time measurement is as accurate as possible. |
| eventBus.post(BuildRuleEvent.finished(AbstractCachingBuildRule.this, |
| result.status, |
| result.cacheResult, |
| Optional.of(result.success))); |
| |
| // Finally, upload to the artifact cache. |
| if (result.success.shouldUploadResultingArtifact()) { |
| buildInfoRecorder.get().performUploadToArtifactCache(context.getArtifactCache(), |
| eventBus); |
| } |
| } |
| |
| @Override |
| public void onFailure(Throwable failure) { |
| recordBuildRuleFailure(new BuildResult(failure)); |
| } |
| |
| private void recordBuildRuleFailure(BuildResult result) { |
| // TODO(mbolin): Delete all genfiles and metadata, as they are not guaranteed to be |
| // valid at this point? |
| |
| // Note that startOfBuildWasRecordedOnTheEventBus will be false if onSuccess() was |
| // never invoked. |
| if (startOfBuildWasRecordedOnTheEventBus) { |
| eventBus.post(BuildRuleEvent.finished(AbstractCachingBuildRule.this, |
| result.status, |
| result.cacheResult, |
| Optional.<BuildRuleSuccess.Type>absent())); |
| } |
| |
| // It seems possible (albeit unlikely) that something could go wrong in |
| // recordBuildRuleSuccess() after buildRuleResult has been resolved such that Buck |
| // would attempt to resolve the future again, which would fail. |
| buildRuleResult.setException(result.failure); |
| } |
| }, |
| context.getExecutor()); |
| } catch (Throwable failure) { |
| // This is a defensive catch block: if buildRuleResult is never satisfied, then Buck will |
| // hang because a callback that is waiting for this rule's future to complete will never be |
| // executed. |
| buildRuleResult.setException(failure); |
| } |
| |
| return buildRuleResult; |
| } |
| |
| /** |
| * This method is invoked once all of this rule's dependencies are built. |
| * <p> |
| * This method should be executed on a fresh Runnable in BuildContext's ListeningExecutorService, |
| * so there is no reason to schedule new work in a new Runnable. |
| * <p> |
| * All exit paths through this method should resolve {@link #buildRuleResult} before exiting. To |
| * that end, this method should never throw an exception, or else Buck will hang waiting for |
| * {@link #buildRuleResult} to be resolved. |
| */ |
| private BuildResult buildOnceDepsAreBuilt(final BuildContext context, |
| OnDiskBuildInfo onDiskBuildInfo, |
| BuildInfoRecorder buildInfoRecorder) throws IOException { |
| // Compute the current RuleKey and compare it to the one stored on disk. |
| RuleKey ruleKey = getRuleKey(); |
| Optional<RuleKey> cachedRuleKey = onDiskBuildInfo.getRuleKey(); |
| |
| // If the RuleKeys match, then there is nothing to build. |
| if (ruleKey.equals(cachedRuleKey.orNull())) { |
| context.logBuildInfo("[UNCHANGED %s]", getFullyQualifiedName()); |
| return new BuildResult(BuildRuleSuccess.Type.MATCHING_RULE_KEY, |
| CacheResult.LOCAL_KEY_UNCHANGED_HIT); |
| } |
| |
| // Deciding whether we need to rebuild is tricky business. We want to rebuild as little as |
| // possible while always being sound. |
| // |
| // For java_library rules that depend only on their first-order deps, |
| // they only need to rebuild themselves if any of the following conditions hold: |
| // (1) The definition of the build rule has changed. |
| // (2) Any of the input files (which includes resources as well as .java files) have changed. |
| // (3) The ABI of any of its dependent java_library rules has changed. |
| // |
| // For other types of build rules, we have to be more conservative when rebuilding. In those |
| // cases, we rebuild if any of the following conditions hold: |
| // (1) The definition of the build rule has changed. |
| // (2) Any of the input files have changed. |
| // (3) Any of the RuleKeys of this rule's deps have changed. |
| // |
| // Because a RuleKey for a rule will change if any of its transitive deps have changed, that |
| // means a change in one of the leaves can result in almost all rules being rebuilt, which is |
| // slow. Fortunately, we limit the effects of this when building Java code when checking the ABI |
| // of deps instead of the RuleKey for deps. |
| if (this instanceof AbiRule) { |
| AbiRule abiRule = (AbiRule)this; |
| |
| RuleKey ruleKeyNoDeps = getRuleKeyWithoutDeps(); |
| Optional<RuleKey> cachedRuleKeyNoDeps = onDiskBuildInfo.getRuleKeyWithoutDeps(); |
| if (ruleKeyNoDeps.equals(cachedRuleKeyNoDeps.orNull())) { |
| // The RuleKey for the definition of this build rule and its input files has not changed. |
| // Therefore, if the ABI of its deps has not changed, there is nothing to rebuild. |
| Sha1HashCode abiKeyForDeps = abiRule.getAbiKeyForDeps(); |
| Optional<Sha1HashCode> cachedAbiKeyForDeps = onDiskBuildInfo.getHash( |
| AbiRule.ABI_KEY_FOR_DEPS_ON_DISK_METADATA); |
| if (abiKeyForDeps.equals(cachedAbiKeyForDeps.orNull())) { |
| // Re-copy the ABI metadata. |
| // TODO(mbolin): This seems really bad: there could be other metadata to copy, too? |
| buildInfoRecorder.addMetadata( |
| AbiRule.ABI_KEY_ON_DISK_METADATA, |
| onDiskBuildInfo.getValue(AbiRule.ABI_KEY_ON_DISK_METADATA).get()); |
| buildInfoRecorder.addMetadata( |
| AbiRule.ABI_KEY_FOR_DEPS_ON_DISK_METADATA, |
| cachedAbiKeyForDeps.get().getHash()); |
| return new BuildResult(BuildRuleSuccess.Type.MATCHING_DEPS_ABI_AND_RULE_KEY_NO_DEPS, |
| CacheResult.LOCAL_KEY_UNCHANGED_HIT); |
| } |
| } |
| } |
| |
| // Before deciding to build, check the ArtifactCache. |
| // The fetched file is now a ZIP file, so it needs to be unzipped. |
| CacheResult cacheResult = tryToFetchArtifactFromBuildCacheAndOverlayOnTopOfProjectFilesystem( |
| buildInfoRecorder, |
| context.getArtifactCache(), |
| context.getProjectRoot(), |
| context); |
| |
| // Run the steps to build this rule since it was not found in the cache. |
| if (cacheResult.isSuccess()) { |
| return new BuildResult(BuildRuleSuccess.Type.FETCHED_FROM_CACHE, cacheResult); |
| } |
| |
| // The only remaining option is to build locally. |
| try { |
| executeCommandsNowThatDepsAreBuilt(context, onDiskBuildInfo, buildInfoRecorder); |
| } catch (IOException | StepFailedException e) { |
| return new BuildResult(e); |
| } |
| |
| // Given that the Buildable has built successfully, record that the output file has been |
| // written, assuming it has one. |
| // TODO(mbolin): Buildable.getSteps() should use BuildableContext such that Buildable is |
| // responsible for invoking recordArtifact() itself. Once that is done, this call to |
| // recordArtifact() should be deleted. |
| String pathToOutputFile = buildable.getPathToOutputFile(); |
| if (pathToOutputFile != null && pathToOutputFile.startsWith(BuckConstant.GEN_DIR)) { |
| String prefix = BuckConstant.GEN_DIR + '/' + getBuildTarget().getBasePathWithSlash(); |
| Path pathToArtifact = Paths.get(pathToOutputFile.substring(prefix.length())); |
| buildInfoRecorder.recordArtifact(pathToArtifact); |
| } |
| |
| return new BuildResult(BuildRuleSuccess.Type.BUILT_LOCALLY, CacheResult.MISS); |
| } |
| |
| private CacheResult tryToFetchArtifactFromBuildCacheAndOverlayOnTopOfProjectFilesystem( |
| BuildInfoRecorder buildInfoRecorder, |
| ArtifactCache artifactCache, |
| Path projectRoot, |
| BuildContext buildContext) { |
| // Create a temp file whose extension must be ".zip" for Filesystems.newFileSystem() to infer |
| // that we are creating a zip-based FileSystem. |
| File zipFile; |
| try { |
| zipFile = File.createTempFile(getFullyQualifiedName().replace('/', '_'), ".zip"); |
| } catch (IOException e) { |
| throw new RuntimeException(e); |
| } |
| |
| // TODO(mbolin): Change ArtifactCache.fetch() so that it returns a File instead of takes one. |
| // Then we could download directly from Cassandra into the on-disk cache and unzip it from |
| // there. |
| CacheResult cacheResult = buildInfoRecorder.fetchArtifactForBuildable(zipFile, artifactCache); |
| if (!cacheResult.isSuccess()) { |
| zipFile.delete(); |
| return cacheResult; |
| } |
| |
| // We unzip the file in the root of the project directory. |
| // Ideally, the following would work: |
| // |
| // Path pathToZip = Paths.get(zipFile.getAbsolutePath()); |
| // FileSystem fs = FileSystems.newFileSystem(pathToZip, /* loader */ null); |
| // Path root = Iterables.getOnlyElement(fs.getRootDirectories()); |
| // MoreFiles.copyRecursively(root, projectRoot); |
| // |
| // Unfortunately, this does not appear to work, in practice, because MoreFiles fails when trying |
| // to resolve a Path for a zip entry against a file Path on disk. |
| |
| try { |
| Unzip.extractZipFile(zipFile.getAbsolutePath(), |
| projectRoot.toAbsolutePath().toString(), |
| /* overwriteExistingFiles */ true); |
| } catch (IOException e) { |
| // In the wild, we have seen some inexplicable failures during this step. For now, we try to |
| // give the user as much information as we can to debug the issue, but return CacheResult.MISS |
| // so that Buck will fall back on doing a local build. |
| buildContext.getEventBus().post(LogEvent.warning( |
| "Failed to unzip the artifact for %s at %s.\n" + |
| "The rule will be built locally, but here is the stacktrace of the failed unzip call:\n" + |
| getBuildTarget(), |
| zipFile.getAbsolutePath(), |
| Throwables.getStackTraceAsString(e))); |
| return CacheResult.MISS; |
| } |
| |
| // We only delete the ZIP file when it has been unzipped successfully. Otherwise, we leave it |
| // around for debugging purposes. |
| zipFile.delete(); |
| return cacheResult; |
| } |
| |
| /** |
| * Execute the commands for this build rule. Requires all dependent rules are already built |
| * successfully. |
| */ |
| private void executeCommandsNowThatDepsAreBuilt(BuildContext context, |
| OnDiskBuildInfo onDiskBuildInfo, |
| BuildInfoRecorder buildInfoRecorder) |
| throws IOException, StepFailedException { |
| context.logBuildInfo("[BUILDING %s]", getFullyQualifiedName()); |
| |
| // Get and run all of the commands. |
| BuildableContext buildableContext = new DefaultBuildableContext(onDiskBuildInfo, |
| buildInfoRecorder); |
| List<Step> steps = buildable.getBuildSteps(context, buildableContext); |
| StepRunner stepRunner = context.getStepRunner(); |
| for (Step step : steps) { |
| stepRunner.runStepForBuildTarget(step, getBuildTarget()); |
| } |
| } |
| |
| /** |
| * For a rule that is read from the build cache, it may have fields that would normally be |
| * populated by executing the steps returned by |
| * {@link Buildable#getBuildSteps(BuildContext, BuildableContext)}. Because |
| * {@link Buildable#getBuildSteps(BuildContext, BuildableContext)} is not invoked for cached rules, a rule |
| * may need to implement this method to populate those fields in some other way. For a cached |
| * rule, this method will be invoked just before the future returned by |
| * {@link #build(BuildContext)} is resolved. |
| * @param onDiskBuildInfo can be used to read metadata from disk to help initialize the rule. |
| */ |
| protected void initializeFromDisk(OnDiskBuildInfo onDiskBuildInfo) {} |
| |
| /** |
| * This is a union type that represents either a success or a failure. This exists so that |
| * {@link #buildOnceDepsAreBuilt(BuildContext, OnDiskBuildInfo, BuildInfoRecorder)} can return a |
| * strongly typed value. |
| */ |
| private static class BuildResult { |
| |
| private final BuildRuleStatus status; |
| private final CacheResult cacheResult; |
| |
| @Nullable private final BuildRuleSuccess.Type success; |
| @Nullable private final Throwable failure; |
| |
| BuildResult(BuildRuleSuccess.Type success, CacheResult cacheResult) { |
| this.status = BuildRuleStatus.SUCCESS; |
| this.cacheResult = Preconditions.checkNotNull(cacheResult); |
| this.success = Preconditions.checkNotNull(success); |
| this.failure = null; |
| } |
| |
| BuildResult(Throwable failure) { |
| this.status = BuildRuleStatus.FAIL; |
| this.cacheResult = CacheResult.MISS; |
| this.success = null; |
| this.failure = Preconditions.checkNotNull(failure); |
| } |
| |
| boolean isSuccess() { |
| return status == BuildRuleStatus.SUCCESS; |
| } |
| } |
| } |