src/com/facebook/buck/rules/BuildInfoRecorder.java - buck - Git at Google

 /*
  * Copyright 2013-present Facebook, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you may
  * not use this file except in compliance with the License. You may obtain
  * a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  * License for the specific language governing permissions and limitations
  * under the License.
  */

 package com.facebook.buck.rules;

 import com.facebook.buck.event.BuckEventBus;
 import com.facebook.buck.event.ConsoleEvent;
 import com.facebook.buck.io.DefaultDirectoryTraverser;
 import com.facebook.buck.io.DirectoryTraversal;
 import com.facebook.buck.io.DirectoryTraverser;
 import com.facebook.buck.io.MoreFiles;
 import com.facebook.buck.io.ProjectFilesystem;
 import com.facebook.buck.model.BuildId;
 import com.facebook.buck.model.BuildTarget;
 import com.facebook.buck.timing.Clock;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Function;
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import com.google.gson.JsonArray;
 import com.google.gson.JsonPrimitive;

 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;

 import javax.annotation.Nullable;

 /**
  * Utility for recording the paths to the output files generated by a build rule, as well as any
  * metadata about those output files. This data will be packaged up into an artifact that will be
  * stored in the cache. The metadata will also be written to disk so it can be read on a subsequent
  * build by an {@link OnDiskBuildInfo}.
  */
 public class BuildInfoRecorder {

   @VisibleForTesting
   static final String ABSOLUTE_PATH_ERROR_FORMAT =
       "Error! '%s' is trying to record artifacts with absolute path: '%s'.";

   private static final DirectoryTraverser DEFAULT_DIRECTORY_TRAVERSER =
       new DefaultDirectoryTraverser();
   private static final Path PATH_TO_ARTIFACT_INFO = Paths.get("buck-out/log/cache_artifact.txt");
   private static final String BUCK_CACHE_DATA_ENV_VAR = "BUCK_CACHE_DATA";

   private final BuildTarget buildTarget;
   private final Path pathToMetadataDirectory;
   private final ProjectFilesystem projectFilesystem;
   private final Clock clock;
   private final BuildId buildId;
   private final String artifactExtraData;
   private final Map<String, String> metadataToWrite;
   private final RuleKey ruleKey;

   /**
    * Every value in this set is a path relative to the project root.
    */
   private final Set<Path> pathsToOutputFiles;

   private final Set<Path> pathsToOutputDirectories;
   private final DirectoryTraverser directoryTraverser;

   BuildInfoRecorder(BuildTarget buildTarget,
       ProjectFilesystem projectFilesystem,
       Clock clock,
       BuildId buildId,
       ImmutableMap<String, String> environment,
       RuleKey ruleKey,
       RuleKey rukeKeyWithoutDeps) {
     this(
         buildTarget,
         projectFilesystem,
         clock,
         buildId,
         environment,
         ruleKey,
         rukeKeyWithoutDeps,
         DEFAULT_DIRECTORY_TRAVERSER);
   }

   BuildInfoRecorder(BuildTarget buildTarget,
       ProjectFilesystem projectFilesystem,
       Clock clock,
       BuildId buildId,
       ImmutableMap<String, String> environment,
       RuleKey ruleKey,
       RuleKey rukeKeyWithoutDeps,
       DirectoryTraverser directoryTraverser) {
     this.buildTarget = buildTarget;
     this.pathToMetadataDirectory = BuildInfo.getPathToMetadataDirectory(buildTarget);
     this.projectFilesystem = projectFilesystem;
     this.clock = clock;
     this.buildId = buildId;

     this.artifactExtraData =
         String.format("artifact_data=%s", environment.get(BUCK_CACHE_DATA_ENV_VAR));

     this.metadataToWrite = Maps.newHashMap();

     metadataToWrite.put(BuildInfo.METADATA_KEY_FOR_RULE_KEY,
         ruleKey.toString());
     metadataToWrite.put(BuildInfo.METADATA_KEY_FOR_RULE_KEY_WITHOUT_DEPS,
         rukeKeyWithoutDeps.toString());
     this.ruleKey = ruleKey;
     this.pathsToOutputFiles = Sets.newHashSet();
     this.pathsToOutputDirectories = Sets.newHashSet();
     this.directoryTraverser = directoryTraverser;
   }

   /**
    * Writes the metadata currently stored in memory to the directory returned by
    * {@link BuildInfo#getPathToMetadataDirectory(BuildTarget)}.
    */
   public void writeMetadataToDisk(boolean clearExistingMetadata) throws IOException {
     if (clearExistingMetadata) {
       projectFilesystem.rmdir(pathToMetadataDirectory);
     }
     projectFilesystem.mkdirs(pathToMetadataDirectory);

     for (Map.Entry<String, String> entry : metadataToWrite.entrySet()) {
       projectFilesystem.writeContentsToPath(
           entry.getValue(),
           pathToMetadataDirectory.resolve(entry.getKey()));
     }
   }

   /**
    * This key/value pair is stored in memory until {@link #writeMetadataToDisk(boolean)} is invoked.
    */
   public void addMetadata(String key, String value) {
     metadataToWrite.put(key, value);
   }

   public void addMetadata(String key, Iterable<String> value) {
     JsonArray values = new JsonArray();
     for (String str : value) {
       values.add(new JsonPrimitive(str));
     }
     addMetadata(key, values.toString());
   }

   /**
    * Creates a zip file of the metadata and recorded artifacts and stores it in the artifact cache.
    */
   public void performUploadToArtifactCache(ArtifactCache artifactCache, BuckEventBus eventBus)
       throws InterruptedException {
     // Skip all of this if caching is disabled. Although artifactCache.store() will be a noop,
     // building up the zip is wasted I/O.
     if (!artifactCache.isStoreSupported()) {
       return;
     }

     ImmutableSet.Builder<Path> pathsToIncludeInZipBuilder = ImmutableSet.<Path>builder()
         .addAll(Iterables.transform(metadataToWrite.keySet(),
             new Function<String, Path>() {
               @Override
               public Path apply(String key) {
                 return pathToMetadataDirectory.resolve(key);
               }
             }))
         .addAll(pathsToOutputFiles);

     try {
       for (Path outputDirectory : pathsToOutputDirectories) {
         pathsToIncludeInZipBuilder.addAll(getEntries(outputDirectory));
       }
     } catch (IOException e) {
       throw new RuntimeException(e);
     }

     ImmutableSet<Path> pathsToIncludeInZip = pathsToIncludeInZipBuilder.build();
     File zip;
     try {
       zip = File.createTempFile(
           MoreFiles.sanitize(buildTarget.getFullyQualifiedName()),
           ".zip");
       long time = TimeUnit.MILLISECONDS.toSeconds(clock.currentTimeMillis());
       String additionalArtifactInfo = String.format(
           "build_id=%s\ntimestamp=%d\n%s\n",
           buildId,
           time,
           artifactExtraData);
       projectFilesystem.createZip(
           pathsToIncludeInZip,
           zip,
           ImmutableMap.of(PATH_TO_ARTIFACT_INFO, additionalArtifactInfo));
     } catch (IOException e) {
       eventBus.post(ConsoleEvent.info("Failed to create zip for %s containing:\n%s",
           buildTarget,
           Joiner.on('\n').join(ImmutableSortedSet.copyOf(pathsToIncludeInZip))));
       e.printStackTrace();
       return;
     }
     artifactCache.store(ruleKey, zip);
     zip.delete();
   }

   private List<Path> getEntries(final Path outputDirectory) throws IOException {
     final ImmutableList.Builder<Path> entries = ImmutableList.builder();
     DirectoryTraversal traversal = new DirectoryTraversal(
         projectFilesystem.getFileForRelativePath(outputDirectory)) {
           @Override
           public void visit(File file, String relativePath) throws IOException {
             entries.add(outputDirectory.resolve(relativePath));
           }
           @Override
           public void visitDirectory(File directory, String relativePath) throws IOException {
             entries.add(outputDirectory.resolve(relativePath));
           }
     };
     directoryTraverser.traverse(traversal);
     return entries.build();
   }

   /**
    * Fetches the artifact associated with the {@link #buildTarget} for this class and writes it to
    * the specified {@code outputFile}.
    */
   public CacheResult fetchArtifactForBuildable(File outputFile, ArtifactCache artifactCache)
       throws InterruptedException {
     return artifactCache.fetch(ruleKey, outputFile);
   }

   /**
    * @param pathToArtifact Relative path to the project root.
    */
   public void recordArtifact(Path pathToArtifact) {
     Preconditions.checkArgument(
         !pathToArtifact.isAbsolute(),
         ABSOLUTE_PATH_ERROR_FORMAT,
         buildTarget,
         pathToArtifact);
     pathsToOutputFiles.add(pathToArtifact);
   }

   public void recordArtifactsInDirectory(Path pathToArtifactsDirectory) {
     Preconditions.checkArgument(
         !pathToArtifactsDirectory.isAbsolute(),
         ABSOLUTE_PATH_ERROR_FORMAT,
         buildTarget,
         pathToArtifactsDirectory);
     pathsToOutputDirectories.add(pathToArtifactsDirectory);
   }

   @Nullable
   @VisibleForTesting
   String getMetadataFor(String key) {
     return metadataToWrite.get(key);
   }
 }
	/*
	* Copyright 2013-present Facebook, Inc.
	*
	* Licensed under the Apache License, Version 2.0 (the "License"); you may
	* not use this file except in compliance with the License. You may obtain
	* a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
	* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
	* License for the specific language governing permissions and limitations
	* under the License.
	*/

	package com.facebook.buck.rules;

	import com.facebook.buck.event.BuckEventBus;
	import com.facebook.buck.event.ConsoleEvent;
	import com.facebook.buck.io.DefaultDirectoryTraverser;
	import com.facebook.buck.io.DirectoryTraversal;
	import com.facebook.buck.io.DirectoryTraverser;
	import com.facebook.buck.io.MoreFiles;
	import com.facebook.buck.io.ProjectFilesystem;
	import com.facebook.buck.model.BuildId;
	import com.facebook.buck.model.BuildTarget;
	import com.facebook.buck.timing.Clock;
	import com.google.common.annotations.VisibleForTesting;
	import com.google.common.base.Function;
	import com.google.common.base.Joiner;
	import com.google.common.base.Preconditions;
	import com.google.common.collect.ImmutableList;
	import com.google.common.collect.ImmutableMap;
	import com.google.common.collect.ImmutableSet;
	import com.google.common.collect.ImmutableSortedSet;
	import com.google.common.collect.Iterables;
	import com.google.common.collect.Maps;
	import com.google.common.collect.Sets;
	import com.google.gson.JsonArray;
	import com.google.gson.JsonPrimitive;

	import java.io.File;
	import java.io.IOException;
	import java.nio.file.Path;
	import java.nio.file.Paths;
	import java.util.List;
	import java.util.Map;
	import java.util.Set;
	import java.util.concurrent.TimeUnit;

	import javax.annotation.Nullable;

	/**
	* Utility for recording the paths to the output files generated by a build rule, as well as any
	* metadata about those output files. This data will be packaged up into an artifact that will be
	* stored in the cache. The metadata will also be written to disk so it can be read on a subsequent
	* build by an {@link OnDiskBuildInfo}.
	*/
	public class BuildInfoRecorder {

	@VisibleForTesting
	static final String ABSOLUTE_PATH_ERROR_FORMAT =
	"Error! '%s' is trying to record artifacts with absolute path: '%s'.";

	private static final DirectoryTraverser DEFAULT_DIRECTORY_TRAVERSER =
	new DefaultDirectoryTraverser();
	private static final Path PATH_TO_ARTIFACT_INFO = Paths.get("buck-out/log/cache_artifact.txt");
	private static final String BUCK_CACHE_DATA_ENV_VAR = "BUCK_CACHE_DATA";

	private final BuildTarget buildTarget;
	private final Path pathToMetadataDirectory;
	private final ProjectFilesystem projectFilesystem;
	private final Clock clock;
	private final BuildId buildId;
	private final String artifactExtraData;
	private final Map<String, String> metadataToWrite;
	private final RuleKey ruleKey;

	/**
	* Every value in this set is a path relative to the project root.
	*/
	private final Set<Path> pathsToOutputFiles;

	private final Set<Path> pathsToOutputDirectories;
	private final DirectoryTraverser directoryTraverser;

	BuildInfoRecorder(BuildTarget buildTarget,
	ProjectFilesystem projectFilesystem,
	Clock clock,
	BuildId buildId,
	ImmutableMap<String, String> environment,
	RuleKey ruleKey,
	RuleKey rukeKeyWithoutDeps) {
	this(
	buildTarget,
	projectFilesystem,
	clock,
	buildId,
	environment,
	ruleKey,
	rukeKeyWithoutDeps,
	DEFAULT_DIRECTORY_TRAVERSER);
	}

	BuildInfoRecorder(BuildTarget buildTarget,
	ProjectFilesystem projectFilesystem,
	Clock clock,
	BuildId buildId,
	ImmutableMap<String, String> environment,
	RuleKey ruleKey,
	RuleKey rukeKeyWithoutDeps,
	DirectoryTraverser directoryTraverser) {
	this.buildTarget = buildTarget;
	this.pathToMetadataDirectory = BuildInfo.getPathToMetadataDirectory(buildTarget);
	this.projectFilesystem = projectFilesystem;
	this.clock = clock;
	this.buildId = buildId;

	this.artifactExtraData =
	String.format("artifact_data=%s", environment.get(BUCK_CACHE_DATA_ENV_VAR));

	this.metadataToWrite = Maps.newHashMap();

	metadataToWrite.put(BuildInfo.METADATA_KEY_FOR_RULE_KEY,
	ruleKey.toString());
	metadataToWrite.put(BuildInfo.METADATA_KEY_FOR_RULE_KEY_WITHOUT_DEPS,
	rukeKeyWithoutDeps.toString());
	this.ruleKey = ruleKey;
	this.pathsToOutputFiles = Sets.newHashSet();
	this.pathsToOutputDirectories = Sets.newHashSet();
	this.directoryTraverser = directoryTraverser;
	}

	/**
	* Writes the metadata currently stored in memory to the directory returned by
	* {@link BuildInfo#getPathToMetadataDirectory(BuildTarget)}.
	*/
	public void writeMetadataToDisk(boolean clearExistingMetadata) throws IOException {
	if (clearExistingMetadata) {
	projectFilesystem.rmdir(pathToMetadataDirectory);
	}
	projectFilesystem.mkdirs(pathToMetadataDirectory);

	for (Map.Entry<String, String> entry : metadataToWrite.entrySet()) {
	projectFilesystem.writeContentsToPath(
	entry.getValue(),
	pathToMetadataDirectory.resolve(entry.getKey()));
	}
	}

	/**
	* This key/value pair is stored in memory until {@link #writeMetadataToDisk(boolean)} is invoked.
	*/
	public void addMetadata(String key, String value) {
	metadataToWrite.put(key, value);
	}

	public void addMetadata(String key, Iterable<String> value) {
	JsonArray values = new JsonArray();
	for (String str : value) {
	values.add(new JsonPrimitive(str));
	}
	addMetadata(key, values.toString());
	}

	/**
	* Creates a zip file of the metadata and recorded artifacts and stores it in the artifact cache.
	*/
	public void performUploadToArtifactCache(ArtifactCache artifactCache, BuckEventBus eventBus)
	throws InterruptedException {
	// Skip all of this if caching is disabled. Although artifactCache.store() will be a noop,
	// building up the zip is wasted I/O.
	if (!artifactCache.isStoreSupported()) {
	return;
	}

	ImmutableSet.Builder<Path> pathsToIncludeInZipBuilder = ImmutableSet.<Path>builder()
	.addAll(Iterables.transform(metadataToWrite.keySet(),
	new Function<String, Path>() {
	@Override
	public Path apply(String key) {
	return pathToMetadataDirectory.resolve(key);
	}
	}))
	.addAll(pathsToOutputFiles);

	try {
	for (Path outputDirectory : pathsToOutputDirectories) {
	pathsToIncludeInZipBuilder.addAll(getEntries(outputDirectory));
	}
	} catch (IOException e) {
	throw new RuntimeException(e);
	}

	ImmutableSet<Path> pathsToIncludeInZip = pathsToIncludeInZipBuilder.build();
	File zip;
	try {
	zip = File.createTempFile(
	MoreFiles.sanitize(buildTarget.getFullyQualifiedName()),
	".zip");
	long time = TimeUnit.MILLISECONDS.toSeconds(clock.currentTimeMillis());
	String additionalArtifactInfo = String.format(
	"build_id=%s\ntimestamp=%d\n%s\n",
	buildId,
	time,
	artifactExtraData);
	projectFilesystem.createZip(
	pathsToIncludeInZip,
	zip,
	ImmutableMap.of(PATH_TO_ARTIFACT_INFO, additionalArtifactInfo));
	} catch (IOException e) {
	eventBus.post(ConsoleEvent.info("Failed to create zip for %s containing:\n%s",
	buildTarget,
	Joiner.on('\n').join(ImmutableSortedSet.copyOf(pathsToIncludeInZip))));
	e.printStackTrace();
	return;
	}
	artifactCache.store(ruleKey, zip);
	zip.delete();
	}

	private List<Path> getEntries(final Path outputDirectory) throws IOException {
	final ImmutableList.Builder<Path> entries = ImmutableList.builder();
	DirectoryTraversal traversal = new DirectoryTraversal(
	projectFilesystem.getFileForRelativePath(outputDirectory)) {
	@Override
	public void visit(File file, String relativePath) throws IOException {
	entries.add(outputDirectory.resolve(relativePath));
	}
	@Override
	public void visitDirectory(File directory, String relativePath) throws IOException {
	entries.add(outputDirectory.resolve(relativePath));
	}
	};
	directoryTraverser.traverse(traversal);
	return entries.build();
	}

	/**
	* Fetches the artifact associated with the {@link #buildTarget} for this class and writes it to
	* the specified {@code outputFile}.
	*/
	public CacheResult fetchArtifactForBuildable(File outputFile, ArtifactCache artifactCache)
	throws InterruptedException {
	return artifactCache.fetch(ruleKey, outputFile);
	}

	/**
	* @param pathToArtifact Relative path to the project root.
	*/
	public void recordArtifact(Path pathToArtifact) {
	Preconditions.checkArgument(
	!pathToArtifact.isAbsolute(),
	ABSOLUTE_PATH_ERROR_FORMAT,
	buildTarget,
	pathToArtifact);
	pathsToOutputFiles.add(pathToArtifact);
	}

	public void recordArtifactsInDirectory(Path pathToArtifactsDirectory) {
	Preconditions.checkArgument(
	!pathToArtifactsDirectory.isAbsolute(),
	ABSOLUTE_PATH_ERROR_FORMAT,
	buildTarget,
	pathToArtifactsDirectory);
	pathsToOutputDirectories.add(pathToArtifactsDirectory);
	}

	@Nullable
	@VisibleForTesting
	String getMetadataFor(String key) {
	return metadataToWrite.get(key);
	}
	}