src/com/facebook/buck/android/CompileStringsStep.java

/*
 * Copyright 2013-present Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License. You may obtain
 * a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations
 * under the License.
 */

package com.facebook.buck.android;

import com.facebook.buck.step.ExecutionContext;
import com.facebook.buck.step.Step;
import com.facebook.buck.util.ProjectFilesystem;
import com.facebook.buck.util.XmlDomParser;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import com.google.common.collect.TreeMultimap;

import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * This {@link Step} takes in a {@link FilterResourcesStep} that provides a list of string resource
 * files (strings.xml), groups them by locales, and for each locale generates a file with all the
 * string resources for that locale.
 *
 * <p>A typical strings.xml file looks like:
 * <pre>
 *   {@code
 *   <?xml version="1.0" encoding="utf-8"?>
 *   <resources>
 *     <string name="resource_name1">I am a string.</string>
 *     <string name="resource_name2">I am another string.</string>
 *     <plurals name="time_hours_ago">
 *       <item quantity="one">1 minute ago</item>
 *       <item quantity="other">%d minutes ago</item>
 *     </plurals>
 *     <string-array name="logging_levels">
 *       <item>Default</item>
 *       <item>Verbose</item>
 *       <item>Debug</item>
 *     </stri (which is accepted, hurray!) and bump .buckversionng-array>
 *   </resources>
 *   }
 * </pre></p>
 *
 * <p>For more information on the xml file format, refer to:
 * <a href="http://developer.android.com/guide/topics/resources/string-resource.html">
 *   String Resources - Android Developers
 * </a></p>
 *
 * <p>So for each supported locale in a project, this step goes through all such xml files for that
 * locale, and builds a map of resource name to resource value, where resource value is either:
 * <ol>
 *   <li> a string </li>
 *   <li> a map of plurals </li>
 *   <li> a list of strings </li>
 * </ol>
 * and dumps this map into the output file. See {@link StringResources} for the file format.</p>
 */
public class CompileStringsStep implements Step {

  @VisibleForTesting
  static final Pattern STRING_FILE_PATTERN = Pattern.compile(
      ".*res/values-([a-z]{2})(?:-r([A-Z]{2}))*/strings.xml");

  @VisibleForTesting
  static final Pattern R_DOT_TXT_STRING_RESOURCE_PATTERN = Pattern.compile(
      "^int (string|plurals|array) (\\w+) 0x([0-9a-f]+)$");

  private final FilterResourcesStep filterResourcesStep;
  private final Path rDotJavaSrcDir;
  private final Path destinationDir;
  private final Map<String, String> regionSpecificToBaseLocaleMap;
  private final Map<String, Integer> resourceNameToIdMap;

  /**
   * Note: The ordering of files in the input list determines which resource value ends up in the
   * output json file, in the event of multiple xml files of a locale sharing the same string
   * resource name - file that appears first in the list wins.
   *
   * @param filterResourcesStep {@link FilterResourcesStep} that filters non english string files.
   * @param rDotJavaSrcDir Path to the directory where aapt generates R.txt file along with the
   *     final R.java files per package.
   * @param destinationDir Output directory for the generated json files.
   */
  public CompileStringsStep(
      FilterResourcesStep filterResourcesStep,
      Path rDotJavaSrcDir,
      Path destinationDir) {
    this.filterResourcesStep = Preconditions.checkNotNull(filterResourcesStep);
    this.rDotJavaSrcDir = Preconditions.checkNotNull(rDotJavaSrcDir);
    this.destinationDir = Preconditions.checkNotNull(destinationDir);
    this.regionSpecificToBaseLocaleMap = Maps.newHashMap();
    this.resourceNameToIdMap = Maps.newHashMap();
  }

  @Override
  public int execute(ExecutionContext context) {
    ProjectFilesystem filesystem = context.getProjectFilesystem();
    try {
      buildResourceNameToIdMap(filesystem);
    } catch (IOException e) {
      context.logError(e, "Failure parsing R.txt file.");
      return 1;
    }

    ImmutableSet<String> filteredStringFiles = filterResourcesStep.getNonEnglishStringFiles();
    ImmutableMultimap<String, String> filesByLocale = groupFilesByLocale(filteredStringFiles);

    Map<String, StringResources> resourcesByLocale = Maps.newHashMap();
    for (String locale : filesByLocale.keySet()) {
      try {
        resourcesByLocale.put(locale, compileStringFiles(filesByLocale.get(locale)));
      } catch (IOException e) {
        context.logError(e, "Error parsing string file for locale: %s", locale);
        return 1;
      }
    }

    // Merge region specific locale resources with the corresponding base locale resources.
    //
    // For example, if there are separate string resources in an android project for locale
    // "es" and "es_US", when an application running on a device with locale set to "Spanish
    // (United States)" requests for a string, the Android runtime first looks for the string in
    // "es_US" set of resources, and if not found, returns the resource from the "es" set.
    // We merge these because we want the individual string json files to be self contained for
    // simplicity.
    for (String regionSpecificLocale : regionSpecificToBaseLocaleMap.keySet()) {
      String baseLocale = regionSpecificToBaseLocaleMap.get(regionSpecificLocale);
      if (!resourcesByLocale.containsKey(baseLocale)) {
        continue;
      }

      resourcesByLocale.put(regionSpecificLocale,
          resourcesByLocale.get(regionSpecificLocale)
              .getMergedResources(resourcesByLocale.get(baseLocale)));
    }

    for (String locale : filesByLocale.keySet()) {
      try {
        filesystem.writeBytesToPath(resourcesByLocale.get(locale).getBinaryFileContent(),
            destinationDir.resolve(locale + ".fbstr"));
      } catch (IOException e) {
        context.logError(e, "Error creating binary file for locale: %s", locale);
        return 1;
      }
    }

    return 0;
  }

  /**
   * Groups a list of file paths matching STRING_FILE_PATTERN by the locale.
   * eg. given the following list:
   *
   * ImmutableSet.of(
   *   '/one/res/values-es/strings.xml',
   *   '/two/res/values-es/strings.xml',
   *   '/three/res/values-pt-rBR/strings.xml',
   *   '/four/res/values/-pt-rPT/strings.xml');
   *
   * returns:
   *
   * ImmutableMap.of(
   *   'es', ImmutableSet.of('/one/res/values-es/strings.xml', '/two/res/values-es/strings.xml'),
   *   'pt_BR', ImmutableSet.of('/three/res/values-pt-rBR/strings.xml'),
   *   'pt_PT', ImmutableSet.of('/four/res/values/-pt-rPT/strings.xml'));
   */
  @VisibleForTesting
  ImmutableMultimap<String, String> groupFilesByLocale(ImmutableSet<String> files) {
    ImmutableMultimap.Builder<String, String> localeToFiles = ImmutableMultimap.builder();

    for (String filepath : files) {
      Matcher matcher = STRING_FILE_PATTERN.matcher(filepath);
      if (!matcher.matches()) {
        continue;
      }

      String baseLocale = matcher.group(1);
      String country = matcher.group(2);
      String locale = country == null ? baseLocale : baseLocale + "_" + country;
      if (country != null && !regionSpecificToBaseLocaleMap.containsKey(locale)) {
        regionSpecificToBaseLocaleMap.put(locale, baseLocale);
      }

      localeToFiles.put(locale, filepath);
    }

    return localeToFiles.build();
  }

  /**
   * Parses the R.txt file generated by aapt, looks for resources of type {@code string},
   * {@code plurals} and {@code array}, and builds a map of resource names to their corresponding
   * ids.
   */
  @VisibleForTesting
  void buildResourceNameToIdMap(ProjectFilesystem filesystem)
      throws IOException {
    List<String> fileLines = filesystem.readLines(rDotJavaSrcDir.resolve("R.txt"));
    for (String line : fileLines) {
      Matcher matcher = R_DOT_TXT_STRING_RESOURCE_PATTERN.matcher(line);
      if (!matcher.matches()) {
        continue;
      }
      resourceNameToIdMap.put(matcher.group(2), Integer.parseInt(matcher.group(3), 16));
    }
  }

  private StringResources compileStringFiles(Collection<String> filepaths)
      throws IOException {
    TreeMap<Integer, String> stringsMap = Maps.newTreeMap();
    TreeMap<Integer, ImmutableMap<String, String>> pluralsMap = Maps.newTreeMap();
    TreeMultimap<Integer, String> arraysMap = TreeMultimap.create();

    for (String stringFilePath : filepaths) {
      File stringFile = (Paths.get(stringFilePath)).toFile();
      Document dom = XmlDomParser.parse(stringFile);

      NodeList stringNodes = dom.getElementsByTagName("string");
      scrapeStringNodes(stringNodes, stringsMap);

      NodeList pluralNodes = dom.getElementsByTagName("plurals");
      scrapePluralsNodes(pluralNodes, pluralsMap);

      NodeList arrayNodes = dom.getElementsByTagName("string-array");
      scrapeStringArrayNodes(arrayNodes, arraysMap);
    }

    return new StringResources(stringsMap, pluralsMap, arraysMap);
  }


  /**
   * Scrapes string resource names and values from the list of xml nodes passed and populates
   * {@code stringsMap}, ignoring resource names that are already present in the map.
   *
   * @param stringNodes A list of {@code <string></string>} nodes.
   * @param stringsMap Map from string resource name to its value.
   */
  @VisibleForTesting
  void scrapeStringNodes(NodeList stringNodes, Map<Integer, String> stringsMap) {
    for (int i = 0; i < stringNodes.getLength(); ++i) {
      Node node = stringNodes.item(i);
      String resourceName = node.getAttributes().getNamedItem("name").getNodeValue();
      if (!resourceNameToIdMap.containsKey(resourceName)) {
        continue;
      }
      int resourceId = resourceNameToIdMap.get(resourceName);
      // Ignore a resource if it has already been found.
      if (!stringsMap.containsKey(resourceId)) {
        stringsMap.put(resourceId, node.getTextContent());
      }
    }
  }

  /**
   * Similar to {@code scrapeStringNodes}, but for plurals nodes.
   */
  @VisibleForTesting
  void scrapePluralsNodes(
      NodeList pluralNodes,
      Map<Integer, ImmutableMap<String, String>> pluralsMap) {

    for (int i = 0; i < pluralNodes.getLength(); ++i) {
      Node node = pluralNodes.item(i);
      String resourceName = node.getAttributes().getNamedItem("name").getNodeValue();
      if (!resourceNameToIdMap.containsKey(resourceName)) {
        continue;
      }
      int resourceId = resourceNameToIdMap.get(resourceName);

      // Ignore a resource if it has already been found.
      if (pluralsMap.containsKey(resourceId)) {
        continue;
      }
      ImmutableMap.Builder<String, String> quantityToStringBuilder = ImmutableMap.builder();

      NodeList itemNodes = ((Element) node).getElementsByTagName("item");
      for (int j = 0; j < itemNodes.getLength(); ++j) {
        Node itemNode = itemNodes.item(j);
        String quantity = itemNode.getAttributes().getNamedItem("quantity").getNodeValue();
        quantityToStringBuilder.put(quantity, itemNode.getTextContent());
      }
      pluralsMap.put(resourceId, quantityToStringBuilder.build());
    }
  }

  /**
   * Similar to {@code scrapeStringNodes}, but for string array nodes.
   */
  @VisibleForTesting
  void scrapeStringArrayNodes(NodeList arrayNodes, Multimap<Integer, String> arraysMap) {
    for (int i = 0; i < arrayNodes.getLength(); ++i) {
      Node node = arrayNodes.item(i);
      String resourceName = node.getAttributes().getNamedItem("name").getNodeValue();
      // Ignore a resource if R.txt does not contain an entry for it.
      if (!resourceNameToIdMap.containsKey(resourceName)) {
        continue;
      }

      int resourceId = resourceNameToIdMap.get(resourceName);
      // Ignore a resource if it has already been found.
      if (arraysMap.containsKey(resourceId)) {
        continue;
      }

      NodeList itemNodes = ((Element)node).getElementsByTagName("item");
      for (int j = 0; j < itemNodes.getLength(); ++j) {
        arraysMap.put(resourceId, itemNodes.item(j).getTextContent());
      }
    }
  }

  /**
   * Used in unit tests to inject the resource name to id map.
   */
  @VisibleForTesting
  void addResourceNameToIdMap(Map<String, Integer> nameToIdMap) {
    resourceNameToIdMap.putAll(nameToIdMap);
  }

  @Override
  public String getShortName() {
    return "compile_strings";
  }

  @Override
  public String getDescription(ExecutionContext context) {
    return "Combine, parse string resource xml files into one binary file per locale.";
  }
}