java/com/google/gerrit/server/rules/prolog/RulesCache.java

// Copyright (C) 2011 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.gerrit.server.rules.prolog;

import static com.google.gerrit.server.project.ProjectConfig.RULES_PL_FILE;
import static com.googlecode.prolog_cafe.lang.PrologMachineCopy.save;

import com.google.common.base.Joiner;
import com.google.common.base.Strings;
import com.google.common.cache.Cache;
import com.google.common.collect.ImmutableList;
import com.google.gerrit.entities.Project;
import com.google.gerrit.entities.RefNames;
import com.google.gerrit.server.cache.CacheModule;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.plugincontext.PluginSetContext;
import com.google.gerrit.server.project.ProjectCacheImpl;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.google.inject.name.Named;
import com.googlecode.prolog_cafe.exceptions.CompileException;
import com.googlecode.prolog_cafe.exceptions.SyntaxException;
import com.googlecode.prolog_cafe.exceptions.TermException;
import com.googlecode.prolog_cafe.lang.BufferingPrologControl;
import com.googlecode.prolog_cafe.lang.JavaObjectTerm;
import com.googlecode.prolog_cafe.lang.ListTerm;
import com.googlecode.prolog_cafe.lang.Prolog;
import com.googlecode.prolog_cafe.lang.PrologClassLoader;
import com.googlecode.prolog_cafe.lang.PrologMachineCopy;
import com.googlecode.prolog_cafe.lang.StructureTerm;
import com.googlecode.prolog_cafe.lang.SymbolTerm;
import com.googlecode.prolog_cafe.lang.Term;
import java.io.IOException;
import java.io.PushbackReader;
import java.io.Reader;
import java.io.StringReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
import java.util.concurrent.ExecutionException;
import org.eclipse.jgit.annotations.Nullable;
import org.eclipse.jgit.errors.LargeObjectException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.util.RawParseUtils;

/**
 * Manages a cache of compiled Prolog rules.
 *
 * <p>Rules are loaded from the {@code site_path/cache/rules/rules-SHA1.jar}, where {@code SHA1} is
 * the SHA1 of the Prolog {@code rules.pl} in a project's {@link RefNames#REFS_CONFIG} branch.
 */
@Singleton
public class RulesCache {
  public static class RulesCacheModule extends CacheModule {
    protected final Config config;

    public RulesCacheModule(Config config) {
      this.config = config;
    }

    @Override
    protected void configure() {
      if (has(ProjectCacheImpl.CACHE_NAME, "memoryLimit")) {
        // As this cache is auxiliary to the project cache, so size it the same when available
        cache(RulesCache.CACHE_NAME, ObjectId.class, PrologMachineCopy.class)
            .maximumWeight(config.getLong("cache", ProjectCacheImpl.CACHE_NAME, "memoryLimit", 0));
      } else {
        cache(RulesCache.CACHE_NAME, ObjectId.class, PrologMachineCopy.class);
      }
    }

    private boolean has(String name, String var) {
      return !Strings.isNullOrEmpty(config.getString("cache", name, var));
    }
  }

  private static final ImmutableList<String> PACKAGE_LIST =
      ImmutableList.of(Prolog.BUILTIN, "gerrit");

  static final String CACHE_NAME = "prolog_rules";

  private final boolean enableProjectRules;
  private final int maxDbSize;
  private final int compileReductionLimit;
  private final int maxSrcBytes;
  private final Path cacheDir;
  private final Path rulesDir;
  private final GitRepositoryManager gitMgr;
  private final PluginSetContext<PredicateProvider> predicateProviders;
  private final ClassLoader systemLoader;
  private final PrologMachineCopy defaultMachine;
  private final Cache<ObjectId, PrologMachineCopy> machineCache;

  @Inject
  protected RulesCache(
      @GerritServerConfig Config config,
      SitePaths site,
      GitRepositoryManager gm,
      PluginSetContext<PredicateProvider> predicateProviders,
      @Named(CACHE_NAME) Cache<ObjectId, PrologMachineCopy> machineCache) {
    maxDbSize = config.getInt("rules", null, "maxPrologDatabaseSize", 256);
    compileReductionLimit = RuleUtil.compileReductionLimit(config);
    maxSrcBytes = config.getInt("rules", null, "maxSourceBytes", 128 << 10);
    enableProjectRules = config.getBoolean("rules", null, "enable", true) && maxSrcBytes > 0;
    cacheDir = site.resolve(config.getString("cache", null, "directory"));
    rulesDir = cacheDir != null ? cacheDir.resolve("rules") : null;
    gitMgr = gm;
    this.predicateProviders = predicateProviders;
    this.machineCache = machineCache;

    systemLoader = getClass().getClassLoader();
    defaultMachine = save(newEmptyMachine(systemLoader));
  }

  public boolean isProjectRulesEnabled() {
    return enableProjectRules;
  }

  /**
   * Locate a cached Prolog machine state, or create one if not available.
   *
   * @return a Prolog machine, after loading the specified rules.
   * @throws CompileException the machine cannot be created.
   */
  public synchronized PrologMachineCopy loadMachine(
      @Nullable Project.NameKey project, @Nullable ObjectId rulesId) throws CompileException {
    if (!enableProjectRules || project == null || rulesId == null) {
      return defaultMachine;
    }

    try {
      return machineCache.get(rulesId, () -> createMachine(project, rulesId));
    } catch (ExecutionException e) {
      if (e.getCause() instanceof CompileException) {
        throw new CompileException(e.getCause().getMessage(), e);
      }
      throw new CompileException("Error while consulting rules from " + project, e);
    }
  }

  public PrologMachineCopy loadMachine(String name, Reader in) throws CompileException {
    PrologMachineCopy pmc = consultRules(name, in);
    if (pmc == null) {
      throw new CompileException("Cannot consult rules from the stream " + name);
    }
    return pmc;
  }

  private PrologMachineCopy createMachine(Project.NameKey project, ObjectId rulesId)
      throws CompileException {
    // If the rules are available as a complied JAR on local disk, prefer
    // that over dynamic consult as the bytecode will be faster.
    //
    if (rulesDir != null) {
      Path jarPath = rulesDir.resolve("rules-" + rulesId.getName() + ".jar");
      if (Files.isRegularFile(jarPath)) {
        URL[] cp = new URL[] {toURL(jarPath)};
        return save(newEmptyMachine(URLClassLoader.newInstance(cp, systemLoader)));
      }
    }

    // Dynamically consult the rules into the machine's internal database.
    //
    String rules = read(project, rulesId);
    PrologMachineCopy pmc = consultRules(RULES_PL_FILE, new StringReader(rules));
    if (pmc == null) {
      throw new CompileException("Cannot consult rules of " + project);
    }
    return pmc;
  }

  @Nullable
  private PrologMachineCopy consultRules(String name, Reader rules) throws CompileException {
    BufferingPrologControl ctl = newEmptyMachine(systemLoader);
    PushbackReader in = new PushbackReader(rules, Prolog.PUSHBACK_SIZE);
    try {
      if (!ctl.execute(
          Prolog.BUILTIN, "consult_stream", SymbolTerm.intern(name), new JavaObjectTerm(in))) {
        return null;
      }
    } catch (SyntaxException e) {
      throw new CompileException(e.toString(), e);
    } catch (TermException e) {
      Term m = e.getMessageTerm();
      if (m instanceof StructureTerm && "syntax_error".equals(m.name()) && m.arity() >= 1) {
        StringBuilder msg = new StringBuilder();
        if (m.arg(0) instanceof ListTerm) {
          msg.append(Joiner.on(' ').join(((ListTerm) m.arg(0)).toJava()));
        } else {
          msg.append(m.arg(0).toString());
        }
        if (m.arity() == 2 && m.arg(1) instanceof StructureTerm && "at".equals(m.arg(1).name())) {
          Term at = m.arg(1).arg(0).dereference();
          if (at instanceof ListTerm) {
            msg.append(" at: ");
            msg.append(prettyProlog(at));
          }
        }
        throw new CompileException(msg.toString(), e);
      }
      throw new CompileException("Error while consulting rules from " + name, e);
    } catch (RuntimeException e) {
      throw new CompileException("Error while consulting rules from " + name, e);
    }
    return save(ctl);
  }

  private static String prettyProlog(Term at) {
    StringBuilder b = new StringBuilder();
    for (Object o : ((ListTerm) at).toJava()) {
      if (o instanceof Term) {
        Term t = (Term) o;
        if (!(t instanceof StructureTerm)) {
          b.append(t.toString()).append(' ');
          continue;
        }
        switch (t.name()) {
          case "atom":
            SymbolTerm atom = (SymbolTerm) t.arg(0);
            b.append(atom.toString());
            break;
          case "var":
            b.append(t.arg(0).toString());
            break;
        }
      } else {
        b.append(o);
      }
    }
    return b.toString().trim();
  }

  private String read(Project.NameKey project, ObjectId rulesId) throws CompileException {
    try (Repository git = gitMgr.openRepository(project)) {
      try {
        ObjectLoader ldr = git.open(rulesId, Constants.OBJ_BLOB);
        byte[] raw = ldr.getCachedBytes(maxSrcBytes);
        return RawParseUtils.decode(raw);
      } catch (LargeObjectException e) {
        throw new CompileException("rules of " + project + " are too large", e);
      } catch (RuntimeException | IOException e) {
        throw new CompileException("Cannot load rules of " + project, e);
      }
    } catch (IOException e) {
      throw new CompileException("Cannot open repository " + project, e);
    }
  }

  private BufferingPrologControl newEmptyMachine(ClassLoader cl) {
    BufferingPrologControl ctl = new BufferingPrologControl();
    ctl.setMaxDatabaseSize(maxDbSize);
    // Use the compiled reduction limit because the first term evaluation is done with
    // consult_stream - an internal, combined Prolog term.
    ctl.setReductionLimit(compileReductionLimit);
    ctl.setPrologClassLoader(
        new PrologClassLoader(new PredicateClassLoader(predicateProviders, cl)));
    ctl.setEnabled(EnumSet.allOf(Prolog.Feature.class), false);

    List<String> packages = new ArrayList<>();
    packages.addAll(PACKAGE_LIST);
    predicateProviders.runEach(
        predicateProvider -> packages.addAll(predicateProvider.getPackages()));

    // Bootstrap the interpreter and ensure there is clean state.
    ctl.initialize(packages.toArray(new String[packages.size()]));
    return ctl;
  }

  private static URL toURL(Path jarPath) throws CompileException {
    try {
      return jarPath.toUri().toURL();
    } catch (MalformedURLException e) {
      throw new CompileException("Cannot create URL for " + jarPath, e);
    }
  }
}