/*
 * Copyright 2012-present Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License. You may obtain
 * a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations
 * under the License.
 */

package com.facebook.buck.junit;

import com.facebook.buck.test.result.type.ResultType;
import com.facebook.buck.test.selectors.TestDescription;

import org.junit.Ignore;
import org.junit.internal.builders.AllDefaultPossibilitiesBuilder;
import org.junit.internal.builders.AnnotatedBuilder;
import org.junit.internal.builders.JUnit4Builder;
import org.junit.runner.Computer;
import org.junit.runner.Description;
import org.junit.runner.JUnitCore;
import org.junit.runner.Request;
import org.junit.runner.Result;
import org.junit.runner.Runner;
import org.junit.runner.manipulation.Filter;
import org.junit.runner.notification.Failure;
import org.junit.runner.notification.RunListener;
import org.junit.runners.model.RunnerBuilder;

import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

/**
 * Class that runs a set of JUnit tests and writes the results to a directory.
 * <p>
 * IMPORTANT! This class limits itself to types that are available in both the JDK and Android Java
 * API. The objective is to limit the set of files added to the ClassLoader that runs the test, as
 * not to interfere with the results of the test.
 */
public final class JUnitRunner extends BaseRunner {
  public JUnitRunner() {
  }

  @Override
  public void run() throws Throwable {
    Filter filter = new Filter() {
      @Override
      public boolean shouldRun(Description description) {
        String methodName = description.getMethodName();
        if (methodName == null) {
          // JUnit will give us an org.junit.runner.Description like this for the test class
          // itself.  It's easier for our filtering to make decisions just at the method level,
          // however, so just always return true here.
          return true;
        } else {
          String className = description.getClassName();
          TestDescription testDescription = new TestDescription(className, methodName);
          if (testSelectorList.isIncluded(testDescription)) {
            boolean isIgnored = description.getAnnotation(Ignore.class) != null;
            if (!isIgnored) {
              seenDescriptions.add(testDescription);
            }
            return !isDryRun;
          } else {
            return false;
          }
        }
      }

      @Override
      public String describe() {
        return FILTER_DESCRIPTION;
      }
    };

    for (String className : testClassNames) {
      final Class<?> testClass = Class.forName(className);
      Ignore ignore = testClass.getAnnotation(Ignore.class);
      boolean isTestClassIgnored = (ignore != null || !isTestClass(testClass));

      List<TestResult> results;
      if (isTestClassIgnored) {
        // Test case has @Ignore annotation, so do nothing.
        results = Collections.emptyList();
      } else {
        results = new ArrayList<>();
        JUnitCore jUnitCore = new JUnitCore();

        Runner suite = new Computer().getSuite(createRunnerBuilder(), new Class<?>[]{testClass});
        Request request = Request.runner(suite);
        request = request.filterWith(filter);

        jUnitCore.addListener(new TestListener(results));
        jUnitCore.run(request);
      }

      results = interpretResults(className, results);
      if (results != null) {
        writeResult(className, results);
      }
    }
  }

  /**
   * This method filters a list of test results prior to writing results to a file.  null is
   * returned to indicate "don't write anything", which is different to writing a file containing
   * 0 results.
   *
   * <p>
   *
   * JUnit handles classes-without-tests in different ways.  If you are not using the
   * org.junit.runner.Request.filterWith facility then JUnit ignores classes-without-tests.
   * However, if you are using a filter then a class-without-tests will cause a
   * NoTestsRemainException to be thrown, which is propagated back as an error.
   */
  /* @Nullable */
  private List<TestResult> interpretResults(String className, List<TestResult> results) {
    // For dry runs, write fake results for every method seen in the given class.
    if (isDryRun) {
      List<TestResult> fakeResults = new ArrayList<>();
      for (TestDescription seenDescription : seenDescriptions) {
        if (seenDescription.getClassName().equals(className)) {
          TestResult fakeResult = new TestResult(
            seenDescription.getClassName(),
              seenDescription.getMethodName(),
              0L,
              ResultType.DRY_RUN,
              null,
              "",
              "");
          fakeResults.add(fakeResult);
        }
      }
      results = fakeResults;
    }

    // When not using any command line filtering options, all results should be recorded.
    if (testSelectorList.isEmpty()) {
      if (isSingleResultCausedByNoTestsRemainException(results)) {
        // ...except for testless-classes, where we pretend nothing ran.
        return new ArrayList<>();
      } else {
        return results;
      }
    }

    // If the results size is 0 (which can happen at least with JUnit 4.11), results are not
    // significant and shouldn't be recorded.
    if (results.size() == 0) {
      return null;
    }

    // In (at least) JUnit 4.8, we have an odd scenario where we have one result telling us we
    // have no results.
    if (isSingleResultCausedByNoTestsRemainException(results)) {
      return null;
    }

    return results;
  }

  /**
   * JUnit doesn't normally consider encountering a testless class an error.  However, when
   * using org.junit.runner.manipulation.Filter, testless classes *are* considered an error,
   * throwing org.junit.runner.manipulation.NoTestsRemainException.
   *
   * If we are using test-selectors then it's possible we will run a test class but never run any
   * of its test methods, because they'd all get filtered out.  When this happens, the results will
   * contain a single failure containing the error from the NoTestsRemainException.
   *
   * (NB: we can't decide at the class level whether we need to run a test class or not; we can only
   * run the test class and all its test methods and handle the erroneous exception JUnit throws if
   * no test-methods were actually run.)
   */
  private boolean isSingleResultCausedByNoTestsRemainException(List<TestResult> results) {
    if (results.size() != 1) {
      return false;
    }

    TestResult testResult = results.get(0);
    if (testResult.isSuccess()) {
      return false;
    }

    if (testResult.failure == null) {
      return false;
    }

    String message = testResult.failure.getMessage();
    if (message == null) {
      return false;
    }

    return message.contains("No tests found matching " + FILTER_DESCRIPTION);
  }

  private boolean isTestClass(Class<?> klass) {
    return klass.getConstructors().length <= 1;
  }

  /**
   * Creates an {@link AllDefaultPossibilitiesBuilder} that returns our custom
   * {@link BuckBlockJUnit4ClassRunner} when a {@link JUnit4Builder} is requested. This ensures that
   * JUnit 4 tests are executed using our runner whereas other types of tests are run with whatever
   * JUnit thinks is best.
   */
  private RunnerBuilder createRunnerBuilder() {
    final JUnit4Builder jUnit4RunnerBuilder = new JUnit4Builder() {
      @Override
      public Runner runnerForClass(Class<?> testClass) throws Throwable {
        return new BuckBlockJUnit4ClassRunner(testClass, defaultTestTimeoutMillis);
      }
    };

    return new AllDefaultPossibilitiesBuilder(/* canUseSuiteMethod */ true) {
      @Override
      protected JUnit4Builder junit4Builder() {
        return jUnit4RunnerBuilder;
      }

      @Override
      protected AnnotatedBuilder annotatedBuilder() {
        // If there is no default timeout specified in .buckconfig, then use
        // the original behavior of AllDefaultPossibilitiesBuilder.
        //
        // Additionally, if we are using test selectors or doing a dry-run then
        // we should use the original behavior to use our
        // BuckBlockJUnit4ClassRunner, which provides the Descriptions needed
        // to do test selecting properly.
        if (defaultTestTimeoutMillis <= 0 || isDryRun || !testSelectorList.isEmpty()) {
          return super.annotatedBuilder();
        }

        return new AnnotatedBuilder(this) {
          @Override
          public Runner buildRunner(Class<? extends Runner> runnerClass,
              Class<?> testClass) throws Exception {
            Runner originalRunner = super.buildRunner(runnerClass, testClass);
            return new DelegateRunnerWithTimeout(originalRunner, defaultTestTimeoutMillis);
          }
        };
      }
    };
  }

  /**
   * Creates RunListener that will prepare individual result for each test
   * and store it to results list afterwards.
   */
  private static class TestListener extends RunListener {
    private final List<TestResult> results;
    private PrintStream originalOut, originalErr, stdOutStream, stdErrStream;
    private ByteArrayOutputStream rawStdOutBytes, rawStdErrBytes;
    private Result result;
    private RunListener resultListener;
    private Failure assumptionFailure;

    // To help give a reasonable (though imprecise) guess at the runtime for unpaired failures
    private long startTime = System.currentTimeMillis();

    public TestListener(List<TestResult> results) {
      this.results = results;
    }

    @Override
    public void testStarted(Description description) throws Exception {
      // Create an intermediate stdout/stderr to capture any debugging statements (usually in the
      // form of System.out.println) the developer is using to debug the test.
      originalOut = System.out;
      originalErr = System.err;
      rawStdOutBytes = new ByteArrayOutputStream();
      rawStdErrBytes = new ByteArrayOutputStream();
      stdOutStream = new PrintStream(
          rawStdOutBytes, true /* autoFlush */, ENCODING);
      stdErrStream = new PrintStream(
          rawStdErrBytes, true /* autoFlush */, ENCODING);
      System.setOut(stdOutStream);
      System.setErr(stdErrStream);

      // Prepare single-test result.
      result = new Result();
      resultListener = result.createListener();
      resultListener.testRunStarted(description);
      resultListener.testStarted(description);
    }

    @Override
    public void testFinished(Description description) throws Exception {
      // Shutdown single-test result.
      resultListener.testFinished(description);
      resultListener.testRunFinished(result);
      resultListener = null;

      // Restore the original stdout/stderr.
      System.setOut(originalOut);
      System.setErr(originalErr);

      // Get the stdout/stderr written during the test as strings.
      stdOutStream.flush();
      stdErrStream.flush();

      int numFailures = result.getFailureCount();
      String className = description.getClassName();
      String methodName = description.getMethodName();
      // In practice, I have seen one case of a test having more than one failure:
      // com.xtremelabs.robolectric.shadows.H2DatabaseTest#shouldUseH2DatabaseMap() had 2
      // failures. However, I am not sure what to make of it, so we let it through.
      if (numFailures < 0) {
        throw new IllegalStateException(String.format(
            "Unexpected number of failures while testing %s#%s(): %d (%s)",
            className,
            methodName,
            numFailures,
            result.getFailures()));
      }

      Failure failure;
      ResultType type;
      if (assumptionFailure != null) {
        failure = assumptionFailure;
        type = ResultType.ASSUMPTION_VIOLATION;
        // Clear the assumption-failure field before the next test result appears.
        assumptionFailure = null;
      } else if (numFailures == 0) {
        failure = null;
        type = ResultType.SUCCESS;
      } else {
        failure = result.getFailures().get(0);
        type = ResultType.FAILURE;
      }

      String stdOut = rawStdOutBytes.size() == 0 ? null : rawStdOutBytes.toString(ENCODING);
      String stdErr = rawStdErrBytes.size() == 0 ? null : rawStdErrBytes.toString(ENCODING);

      results.add(new TestResult(className,
          methodName,
          result.getRunTime(),
          type,
          failure == null ? null : failure.getException(),
          stdOut,
          stdErr));
    }

    /**
     * The regular listener we created from the singular result, in this class, will not by
     * default treat assumption failures as regular failures, and will not store them.  As a
     * consequence, we store them ourselves!
     *
     * We store the assumption-failure in a temporary field, which we'll make sure we clear each
     * time we write results.
     */
    @Override
    public void testAssumptionFailure(Failure failure) {
      assumptionFailure = failure;
      if (resultListener != null) {
        // Left in only to help catch future bugs -- right now this does nothing.
        resultListener.testAssumptionFailure(failure);
      }
    }

    @Override
    public void testFailure(Failure failure) throws Exception {
      if (resultListener == null) {
        recordUnpairedFailure(failure);
      } else {
        resultListener.testFailure(failure);
      }
    }

    @Override
    public void testIgnored(Description description) throws Exception {
      if (resultListener != null) {
        resultListener.testIgnored(description);
      }
    }

    /**
     * It's possible to encounter a Failure before we've started any tests (and therefore before
     * testStarted() has been called).  The known example is a @BeforeClass that throws an
     * exception, but there may be others.
     * <p>
     * Recording these unexpected failures helps us propagate failures back up to the "buck test"
     * process.
     */
    private void recordUnpairedFailure(Failure failure) {
      long runtime = System.currentTimeMillis() - startTime;
      Description description = failure.getDescription();
      results.add(new TestResult(
          description.getClassName(),
          description.getMethodName(),
          runtime,
          ResultType.FAILURE,
          failure.getException(),
          null,
          null));
    }
  }
}