tools/node_tools/node_modules_licenses/licenses-map.ts

/**
 * @license
 * Copyright (C) 2020 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import * as path from "path";
import * as fs from "fs";
import {isSharedFileLicenseInfo, LicenseInfo, PackageInfo} from "./package-license-info";
import {LicenseName, PackageName, FilePath, PackageVersion} from "./base-types";
import { InstalledPackage, InsalledPackagesBuilder } from "./installed-node-modules-map";
import {SharedLicensesProvider} from "./shared-licenses-provider";
import {fail} from "./utils";

interface FilteredFiles {
  installedPackage: InstalledPackage;
  /** Path relative to installedPackage */
  includedFiles: FilePath[];
  /** Path relative to installedPackage */
  excludedFiles: FilePath[];
}
interface PackageLicensedFiles {
  packageInfo: PackageInfo;
  filteredFiles: FilteredFiles;
}

enum LicensedFilesDisplayInfoType {
  AllFiles = "AllFiles",
  OnlySpecificFiles = "OnlySpecificFiles",
  AllFilesExceptSpecific = "AllFilesExceptSpecific",
}

interface AllFiles {
  kind: LicensedFilesDisplayInfoType.AllFiles;
}

interface AllFilesExceptSpecificFiles {
  kind: LicensedFilesDisplayInfoType.AllFilesExceptSpecific;
  /** Path relative to installedPackage */
  files: FilePath[]
}

interface OnlySpecificFiles {
  kind: LicensedFilesDisplayInfoType.OnlySpecificFiles;
  /** Path relative to installedPackage */
  files: FilePath[];
}

export type FilteredFilesDisplayInfo = AllFiles | AllFilesExceptSpecificFiles | OnlySpecificFiles;

export interface LicenseMapPackageInfo {
  name: PackageName;
  version: PackageVersion;
  licensedFiles: FilteredFilesDisplayInfo;
}

export interface LicenseMapItem {
  licenseName: LicenseName,
  licenseText: string;
  packages: LicenseMapPackageInfo[];
}

export type LicensesMap = { [licenseName: string]:  LicenseMapItem }

/** LicenseMapGenerator calculates license map for nodeModulesFiles.
 */
export class LicenseMapGenerator {
  /**
   *  packages - information about licenses for packages
   *  sharedLicensesProvider - returns text for shared licenses by name
   */
  public constructor(private readonly packages: ReadonlyArray<PackageInfo>, private readonly sharedLicensesProvider: SharedLicensesProvider) {
  }

  /** generateMap calculates LicenseMap for nodeModulesFiles
   * Each (key, value) pair in this map has the following information:
   * The key is a license name.
   * The value contains information about packages and files for which this license
   * is applied.
   *
   * The method tries to provide information in a compact way:
   * 1) If all files in the package have the same license, then it stores a name of the package
   *    without list of files
   * 2) If different files in the package has different licenses, then the method calculates
   *    which list is shorter - list of included files or list of excluded files.
   */
  public generateMap(nodeModulesFiles: ReadonlyArray<string>): LicensesMap {
    const installedPackages = this.getInstalledPackages(nodeModulesFiles);
    // Static packages that are not inside `node_modules` directories.
    // gr-page.ts was derived from page.js, so we reproduce the original LICENSE.
    installedPackages.push({name: 'polygerrit-gr-page', version: 'current', rootPath: 'polygerrit-ui/app/elements/core/gr-router/', files: ['gr-page.ts']});
    const licensedFilesGroupedByLicense = this.getLicensedFilesGroupedByLicense(installedPackages);

    const result: LicensesMap = {};
    licensedFilesGroupedByLicense.forEach((packageLicensedFiles, licenseName) => {
      result[licenseName] = this.getLicenseMapItem(licenseName, packageLicensedFiles);
    });
    return result;
  }

  private getLicenseMapItem(licenseName: string, packagesLicensedFiles: PackageLicensedFiles[]): LicenseMapItem {
    const packages: LicenseMapPackageInfo[] = [];
    let licenseText: string = "";
    packagesLicensedFiles.forEach((packageLicensedFiles) => {
      const packageLicenseText = this.getLicenseText(packageLicensedFiles);
      if (licenseText.length !== 0 && packageLicenseText !== licenseText) {
        fail(`There are different license texts for license '${licenseName}'.\n` +
            "Probably, you have multiple version of the same package.\n" +
            "In this case you must define different license name for each version"
        );
      }
      licenseText = packageLicenseText;
      packages.push({
        name: packageLicensedFiles.packageInfo.name,
        version: packageLicensedFiles.filteredFiles.installedPackage.version,
        licensedFiles: this.getFilteredFilesDisplayInfo(packageLicensedFiles.filteredFiles)
      });
    });
    return {
      licenseName,
      licenseText,
      packages
    };
  }

  /** getFilteredFilesDisplayInfo calculates the best method to display information about
   * filteredFiles in filteredFiles.installedPackage
   * In the current implementation - the best method is a method with a shorter list of files
   *
   * Each {@link PackageInfo} has a filter for files (by default all files
   * are included). Applying filter to files from an installedPackage returns 2 lists of files:
   * includedFiles (i.e. files with {@link PackageInfo.license} license) and excludedFiles
   * (i.e. files which have different license(s)).
   *
   * A text representaion of license-map must have full information about per-file licenses if
   * needed, but we want to produce files lists there as short as possible
   */
  private getFilteredFilesDisplayInfo(filteredFiles: FilteredFiles): FilteredFilesDisplayInfo {
    if (filteredFiles.includedFiles.length > 0 && filteredFiles.excludedFiles.length === 0) {
      /** All files from package are included (i.e. all files have the same license).
       * It is enough to print only installedPackage name.
       * */
      return {
        kind: LicensedFilesDisplayInfoType.AllFiles
      }
    } else if (filteredFiles.includedFiles.length <= filteredFiles.excludedFiles.length) {
      /** After applying filter, the number of files with filteredFiles.license is less than
       * the number of excluded files.
       * It is more convenient to print information about license in the format:
       * GIT license - fontPackage (only files A,B,C)*/
      return {
        kind: LicensedFilesDisplayInfoType.OnlySpecificFiles,
        files: filteredFiles.includedFiles
      };
    } else {
      /** Only few files from filteredFiles.installedPackage has filteredFiles.license.
       * It is more convenient to print information about license in the format:
       * Apache license - fontPackage (except files A,B,C)
       */
      return {
        kind: LicensedFilesDisplayInfoType.AllFilesExceptSpecific,
        files: filteredFiles.excludedFiles
      }
    }
  }

  private getLicensedFilesGroupedByLicense(installedPackages: InstalledPackage[]): Map<LicenseName, PackageLicensedFiles[]> {
    const result: Map<LicenseName, PackageLicensedFiles[]> = new Map();
    installedPackages.forEach(installedPackage => {
      // It is possible that different files in package have different licenses.
      // See the getPackageInfosForInstalledPackage method for details.
      const packageInfos = this.findPackageInfosForInstalledPackage(installedPackage);
      if(packageInfos.length === 0) {
        fail(`License for package '${installedPackage.name}-${installedPackage.version}' was not found`);
      }

      const allPackageLicensedFiles: Set<string> = new Set();
      packageInfos.forEach(packInfo => {
          const licensedFiles = this.filterFilesByPackageInfo(installedPackage, packInfo);
          if(licensedFiles.includedFiles.length === 0) {
            return;
          }
          const license = packInfo.license;
          if (!license.type.allowed) {
            fail(`Polygerrit bundle use files with invalid licence ${license.name} from` +
                ` the package ${installedPackage.name}`);
          }
          if(!result.has(license.name)) {
            result.set(license.name, []);
          }
          result.get(license.name)!.push({
            packageInfo: packInfo,
            filteredFiles: licensedFiles
          });
          licensedFiles.includedFiles.forEach((fileName) => {
            if(allPackageLicensedFiles.has(fileName)) {
              fail(`File '${fileName}' from '${installedPackage.name}' has multiple licenses.`)
            }
            allPackageLicensedFiles.add(fileName);
          });
        });
      if(allPackageLicensedFiles.size !== installedPackage.files.length) {
        fail(`Some files from '${installedPackage.name}' doesn't have a license, but they are used during build`);
      }
      });
    return result;
  }

  /** getInstalledPackages Collects information about all installed packages */
  private getInstalledPackages(nodeModulesFiles: ReadonlyArray<string>): InstalledPackage[] {
    const fullNonPackageNames: string[] = [];
    for (const p of this.packages) {
      if (p.nonPackages) {
        fullNonPackageNames.push(...p.nonPackages.map(name => `${p.name}/${name}`));
      }
    }
    const builder = new InsalledPackagesBuilder(new Set(fullNonPackageNames));
    // Register all package.json files - such files exists in the root folder of each module
    nodeModulesFiles.filter(f => path.basename(f) === "package.json")
      .forEach(packageJsonFile => builder.addPackageJson(packageJsonFile));
    // Iterate through all files. builder adds each file to appropriate package
    nodeModulesFiles.forEach(f => builder.addFile(f));
    return builder.build();
  }

  /**
   * findPackageInfosForInstalledPackage finds all possible licenses for the installedPackage.
   * It is possible, that different files in package have different licenses.
   * For example, @polymer/font-roboto-local package has files with 2 different licenses
   * - .js files - Polymer license (i.e. BSD),
   * - font files - Apache 2.0 license
   * In this case, the package is defined several times with different filesFilter
   */
  private findPackageInfosForInstalledPackage(installedPackage: InstalledPackage): PackageInfo[] {
    return this.packages.filter(packInfo => {
      if(packInfo.name !== installedPackage.name) {
        return false;
      }
      return !packInfo.versions || packInfo.versions.indexOf(installedPackage.version) >= 0;
    });
  }

  /** Returns list of files which have license defined in packageInfo */
  private filterFilesByPackageInfo(installedPackage: InstalledPackage, packageInfo: PackageInfo): FilteredFiles {
    const filter = packageInfo.filesFilter;
    if(!filter) {
      return {
        installedPackage,
        includedFiles: installedPackage.files,
        excludedFiles: [],
      }
    }
    return installedPackage.files.reduce((result: FilteredFiles, f) => {
      if(filter(f)) {
        result.includedFiles.push(f);
      } else {
        result.excludedFiles.push(f);
      }
      return result;
    }, {
      installedPackage,
      includedFiles: [],
      excludedFiles: []
    });
  }

  private getLicenseText(packageInfo: PackageLicensedFiles) {
    if(isSharedFileLicenseInfo(packageInfo.packageInfo.license)) {
      return this.sharedLicensesProvider.getText(packageInfo.packageInfo.license.sharedLicenseFile);
    } else {
      const filePath = path.join(packageInfo.filteredFiles.installedPackage.rootPath,
          packageInfo.packageInfo.license.packageLicenseFile)
      return fs.readFileSync(filePath, {encoding: 'utf-8'});
    }
  }
}