tools/node_tools/node_modules_licenses/licenses-map.ts - gerrit - Git at Google

 /**
  * @license
  * Copyright (C) 2020 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 import * as path from "path";
 import * as fs from "fs";
 import {isSharedFileLicenseInfo, LicenseInfo, PackageInfo} from "./package-license-info";
 import {LicenseName, PackageName, FilePath, PackageVersion} from "./base-types";
 import { InstalledPackage, InsalledPackagesBuilder } from "./installed-node-modules-map";
 import {SharedLicensesProvider} from "./shared-licenses-provider";
 import {fail} from "./utils";

 interface FilteredFiles {
   installedPackage: InstalledPackage;
   /** Path relative to installedPackage */
   includedFiles: FilePath[];
   /** Path relative to installedPackage */
   excludedFiles: FilePath[];
 }
 interface PackageLicensedFiles {
   packageInfo: PackageInfo;
   filteredFiles: FilteredFiles;
 }

 enum LicensedFilesDisplayInfoType {
   AllFiles = "AllFiles",
   OnlySpecificFiles = "OnlySpecificFiles",
   AllFilesExceptSpecific = "AllFilesExceptSpecific",
 }

 interface AllFiles {
   kind: LicensedFilesDisplayInfoType.AllFiles;
 }

 interface AllFilesExceptSpecificFiles {
   kind: LicensedFilesDisplayInfoType.AllFilesExceptSpecific;
   /** Path relative to installedPackage */
   files: FilePath[]
 }

 interface OnlySpecificFiles {
   kind: LicensedFilesDisplayInfoType.OnlySpecificFiles;
   /** Path relative to installedPackage */
   files: FilePath[];
 }

 export type FilteredFilesDisplayInfo = AllFiles | AllFilesExceptSpecificFiles | OnlySpecificFiles;

 export interface LicenseMapPackageInfo {
   name: PackageName;
   version: PackageVersion;
   licensedFiles: FilteredFilesDisplayInfo;
 }

 export interface LicenseMapItem {
   licenseName: LicenseName,
   licenseText: string;
   packages: LicenseMapPackageInfo[];
 }

 export type LicensesMap = { [licenseName: string]:  LicenseMapItem }

 /** LicenseMapGenerator calculates license map for nodeModulesFiles.
  */
 export class LicenseMapGenerator {
   /**
    *  packages - information about licenses for packages
    *  sharedLicensesProvider - returns text for shared licenses by name
    */
   public constructor(private readonly packages: ReadonlyArray<PackageInfo>, private readonly sharedLicensesProvider: SharedLicensesProvider) {
   }

   /** generateMap calculates LicenseMap for nodeModulesFiles
    * Each (key, value) pair in this map has the following information:
    * The key is a license name.
    * The value contains information about packages and files for which this license
    * is applied.
    *
    * The method tries to provide information in a compact way:
    * 1) If all files in the package have the same license, then it stores a name of the package
    *    without list of files
    * 2) If different files in the package has different licenses, then the method calculates
    *    which list is shorter - list of included files or list of excluded files.
    */
   public generateMap(nodeModulesFiles: ReadonlyArray<string>): LicensesMap {
     const installedPackages = this.getInstalledPackages(nodeModulesFiles);
     const licensedFilesGroupedByLicense = this.getLicensedFilesGroupedByLicense(installedPackages);

     const result: LicensesMap = {};
     licensedFilesGroupedByLicense.forEach((packageLicensedFiles, licenseName) => {
       result[licenseName] = this.getLicenseMapItem(licenseName, packageLicensedFiles);
     });
     return result;
   }

   private getLicenseMapItem(licenseName: string, packagesLicensedFiles: PackageLicensedFiles[]): LicenseMapItem {
     const packages: LicenseMapPackageInfo[] = [];
     let licenseText: string = "";
     packagesLicensedFiles.forEach((packageLicensedFiles) => {
       const packageLicenseText = this.getLicenseText(packageLicensedFiles);
       if (licenseText.length !== 0 && packageLicenseText !== licenseText) {
         fail(`There are different license texts for license '${licenseName}'.\n` +
             "Probably, you have multiple version of the same package.\n" +
             "In this case you must define different license name for each version"
         );
       }
       licenseText = packageLicenseText;
       packages.push({
         name: packageLicensedFiles.packageInfo.name,
         version: packageLicensedFiles.filteredFiles.installedPackage.version,
         licensedFiles: this.getFilteredFilesDisplayInfo(packageLicensedFiles.filteredFiles)
       });
     });
     return {
       licenseName,
       licenseText,
       packages
     };
   }

   /** getFilteredFilesDisplayInfo calculates the best method to display information about
    * filteredFiles in filteredFiles.installedPackage
    * In the current implementation - the best method is a method with a shorter list of files
    *
    * Each {@link PackageInfo} has a filter for files (by default all files
    * are included). Applying filter to files from an installedPackage returns 2 lists of files:
    * includedFiles (i.e. files with {@link PackageInfo.license} license) and excludedFiles
    * (i.e. files which have different license(s)).
    *
    * A text representaion of license-map must have full information about per-file licenses if
    * needed, but we want to produce files lists there as short as possible
    */
   private getFilteredFilesDisplayInfo(filteredFiles: FilteredFiles): FilteredFilesDisplayInfo {
     if (filteredFiles.includedFiles.length > 0 && filteredFiles.excludedFiles.length === 0) {
       /** All files from package are included (i.e. all files have the same license).
        * It is enough to print only installedPackage name.
        * */
       return {
         kind: LicensedFilesDisplayInfoType.AllFiles
       }
     } else if (filteredFiles.includedFiles.length <= filteredFiles.excludedFiles.length) {
       /** After applying filter, the number of files with filteredFiles.license is less than
        * the number of excluded files.
        * It is more convenient to print information about license in the format:
        * GIT license - fontPackage (only files A,B,C)*/
       return {
         kind: LicensedFilesDisplayInfoType.OnlySpecificFiles,
         files: filteredFiles.includedFiles
       };
     } else {
       /** Only few files from filteredFiles.installedPackage has filteredFiles.license.
        * It is more convenient to print information about license in the format:
        * Apache license - fontPackage (except files A,B,C)
        */
       return {
         kind: LicensedFilesDisplayInfoType.AllFilesExceptSpecific,
         files: filteredFiles.excludedFiles
       }
     }
   }

   private getLicensedFilesGroupedByLicense(installedPackages: InstalledPackage[]): Map<LicenseName, PackageLicensedFiles[]> {
     const result: Map<LicenseName, PackageLicensedFiles[]> = new Map();
     installedPackages.forEach(installedPackage => {
       // It is possible that different files in package have different licenses.
       // See the getPackageInfosForInstalledPackage method for details.
       const packageInfos = this.findPackageInfosForInstalledPackage(installedPackage);
       if(packageInfos.length === 0) {
         fail(`License for package '${installedPackage.name}-${installedPackage.version}' was not found`);
       }

       const allPackageLicensedFiles: Set<string> = new Set();
       packageInfos.forEach(packInfo => {
           const licensedFiles = this.filterFilesByPackageInfo(installedPackage, packInfo);
           if(licensedFiles.includedFiles.length === 0) {
             return;
           }
           const license = packInfo.license;
           if (!license.type.allowed) {
             fail(`Polygerrit bundle use files with invalid licence ${license.name} from` +
                 ` the package ${installedPackage.name}`);
           }
           if(!result.has(license.name)) {
             result.set(license.name, []);
           }
           result.get(license.name)!.push({
             packageInfo: packInfo,
             filteredFiles: licensedFiles
           });
           licensedFiles.includedFiles.forEach((fileName) => {
             if(allPackageLicensedFiles.has(fileName)) {
               fail(`File '${fileName}' from '${installedPackage.name}' has multiple licenses.`)
             }
             allPackageLicensedFiles.add(fileName);
           });
         });
       if(allPackageLicensedFiles.size !== installedPackage.files.length) {
         fail(`Some files from '${installedPackage.name}' doesn't have a license, but they are used during build`);
       }
       });
     return result;
   }

   /** getInstalledPackages Collects information about all installed packages */
   private getInstalledPackages(nodeModulesFiles: ReadonlyArray<string>): InstalledPackage[] {
     const fullNonPackageNames: string[] = [];
     for (const p of this.packages) {
       if (p.nonPackages) {
         fullNonPackageNames.push(...p.nonPackages.map(name => `${p.name}/${name}`));
       }
     }
     const builder = new InsalledPackagesBuilder(new Set(fullNonPackageNames));
     // Register all package.json files - such files exists in the root folder of each module
     nodeModulesFiles.filter(f => path.basename(f) === "package.json")
       .forEach(packageJsonFile => builder.addPackageJson(packageJsonFile));
     // Iterate through all files. builder adds each file to appropriate package
     nodeModulesFiles.forEach(f => builder.addFile(f));
     return builder.build();
   }

   /**
    * findPackageInfosForInstalledPackage finds all possible licenses for the installedPackage.
    * It is possible, that different files in package have different licenses.
    * For example, @polymer/font-roboto-local package has files with 2 different licenses
    * - .js files - Polymer license (i.e. BSD),
    * - font files - Apache 2.0 license
    * In this case, the package is defined several times with different filesFilter
    */
   private findPackageInfosForInstalledPackage(installedPackage: InstalledPackage): PackageInfo[] {
     return this.packages.filter(packInfo => {
       if(packInfo.name !== installedPackage.name) {
         return false;
       }
       return !packInfo.versions || packInfo.versions.indexOf(installedPackage.version) >= 0;
     });
   }

   /** Returns list of files which have license defined in packageInfo */
   private filterFilesByPackageInfo(installedPackage: InstalledPackage, packageInfo: PackageInfo): FilteredFiles {
     const filter = packageInfo.filesFilter;
     if(!filter) {
       return {
         installedPackage,
         includedFiles: installedPackage.files,
         excludedFiles: [],
       }
     }
     return installedPackage.files.reduce((result: FilteredFiles, f) => {
       if(filter(f)) {
         result.includedFiles.push(f);
       } else {
         result.excludedFiles.push(f);
       }
       return result;
     }, {
       installedPackage,
       includedFiles: [],
       excludedFiles: []
     });
   }

   private getLicenseText(packageInfo: PackageLicensedFiles) {
     if(isSharedFileLicenseInfo(packageInfo.packageInfo.license)) {
       return this.sharedLicensesProvider.getText(packageInfo.packageInfo.license.sharedLicenseFile);
     } else {
       const filePath = path.join(packageInfo.filteredFiles.installedPackage.rootPath,
           packageInfo.packageInfo.license.packageLicenseFile)
       return fs.readFileSync(filePath, {encoding: 'utf-8'});
     }
   }
 }
	/**
	* @license
	* Copyright (C) 2020 The Android Open Source Project
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	import * as path from "path";
	import * as fs from "fs";
	import {isSharedFileLicenseInfo, LicenseInfo, PackageInfo} from "./package-license-info";
	import {LicenseName, PackageName, FilePath, PackageVersion} from "./base-types";
	import { InstalledPackage, InsalledPackagesBuilder } from "./installed-node-modules-map";
	import {SharedLicensesProvider} from "./shared-licenses-provider";
	import {fail} from "./utils";

	interface FilteredFiles {
	installedPackage: InstalledPackage;
	/** Path relative to installedPackage */
	includedFiles: FilePath[];
	/** Path relative to installedPackage */
	excludedFiles: FilePath[];
	}
	interface PackageLicensedFiles {
	packageInfo: PackageInfo;
	filteredFiles: FilteredFiles;
	}

	enum LicensedFilesDisplayInfoType {
	AllFiles = "AllFiles",
	OnlySpecificFiles = "OnlySpecificFiles",
	AllFilesExceptSpecific = "AllFilesExceptSpecific",
	}

	interface AllFiles {
	kind: LicensedFilesDisplayInfoType.AllFiles;
	}

	interface AllFilesExceptSpecificFiles {
	kind: LicensedFilesDisplayInfoType.AllFilesExceptSpecific;
	/** Path relative to installedPackage */
	files: FilePath[]
	}

	interface OnlySpecificFiles {
	kind: LicensedFilesDisplayInfoType.OnlySpecificFiles;
	/** Path relative to installedPackage */
	files: FilePath[];
	}

	export type FilteredFilesDisplayInfo = AllFiles \| AllFilesExceptSpecificFiles \| OnlySpecificFiles;

	export interface LicenseMapPackageInfo {
	name: PackageName;
	version: PackageVersion;
	licensedFiles: FilteredFilesDisplayInfo;
	}

	export interface LicenseMapItem {
	licenseName: LicenseName,
	licenseText: string;
	packages: LicenseMapPackageInfo[];
	}

	export type LicensesMap = { [licenseName: string]: LicenseMapItem }

	/** LicenseMapGenerator calculates license map for nodeModulesFiles.
	*/
	export class LicenseMapGenerator {
	/**
	* packages - information about licenses for packages
	* sharedLicensesProvider - returns text for shared licenses by name
	*/
	public constructor(private readonly packages: ReadonlyArray<PackageInfo>, private readonly sharedLicensesProvider: SharedLicensesProvider) {
	}

	/** generateMap calculates LicenseMap for nodeModulesFiles
	* Each (key, value) pair in this map has the following information:
	* The key is a license name.
	* The value contains information about packages and files for which this license
	* is applied.
	*
	* The method tries to provide information in a compact way:
	* 1) If all files in the package have the same license, then it stores a name of the package
	* without list of files
	* 2) If different files in the package has different licenses, then the method calculates
	* which list is shorter - list of included files or list of excluded files.
	*/
	public generateMap(nodeModulesFiles: ReadonlyArray<string>): LicensesMap {
	const installedPackages = this.getInstalledPackages(nodeModulesFiles);
	const licensedFilesGroupedByLicense = this.getLicensedFilesGroupedByLicense(installedPackages);

	const result: LicensesMap = {};
	licensedFilesGroupedByLicense.forEach((packageLicensedFiles, licenseName) => {
	result[licenseName] = this.getLicenseMapItem(licenseName, packageLicensedFiles);
	});
	return result;
	}

	private getLicenseMapItem(licenseName: string, packagesLicensedFiles: PackageLicensedFiles[]): LicenseMapItem {
	const packages: LicenseMapPackageInfo[] = [];
	let licenseText: string = "";
	packagesLicensedFiles.forEach((packageLicensedFiles) => {
	const packageLicenseText = this.getLicenseText(packageLicensedFiles);
	if (licenseText.length !== 0 && packageLicenseText !== licenseText) {
	fail(`There are different license texts for license '${licenseName}'.\n` +
	"Probably, you have multiple version of the same package.\n" +
	"In this case you must define different license name for each version"
	);
	}
	licenseText = packageLicenseText;
	packages.push({
	name: packageLicensedFiles.packageInfo.name,
	version: packageLicensedFiles.filteredFiles.installedPackage.version,
	licensedFiles: this.getFilteredFilesDisplayInfo(packageLicensedFiles.filteredFiles)
	});
	});
	return {
	licenseName,
	licenseText,
	packages
	};
	}

	/** getFilteredFilesDisplayInfo calculates the best method to display information about
	* filteredFiles in filteredFiles.installedPackage
	* In the current implementation - the best method is a method with a shorter list of files
	*
	* Each {@link PackageInfo} has a filter for files (by default all files
	* are included). Applying filter to files from an installedPackage returns 2 lists of files:
	* includedFiles (i.e. files with {@link PackageInfo.license} license) and excludedFiles
	* (i.e. files which have different license(s)).
	*
	* A text representaion of license-map must have full information about per-file licenses if
	* needed, but we want to produce files lists there as short as possible
	*/
	private getFilteredFilesDisplayInfo(filteredFiles: FilteredFiles): FilteredFilesDisplayInfo {
	if (filteredFiles.includedFiles.length > 0 && filteredFiles.excludedFiles.length === 0) {
	/** All files from package are included (i.e. all files have the same license).
	* It is enough to print only installedPackage name.
	* */
	return {
	kind: LicensedFilesDisplayInfoType.AllFiles
	}
	} else if (filteredFiles.includedFiles.length <= filteredFiles.excludedFiles.length) {
	/** After applying filter, the number of files with filteredFiles.license is less than
	* the number of excluded files.
	* It is more convenient to print information about license in the format:
	* GIT license - fontPackage (only files A,B,C)*/
	return {
	kind: LicensedFilesDisplayInfoType.OnlySpecificFiles,
	files: filteredFiles.includedFiles
	};
	} else {
	/** Only few files from filteredFiles.installedPackage has filteredFiles.license.
	* It is more convenient to print information about license in the format:
	* Apache license - fontPackage (except files A,B,C)
	*/
	return {
	kind: LicensedFilesDisplayInfoType.AllFilesExceptSpecific,
	files: filteredFiles.excludedFiles
	}
	}
	}

	private getLicensedFilesGroupedByLicense(installedPackages: InstalledPackage[]): Map<LicenseName, PackageLicensedFiles[]> {
	const result: Map<LicenseName, PackageLicensedFiles[]> = new Map();
	installedPackages.forEach(installedPackage => {
	// It is possible that different files in package have different licenses.
	// See the getPackageInfosForInstalledPackage method for details.
	const packageInfos = this.findPackageInfosForInstalledPackage(installedPackage);
	if(packageInfos.length === 0) {
	fail(`License for package '${installedPackage.name}-${installedPackage.version}' was not found`);
	}

	const allPackageLicensedFiles: Set<string> = new Set();
	packageInfos.forEach(packInfo => {
	const licensedFiles = this.filterFilesByPackageInfo(installedPackage, packInfo);
	if(licensedFiles.includedFiles.length === 0) {
	return;
	}
	const license = packInfo.license;
	if (!license.type.allowed) {
	fail(`Polygerrit bundle use files with invalid licence ${license.name} from` +
	` the package ${installedPackage.name}`);
	}
	if(!result.has(license.name)) {
	result.set(license.name, []);
	}
	result.get(license.name)!.push({
	packageInfo: packInfo,
	filteredFiles: licensedFiles
	});
	licensedFiles.includedFiles.forEach((fileName) => {
	if(allPackageLicensedFiles.has(fileName)) {
	fail(`File '${fileName}' from '${installedPackage.name}' has multiple licenses.`)
	}
	allPackageLicensedFiles.add(fileName);
	});
	});
	if(allPackageLicensedFiles.size !== installedPackage.files.length) {
	fail(`Some files from '${installedPackage.name}' doesn't have a license, but they are used during build`);
	}
	});
	return result;
	}

	/** getInstalledPackages Collects information about all installed packages */
	private getInstalledPackages(nodeModulesFiles: ReadonlyArray<string>): InstalledPackage[] {
	const fullNonPackageNames: string[] = [];
	for (const p of this.packages) {
	if (p.nonPackages) {
	fullNonPackageNames.push(...p.nonPackages.map(name => `${p.name}/${name}`));
	}
	}
	const builder = new InsalledPackagesBuilder(new Set(fullNonPackageNames));
	// Register all package.json files - such files exists in the root folder of each module
	nodeModulesFiles.filter(f => path.basename(f) === "package.json")
	.forEach(packageJsonFile => builder.addPackageJson(packageJsonFile));
	// Iterate through all files. builder adds each file to appropriate package
	nodeModulesFiles.forEach(f => builder.addFile(f));
	return builder.build();
	}

	/**
	* findPackageInfosForInstalledPackage finds all possible licenses for the installedPackage.
	* It is possible, that different files in package have different licenses.
	* For example, @polymer/font-roboto-local package has files with 2 different licenses
	* - .js files - Polymer license (i.e. BSD),
	* - font files - Apache 2.0 license
	* In this case, the package is defined several times with different filesFilter
	*/
	private findPackageInfosForInstalledPackage(installedPackage: InstalledPackage): PackageInfo[] {
	return this.packages.filter(packInfo => {
	if(packInfo.name !== installedPackage.name) {
	return false;
	}
	return !packInfo.versions \|\| packInfo.versions.indexOf(installedPackage.version) >= 0;
	});
	}

	/** Returns list of files which have license defined in packageInfo */
	private filterFilesByPackageInfo(installedPackage: InstalledPackage, packageInfo: PackageInfo): FilteredFiles {
	const filter = packageInfo.filesFilter;
	if(!filter) {
	return {
	installedPackage,
	includedFiles: installedPackage.files,
	excludedFiles: [],
	}
	}
	return installedPackage.files.reduce((result: FilteredFiles, f) => {
	if(filter(f)) {
	result.includedFiles.push(f);
	} else {
	result.excludedFiles.push(f);
	}
	return result;
	}, {
	installedPackage,
	includedFiles: [],
	excludedFiles: []
	});
	}

	private getLicenseText(packageInfo: PackageLicensedFiles) {
	if(isSharedFileLicenseInfo(packageInfo.packageInfo.license)) {
	return this.sharedLicensesProvider.getText(packageInfo.packageInfo.license.sharedLicenseFile);
	} else {
	const filePath = path.join(packageInfo.filteredFiles.installedPackage.rootPath,
	packageInfo.packageInfo.license.packageLicenseFile)
	return fs.readFileSync(filePath, {encoding: 'utf-8'});
	}
	}
	}