feat(@angular-devkit/build-angular): support i18n message extraction with Ivy

This change adds support for extracting i18n translation messages with an Ivy enabled application. This is accomplished by using the new extraction capabilities present in the `@angular/localize` package and will require version 10.1 or later of the package.  Since this change uses an new extraction method, it currently must be enabled during extraction by using the `--ivy` flag. The flag is a precaution to prevent unintentional breakage for existing applications but will become the default behavior for all Ivy enabled applications in a future release.

Closes #18275

Author: clydin

packages/angular_devkit/build_angular/package.json

@@ -75,6 +75,7 @@
   },
   "peerDependencies": {
     "@angular/compiler-cli": ">=10.1.0-next.0 < 11",
+    "@angular/localize": ">=10.1.0-next.0 < 11",
     "ng-packagr": "^10.0.0",
     "typescript": ">=3.9 < 3.10"
   },

packages/angular_devkit/build_angular/src/extract-i18n/index.ts

@@ -12,7 +12,10 @@ import {
 } from '@angular-devkit/architect';
 import { BuildResult, WebpackLoggingCallback, runWebpack } from '@angular-devkit/build-webpack';
 import { JsonObject } from '@angular-devkit/core';
+import type { ɵParsedMessage as LocalizeMessage } from '@angular/localize';
+import * as fs from 'fs';
 import * as path from 'path';
+import { gte as semverGte } from 'semver';
 import * as webpack from 'webpack';
 import {
   getAotConfig,
@@ -44,6 +47,32 @@ function getI18nOutfile(format: string | undefined) {
   }
 }
 
+async function getSerializer(format: Format, sourceLocale: string, basePath: string, useLegacyIds = true) {
+  switch (format) {
+    case Format.Xmb:
+      const { XmbTranslationSerializer } =
+      await import('@angular/localize/src/tools/src/extract/translation_files/xmb_translation_serializer');
+
+      // tslint:disable-next-line: no-any
+      return new XmbTranslationSerializer(basePath as any, useLegacyIds);
+    case Format.Xlf:
+    case Format.Xlif:
+    case Format.Xliff:
+      const { Xliff1TranslationSerializer } =
+        await import('@angular/localize/src/tools/src/extract/translation_files/xliff1_translation_serializer');
+
+      // tslint:disable-next-line: no-any
+      return new Xliff1TranslationSerializer(sourceLocale, basePath as any, useLegacyIds);
+    case Format.Xlf2:
+    case Format.Xliff2:
+      const { Xliff2TranslationSerializer } =
+        await import('@angular/localize/src/tools/src/extract/translation_files/xliff2_translation_serializer');
+
+      // tslint:disable-next-line: no-any
+      return new Xliff2TranslationSerializer(sourceLocale, basePath as any, useLegacyIds);
+  }
+}
+
 class InMemoryOutputPlugin {
   apply(compiler: webpack.Compiler): void {
     // tslint:disable-next-line:no-any
@@ -78,6 +107,9 @@ export async function execute(
     case Format.Xliff2:
       options.format = Format.Xlf2;
       break;
+    case undefined:
+      options.format = Format.Xlf;
+      break;
   }
 
   // We need to determine the outFile name so that AngularCompiler can retrieve it.
@@ -87,22 +119,25 @@ export async function execute(
     outFile = path.join(options.outputPath, outFile);
   }
 
-  const projectName = context.target && context.target.project;
-  if (!projectName) {
+  if (!context.target || !context.target.project) {
     throw new Error('The builder requires a target.');
   }
-  // target is verified in the above call
-  // tslint:disable-next-line: no-non-null-assertion
-  const metadata = await context.getProjectMetadata(context.target!);
+
+  const metadata = await context.getProjectMetadata(context.target);
   const i18n = createI18nOptions(metadata);
 
-  const { config } = await generateBrowserWebpackConfigFromContext(
+  let usingIvy = false;
+  const ivyMessages: LocalizeMessage[] = [];
+  const { config, projectRoot } = await generateBrowserWebpackConfigFromContext(
     {
       ...browserOptions,
       optimization: {
         scripts: false,
         styles: false,
       },
+      sourceMap: {
+        scripts: true,
+      },
       buildOptimizer: false,
       i18nLocale: options.i18nLocale || i18n.sourceLocale,
       i18nFormat: options.format,
@@ -115,15 +150,70 @@ export async function execute(
       deleteOutputPath: false,
     },
     context,
-    wco => [
-      { plugins: [new InMemoryOutputPlugin()] },
-      getCommonConfig(wco),
-      getAotConfig(wco, true),
-      getStylesConfig(wco),
-      getStatsConfig(wco),
-    ],
+    (wco) => {
+      const isIvyApplication = wco.tsConfig.options.enableIvy !== false;
+
+      // Ivy-based extraction is currently opt-in
+      if (options.ivy) {
+        if (!isIvyApplication) {
+          context.logger.warn(
+            'Ivy extraction enabled but application is not Ivy enabled. Extraction may fail.',
+          );
+        }
+        usingIvy = true;
+      } else if (isIvyApplication) {
+        context.logger.warn(
+          'Ivy extraction not enabled but application is Ivy enabled. ' +
+          'If the extraction fails, the `--ivy` flag will enable Ivy extraction.',
+        );
+      }
+
+      const partials = [
+        { plugins: [new InMemoryOutputPlugin()] },
+        getCommonConfig(wco),
+        // Only use VE extraction if not using Ivy
+        getAotConfig(wco, !usingIvy),
+        getStylesConfig(wco),
+        getStatsConfig(wco),
+      ];
+
+      // Add Ivy application file extractor support
+      if (usingIvy) {
+        partials.unshift({
+          module: {
+            rules: [
+              {
+                test: /\.ts$/,
+                loader: require.resolve('./ivy-extract-loader'),
+                options: {
+                  messageHandler: (messages: LocalizeMessage[]) => ivyMessages.push(...messages),
+                },
+              },
+            ],
+          },
+        });
+      }
+
+      return partials;
+    },
   );
 
+  if (usingIvy) {
+    let validLocalizePackage = false;
+    try {
+      const { version: localizeVersion } = require('@angular/localize/package.json');
+      validLocalizePackage = semverGte(localizeVersion, '10.1.0-next.0', { includePrerelease: true });
+    } catch {}
+
+    if (!validLocalizePackage) {
+      context.logger.error(
+        "Ivy extraction requires the '@angular/localize' package version 10.1.0 or higher.",
+      );
+
+      return { success: false };
+    }
+  }
+
   const logging: WebpackLoggingCallback = (stats, config) => {
     const json = stats.toJson({ errors: true, warnings: true });
 
@@ -136,10 +226,39 @@ export async function execute(
     }
   };
 
-  return runWebpack(config, context, {
+  const webpackResult = await runWebpack(config, context, {
     logging,
     webpackFactory: await import('webpack'),
   }).toPromise();
+
+  // Complete if using VE
+  if (!usingIvy) {
+    return webpackResult;
+  }
+
+  // Nothing to process if the Webpack build failed
+  if (!webpackResult.success) {
+    return webpackResult;
+  }
+
+  // Serialize all extracted messages
+  const serializer = await getSerializer(
+    options.format,
+    i18n.sourceLocale,
+    config.context || projectRoot,
+  );
+  const content = serializer.serialize(ivyMessages);
+
+  // Ensure directory exists
+  const outputPath = path.dirname(outFile);
+  if (!fs.existsSync(outputPath)) {
+    fs.mkdirSync(outputPath, { recursive: true });
+  }
+
+  // Write translation file
+  fs.writeFileSync(outFile, content);
+
+  return webpackResult;
 }
 
 export default createBuilder<JsonObject & ExtractI18nBuilderOptions>(execute);

packages/angular_devkit/build_angular/src/extract-i18n/ivy-extract-loader.ts

@@ -0,0 +1,85 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import { MessageExtractor } from '@angular/localize/src/tools/src/extract/extraction';
+import { getOptions } from 'loader-utils';
+import * as nodePath from 'path';
+
+interface LocalizeExtractLoaderOptions {
+  messageHandler: (messages: import('@angular/localize').ɵParsedMessage[]) => void;
+}
+
+export default function localizeExtractLoader(
+  this: import('webpack').loader.LoaderContext,
+  content: string,
+  // Source map types are broken in the webpack type definitions
+  // tslint:disable-next-line: no-any
+  map: any,
+) {
+  const loaderContext = this;
+
+  // Casts are needed to workaround the loader-utils typings limited support for option values
+  const options = (getOptions(this) as unknown) as LocalizeExtractLoaderOptions | undefined;
+
+  // Setup a Webpack-based logger instance
+  const logger = {
+    // level 2 is warnings
+    level: 2,
+    debug(...args: string[]): void {
+      // tslint:disable-next-line: no-console
+      console.debug(...args);
+    },
+    info(...args: string[]): void {
+      loaderContext.emitWarning(args.join(''));
+    },
+    warn(...args: string[]): void {
+      loaderContext.emitWarning(args.join(''));
+    },
+    error(...args: string[]): void {
+      loaderContext.emitError(args.join(''));
+    },
+  };
+
+  // Setup a virtual file system instance for the extractor
+  // * MessageExtractor itself uses readFile and resolve
+  // * Internal SourceFileLoader (sourcemap support) uses dirname, exists, readFile, and resolve
+  const filesystem = {
+    readFile(path: string): string {
+      if (path === loaderContext.resourcePath) {
+        return content;
+      } else if (path === loaderContext.resourcePath + '.map') {
+        return typeof map === 'string' ? map : JSON.stringify(map);
+      } else {
+        throw new Error('Unknown file requested.');
+      }
+    },
+    resolve(...paths: string[]): string {
+      return nodePath.resolve(...paths);
+    },
+    exists(path: string): boolean {
+      return path === loaderContext.resourcePath || path === loaderContext.resourcePath + '.map';
+    },
+    dirname(path: string): string {
+      return nodePath.dirname(path);
+    },
+  };
+
+  // tslint:disable-next-line: no-any
+  const extractor = new MessageExtractor(filesystem as any, logger, {
+    // tslint:disable-next-line: no-any
+    basePath: this.rootContext as any,
+    useSourceMaps: !!map,
+  });
+
+  const messages = extractor.extractMessages(loaderContext.resourcePath);
+  if (messages.length > 0) {
+    options?.messageHandler(messages);
+  }
+
+  // Pass through the original content now that messages have been extracted
+  this.callback(undefined, content, map);
+}

packages/angular_devkit/build_angular/src/extract-i18n/schema.json

@@ -41,6 +41,10 @@
       "description": "Specifies the source language of the application.",
       "x-deprecated": "Use 'i18n' project level sub-option 'sourceLocale' instead."
     },
+    "ivy": {
+      "type": "boolean",
+      "description": "Use Ivy compiler to extract translations."
+    },
     "progress": {
       "type": "boolean",
       "description": "Log progress to the console.",

tests/legacy-cli/e2e/tests/i18n/extract-ivy.ts

@@ -0,0 +1,60 @@
+import { join } from 'path';
+import { getGlobalVariable } from '../../utils/env';
+import { writeFile } from '../../utils/fs';
+import { ng, npm } from '../../utils/process';
+import { updateJsonFile } from '../../utils/project';
+import { expectToFail } from '../../utils/utils';
+import { readNgVersion } from '../../utils/version';
+
+export default async function() {
+  // Ivy only test
+  if (getGlobalVariable('argv')['ve']) {
+    return;
+  }
+
+  // Setup an i18n enabled component
+  await ng('generate', 'component', 'i18n-test');
+  await writeFile(
+    join('src/app/i18n-test', 'i18n-test.component.html'),
+    '<p i18n>Hello world</p>',
+  );
+
+  // Should fail with --ivy flag if `@angular/localize` is missing
+  const { message: message1 } = await expectToFail(() => ng('xi18n', '--ivy'));
+  if (!message1.includes(`Ivy extraction requires the '@angular/localize' package version 10.1.0 or higher.`)) {
+    throw new Error('Expected localize package error message when missing');
+  }
+
+  // Should fail with --ivy flag if `@angular/localize` is wrong version
+  await npm('install', '@angular/localize@9');
+  const { message: message2 } = await expectToFail(() => ng('xi18n', '--ivy'));
+  if (!message2.includes(`Ivy extraction requires the '@angular/localize' package version 10.1.0 or higher.`)) {
+    throw new Error('Expected localize package error message when wrong version');
+  }
+
+  // Install correct version
+  let localizeVersion = '@angular/localize@' + readNgVersion();
+  if (getGlobalVariable('argv')['ng-snapshots']) {
+    localizeVersion = require('../../ng-snapshot/package.json').dependencies['@angular/localize'];
+  }
+  await npm('install', `${localizeVersion}`);
+
+  // Should show ivy enabled application warning without --ivy flag
+  const { stderr: message3 } = await ng('xi18n');
+  if (!message3.includes(`Ivy extraction not enabled but application is Ivy enabled.`)) {
+    throw new Error('Expected ivy enabled application warning');
+  }
+
+  // Disable Ivy
+  await updateJsonFile('tsconfig.json', config => {
+    const { angularCompilerOptions = {} } = config;
+    angularCompilerOptions.enableIvy = false;
+    config.angularCompilerOptions = angularCompilerOptions;
+  });
+
+  // Should show ivy disabled application warning with --ivy flag and enableIvy false
+  const { message: message4 } = await expectToFail(() => ng('xi18n', '--ivy'));
+  if (!message4.includes(`Ivy extraction enabled but application is not Ivy enabled.`)) {
+    throw new Error('Expected ivy disabled application warning');
+  }
+}