anvil-build/anvil/rules/closure_js_rules.py

# Copyright 2012 Google Inc. All Rights Reserved.

"""Closure compiler rules for the build system.

Contains the following rules:
closure_js_lint
closure_js_fixstyle
closure_js_library

Assumes Closure Linter is installed and on the path.
"""

__author__ = 'benvanik@google.com (Ben Vanik)'


import io
import os
import re

from anvil.context import RuleContext
from anvil.rule import Rule, build_rule
from anvil.task import (Task, ExecutableTask, JavaExecutableTask,
    WriteFileTask)
import anvil.util


@build_rule('closure_js_lint')
class ClosureJsLintRule(Rule):
  """A set of linted JS files.
  Passes all source files through the Closure style linter.

  Similar to file_set, all source files are globbed together and de-duplicated
  before being passed on as outputs. If a src_filter is provided then it is used
  to filter all sources.

  Inputs:
    srcs: Source JS file paths.
    namespaces: A list of Closurized namespaces.
    linter_path: Path to the closure_linter module. If omitted the system
        gjslint will be used.

  Outputs:
    All of the source file paths, passed-through unmodified.
  """

  def __init__(self, name, namespaces, linter_path=None, *args, **kwargs):
    """Initializes a Closure JS lint rule.

    Args:
      name: Rule name.
      namespaces: A list of Closurized namespaces.
      linter_path: Path to the closure_linter module. If omitted the system
          gjslint will be used.
    """
    super(ClosureJsLintRule, self).__init__(name, *args, **kwargs)
    self.src_filter = '*.js'
    self._command = 'gjslint'
    self._extra_args = ['--nobeep',]
    self.namespaces = []
    self.namespaces.extend(namespaces)
    self.linter_path = linter_path

  class _Context(RuleContext):
    def begin(self):
      super(ClosureJsLintRule._Context, self).begin()
      self._append_output_paths(self.src_paths)

      # Skip if cache hit
      if self._check_if_cached():
        self._succeed()
        return

      namespaces = ','.join(self.rule.namespaces)
      args = [
          '--strict',
          '--jslint_error=all',
          '--closurized_namespaces=%s' % (namespaces),
          ]

      command = self.rule._command
      env = None
      if self.rule.linter_path:
        command = 'python'
        env = {
            'PYTHONPATH': self.rule.linter_path,
            }
        args = [
            os.path.join(
                self.rule.linter_path,
                'closure_linter/%s.py' % (self.rule._command)),
            ] + args

      # TODO(benvanik): only changed paths
      # Exclude any path containing build-*
      for src_path in self.file_delta.changed_files:
        if (src_path.find('build-out%s' % os.sep) == -1 and
            src_path.find('build-gen%s' % os.sep) == -1):
          args.append(src_path)

      d = self._run_task_async(ExecutableTask(
          self.build_env, command, args, env=env,
          pretty_name=str(self.rule)))
      # TODO(benvanik): pull out errors?
      self._chain(d)


@build_rule('closure_js_fixstyle')
class ClosureJsFixStyleRule(ClosureJsLintRule):
  """A set of style-corrected JS files.
  Passes all source files through the Closure style fixer.

  Similar to file_set, all source files are globbed together and de-duplicated
  before being passed on as outputs. If a src_filter is provided then it is used
  to filter all sources.

  This rule is special in that it fixes the style of the source files in-place,
  overwriting them in their source path.

  Inputs:
    srcs: Source JS file paths.
    namespaces: A list of Closurized namespaces.
    linter_path: Path to the closure_linter module. If omitted the system
        gjslint will be used.

  Outputs:
    All of the source file paths, passed-through post-correction.
  """

  def __init__(self, name, namespaces, linter_path=None, *args, **kwargs):
    """Initializes a Closure JS style fixing rule.

    Args:
      name: Rule name.
      namespaces: A list of Closurized namespaces.
      linter_path: Path to the closure_linter module. If omitted the system
          gjslint will be used.
    """
    super(ClosureJsFixStyleRule, self).__init__(name, namespaces,
        linter_path=linter_path, *args, **kwargs)
    self._command = 'fixjsstyle'
    self._extra_args = []


# TODO(benvanik): support non-closure code
# TODO(benvanik): support AMD modules
@build_rule('closure_js_library')
class ClosureJsLibraryRule(Rule):
  """A Closure compiler JavaScript library.
  Uses the Closure compiler to build a library from the given source files.

  Processes input files using the Closure compiler in the given mode.
  goog.provide and goog.require are used to order the files when concatenated.
  In SIMPLE and ADVANCED modes dependencies are used to remove dead code.

  If UNCOMPILED mode is used, only a -deps.js file is generated. This is a fast
  operation and can be used when interactively running code in a browser in
  uncompiled mode.

  In DEPS mode then only a -deps.js file is generated.

  A compiler JAR must be provided.

  Inputs:
    srcs: All source JS files.
    mode: Compilation mode, one of ['DEPS', UNCOMPILED', 'SIMPLE', 'ADVANCED'].
    compiler_jar: Path to a compiler .jar file.
    entry_points: A list of entry points, such as 'myapp.start'.
    pretty_print: True to pretty print the output.
    debug: True to enable Closure DEBUG consts.
    compiler_flags: A list of string compiler flags.
    externs: Additional extern .js files.
    wrap_with_global: Wrap all output in a closure and call with the given
          global object.
          Example - 'global' -> (function(){...code...}).call(global);
    out: Optional output name. If none is provided than the rule name will be
        used.
    deps_out: Base name for -deps.js file.
        Example - 'library' -> 'library-deps.js'
    file_list_out: A list of files in sorted order required for the given
        entry points.
        Example - 'all_files.txt'

  Outputs:
    A single compiled JS file. If no out is specified a file with the name of
    the rule will be created.
  """

  def __init__(self, name, mode, compiler_jar, entry_points,
        pretty_print=False, debug=False,
        compiler_flags=None, externs=None, wrap_with_global=None,
        out=None, deps_out=None, file_list_out=None,
        *args, **kwargs):
    """Initializes a Closure JS library rule.

    Args:
      name: Rule name.
      mode: Compilation mode, one of ['UNCOMPILED', 'SIMPLE', 'ADVANCED'].
      compiler_jar: Path to a compiler .jar file.
      entry_points: A list of entry points, such as 'myapp.start'.
      pretty_print: True to pretty print the output.
      debug: True to enable Closure DEBUG consts.
      compiler_flags: A list of string compiler flags.
      externs: Additional extern .js files.
      wrap_with_global: Wrap all output in a closure and call with the given
          global object.
          Example - 'global' -> (function(){...code...}).call(global);
      out: Optional output name.
      deps_out: Base name for -deps.js file.
          Example - 'library' -> 'library-deps.js'
      file_list_out: A list of files in sorted order required for the given
          entry points.
          Example - 'all_files.txt'
    """
    super(ClosureJsLibraryRule, self).__init__(name, *args, **kwargs)
    self.src_filter = '*.js'
    self.mode = mode
    self.compiler_jar = compiler_jar
    self._append_dependent_paths([self.compiler_jar])
    self.pretty_print = pretty_print
    self.debug = debug

    self.entry_points = []
    if isinstance(entry_points, str):
      self.entry_points.append(entry_points)
    elif entry_points:
      self.entry_points.extend(entry_points)

    self.compiler_flags = []
    if compiler_flags:
      self.compiler_flags.extend(compiler_flags)

    self.externs = []
    if externs:
      self.externs.extend(externs)
    self._append_dependent_paths(self.externs)

    self.wrap_with_global = wrap_with_global
    self.out = out
    self.deps_out = deps_out
    self.file_list_out = file_list_out

  class _Context(RuleContext):
    def begin(self):
      super(ClosureJsLibraryRule._Context, self).begin()

      jar_path = self._resolve_input_files([self.rule.compiler_jar])[0]
      args = [
          '--process_closure_primitives',
          '--generate_exports',
          '--summary_detail_level=3',
          '--warning_level=VERBOSE',
          ]
      args.extend(self.rule.compiler_flags)

      deps_only = False
      compiling = False
      if self.rule.mode == 'DEPS':
        deps_only = True
      elif self.rule.mode == 'UNCOMPILED':
        compiling = True
        args.append('--compilation_level=WHITESPACE_ONLY')
        args.append('--formatting=PRETTY_PRINT')
        args.append('--formatting=PRINT_INPUT_DELIMITER')
      elif self.rule.mode == 'SIMPLE':
        compiling = True
        args.append('--compilation_level=SIMPLE_OPTIMIZATIONS')
      elif self.rule.mode == 'ADVANCED':
        compiling = True
        args.append('--compilation_level=ADVANCED_OPTIMIZATIONS')

      if self.rule.pretty_print:
        args.append('--formatting=PRETTY_PRINT')
        args.append('--formatting=PRINT_INPUT_DELIMITER')

      if not self.rule.debug:
        args.append('--define=goog.DEBUG=false')
        args.append('--define=goog.asserts.ENABLE_ASSERTS=false')

      if self.rule.wrap_with_global:
        args.append('--output_wrapper="(function(){%%output%%}).call(%s);"' % (
            self.rule.wrap_with_global))

      extern_paths = self._resolve_input_files(self.rule.externs)
      for extern_path in extern_paths:
        args.append('--externs=%s' % (extern_path))

      for entry_point in self.rule.entry_points:
        args.append('--closure_entry_point=%s' % (entry_point))

      # Main js library
      if compiling:
        output_path = self._get_out_path(name=self.rule.out, suffix='.js')
        self._ensure_output_exists(os.path.dirname(output_path))
        self._append_output_paths([output_path])
        args.append('--js_output_file=%s' % (output_path))

      # deps.js file
      deps_name = self.rule.deps_out or self.rule.out
      deps_js_path = self._get_out_path(name=deps_name, suffix='-deps.js')
      self._ensure_output_exists(os.path.dirname(deps_js_path))
      self._append_output_paths([deps_js_path])

      # File manifest
      file_list_path = None
      if self.rule.file_list_out:
        file_list_path = self._get_out_path(name=self.rule.file_list_out)
        self._ensure_output_exists(os.path.dirname(file_list_path))
        self._append_output_paths([file_list_path])

      # Skip if cache hit
      if self._check_if_cached():
        self._succeed()
        return

      # Issue dependency scanning to build the deps graph
      d = self._run_task_async(_ScanJsDependenciesTask(
          self.build_env, self.src_paths))

      # Launch the compilation and deps.js gen tasks when scanning completes
      def _deps_scanned(dep_graph):
        ds = []

        # Grab all used files
        used_paths = dep_graph.get_transitive_closure(self.rule.entry_points)

        # Generate/write JS deps
        ds.append(self._run_task_async(WriteFileTask(
            self.build_env, dep_graph.get_deps_js(), deps_js_path)))

        # Generate/write manifest
        if file_list_path:
          rel_used_paths = [
              os.path.relpath(path, self._get_rule_path())
              for path in used_paths]
          ds.append(self._run_task_async(WriteFileTask(
              self.build_env, u'\n'.join(rel_used_paths), file_list_path)))

        # Compile main lib
        if compiling:
          for src_path in used_paths:
            args.append('--js=%s' % (src_path))
          ds.append(self._run_task_async(JavaExecutableTask(
              self.build_env, jar_path, args,
              pretty_name=str(self.rule))))
          # TODO(benvanik): pull out (stdout, stderr) from result and the
          #     exception to get better error logging
        else:
          # deps-only - pass along all inputs as outputs
          self._append_output_paths(used_paths)

        self._chain(ds)

      # Kickoff chain
      d.add_callback_fn(_deps_scanned)
      self._chain_errback(d)


class _ScanJsDependenciesTask(Task):
  def __init__(self, build_env, src_paths, *args, **kwargs):
    super(_ScanJsDependenciesTask, self).__init__(build_env, *args, **kwargs)
    self.src_paths = src_paths

  def execute(self):
    deps_graph = JsDependencyGraph(self.build_env, self.src_paths)
    return deps_graph


class JsDependencyFile(object):
  """A single source JS file.
  Parses the file and finds all goog.provide and goog.require statements.
  """

  _PROVIDEREQURE_REGEX = re.compile(
      'goog\.(provide|require)\(\s*[\'"](.+)[\'"]\s*\)')
  # TODO(benvanik): a real comment search for @provideGoog.
  _GOOG_BASE_LINE = (
      ' * @provideGoog\n')

  def __init__(self, src_path):
    """Initializes a JS dependency file.

    Args:
      src_path: Source JS file path.
    """
    self.src_path = src_path
    self.provides = []
    self.requires = []
    self.is_base_js = False
    self.is_css_rename_map = False
    with io.open(self.src_path, 'rb') as f:
      self._scan(f)

  def _scan(self, f):
    """Scans the given file for provides/requires and returns the results.

    Args:
      f: Input file.
    """
    provides = set()
    requires = set()

    for line in f.readlines():
      match = self._PROVIDEREQURE_REGEX.match(line)
      if match:
        if match.group(1) == 'provide':
          provides.add(str(match.group(2)))
        else:
          requires.add(str(match.group(2)))
      elif line == self._GOOG_BASE_LINE:
        provides.add('goog')
        self.is_base_js = True
      elif line.startswith('goog.setCssNameMapping('):
        self.is_css_rename_map = True

    self.provides = list(provides)
    self.provides.sort()
    self.requires = list(requires)
    self.requires.sort()


class JsDependencyGraph(object):
  """Represents a JS dependency graph.
  This scans all source JavaScript files to identify their dependencies.
  The result is a queryable list
  """

  def __init__(self, build_env, src_paths, *args, **kwargs):
    """Initializes a JS dependency graph.

    Args:
      build_env: BuildEnvironment.
      src_paths: A list of source JS paths.
    """
    self.build_env = build_env
    self.src_paths = list(src_paths)
    self.dep_files = {}
    self._provide_map = {}
    self.base_dep_file = None
    self.base_js_path = None

    # Scan all files
    for src_path in self.src_paths:
      dep_file = JsDependencyFile(src_path)
      self.dep_files[src_path] = dep_file
      if dep_file.is_base_js:
        self.base_dep_file = dep_file
        self.base_js_path = os.path.dirname(dep_file.src_path)
      for provide in dep_file.provides:
        assert not provide in self._provide_map
        self._provide_map[provide] = dep_file

  def get_deps_js(self):
    """Generates the contents of a deps.js file from the dependency graph.

    Returns:
      A string containing all of the lines of a deps.js file.
    """
    # Path that all dependencies will be relative from
    base_path = self.build_env.root_path
    if self.base_js_path:
      base_path = self.base_js_path

    lines = [
        '// Automatically generated by anvil-build - do not modify',
        '',
        ]

    # Write in path order (to make the file easier to debug)
    src_paths = self.dep_files.keys()
    src_paths.sort()
    for src_path in src_paths:
      dep_file = self.dep_files[src_path]
      rel_path = os.path.relpath(dep_file.src_path, base_path)
      rel_path = anvil.util.strip_build_paths(rel_path)
      lines.append('goog.addDependency(\'%s\', %s, %s);' % (
          anvil.util.ensure_forwardslashes(rel_path),
          dep_file.provides, dep_file.requires))
    return u'\n'.join(lines)

  def get_transitive_closure(self, entry_points):
    """Identifies the transitive closure of the dependency graph for the given
    entry points.

    Args:
      entry_points: Closure entry points, such as 'my.start'.

    Returns:
      A list of all file paths required by the given entry points.
      The files are sorted in proper dependency order.
    """
    deps_list = []

    # Always base.js first
    if self.base_dep_file:
      deps_list.append(self.base_dep_file.src_path)

    # Followed by all files in the transitive closure, in order
    for entry_point in entry_points:
      self._add_dependencies(deps_list, entry_point)

    # And finally, any files that look special
    for dep_file in self.dep_files.values():
      if dep_file.is_css_rename_map:
        deps_list.append(dep_file.src_path)

    return deps_list

  def _add_dependencies(self, deps_list, namespace):
    if not namespace in self._provide_map:
      print('Namespace %s not provided' % (namespace))
    assert namespace in self._provide_map
    dep_file = self._provide_map[namespace]
    if dep_file.src_path in deps_list:
      return
    for require in dep_file.requires:
      if require in dep_file.provides:
        print('Namespace %s both provided and required in the same file' % (
            require))
      assert not require in dep_file.provides
      self._add_dependencies(deps_list, require)
    deps_list.append(dep_file.src_path)