Support for local dependencies for Node.js build command

.gitignore

@@ -161,6 +161,7 @@ typings/
 
 # Output of 'npm pack'
 *.tgz
+!tests/integration/workflows/nodejs_npm/testdata/relative-dependencies/lib-1.0.0.tgz
 
 # Except test file
 !tests/functional/workflows/ruby_bundler/test_data/test.tgz

aws_lambda_builders/workflows/nodejs_npm/actions.py

@@ -195,3 +195,54 @@ def execute(self):
 
         except OSError as ex:
             raise ActionFailedError(str(ex))
+
+class NodejsNpmRewriteLocalDependenciesAction(BaseAction):
+
+    """
+    A Lambda Builder Action that rewrites local dependencies
+    """
+
+    NAME = 'RewriteLocalDependencies'
+    DESCRIPTION = "Rewrites local dependencies"
+    PURPOSE = Purpose.RESOLVE_DEPENDENCIES
+
+    def __init__(
+        self,
+        work_dir,
+        original_package_dir,
+        scratch_dir,
+        npm_modules_utils,
+        osutils
+    ):
+        super(NodejsNpmRewriteLocalDependenciesAction, self).__init__()
+        self.work_dir = work_dir
+        self.original_package_dir = original_package_dir
+        self.scratch_dir = scratch_dir
+        self.npm_modules_utils = npm_modules_utils
+        self.osutils = osutils
+
+    def __rewrite_local_dependencies(self, work_dir, original_package_dir):
+        for dependency_key in ['dependencies', 'optionalDependencies']:
+            for (name, module_path) in self.npm_modules_utils.get_local_dependencies(work_dir, dependency_key).items():
+                if module_path.startswith('file:'):
+                    module_path = module_path[5:]
+
+                actual_path = self.osutils.joinpath(original_package_dir, module_path)
+                if self.osutils.is_dir(actual_path):
+                    if self.npm_modules_utils.has_local_dependencies(actual_path):
+                        module_path = self.npm_modules_utils.clean_copy(actual_path, delete_package_lock=True)
+                        self.__rewrite_local_dependencies(module_path, actual_path)
+                        actual_path = module_path
+
+                    new_module_path = self.npm_modules_utils.pack_to_tar(actual_path)
+                else:
+                    new_module_path = actual_path
+
+                new_module_path = 'file:{}'.format(new_module_path)
+                self.npm_modules_utils.update_dependency(work_dir, name, new_module_path, dependency_key)
+
+    def execute(self):
+        try:
+            self.__rewrite_local_dependencies(self.work_dir, self.original_package_dir)
+        except OSError as ex:
+            raise ActionFailedError(str(ex))

aws_lambda_builders/workflows/nodejs_npm/npm.py

@@ -3,6 +3,7 @@
 """
 
 import logging
+import json
 
 LOG = logging.getLogger(__name__)
 
@@ -88,3 +89,150 @@ def run(self, args, cwd=None):
             raise NpmExecutionError(message=err.decode('utf8').strip())
 
         return out.decode('utf8').strip()
+
+
+class NpmModulesUtils(object):
+
+    """
+    Utility class that abstracts operations on NPM packages
+    and manifest files
+    """
+
+    def __init__(self, osutils, subprocess_npm, scratch_dir):
+        """
+        :type osutils: aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils
+        :param osutils: An instance of OS Utilities for file manipulation
+
+        :type subprocess_npm: aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm
+        :param subprocess_npm: An instance of NPM Subprocess for executing the NPM binary
+
+        :type scratch_dir: str
+        :param scratch_dir: a writable temporary directory
+        """
+
+        self.osutils = osutils
+        self.subprocess_npm = subprocess_npm
+        self.scratch_dir = scratch_dir
+
+    def clean_copy(self, package_dir, delete_package_lock=False):
+
+        """
+        Produces a clean copy of a NPM package source from a project directory,
+        so it can be packaged without temporary files, development or test resources
+        or dependencies.
+
+        :type package_dir: str
+        :param package_dir: Path to a NPM project directory
+
+        :type delete_package_lock: bool
+        :param delete_package_lock: If true, package-lock.json will be removed in the copy
+        """
+
+        target_dir = self.osutils.tempdir(self.scratch_dir)
+
+        package_path = 'file:{}'.format(self.osutils.abspath(package_dir))
+
+        LOG.debug('NODEJS packaging %s to %s', package_path, self.scratch_dir)
+
+        tarfile_name = self.subprocess_npm.run(['pack', '-q', package_path], cwd=self.scratch_dir)
+
+        LOG.debug('NODEJS packed to %s', tarfile_name)
+
+        tarfile_path = self.osutils.joinpath(self.scratch_dir, tarfile_name)
+
+        LOG.debug('NODEJS extracting to %s', target_dir)
+
+        self.osutils.extract_tarfile(tarfile_path, target_dir)
+
+        package_lock = self.osutils.joinpath(target_dir, 'package', 'package-lock.json')
+        if delete_package_lock and self.osutils.file_exists(package_lock):
+            self.osutils.remove_file(package_lock)
+
+        return self.osutils.joinpath(target_dir, 'package')
+
+    def is_local_dependency(self, module_path):
+        """
+        Calculates if the module path from a dependency reference is
+        local or remote
+
+        :type module_path: str
+        :param module_path: Dependency reference value (from package.json)
+
+        """
+
+        return module_path.startswith('file:') or \
+            module_path.startswith('.') or \
+            module_path.startswith('/') or \
+            module_path.startswith('~/')
+
+    def get_local_dependencies(self, package_dir, dependency_key='dependencies'):
+        """
+        Returns a dictionary with only local dependencies from a package.json manifest
+
+        :type package_dir: str
+        :param package_dir: path to a NPM project directory (containing package.json)
+
+        :type dependency_key: str
+        :param dependency_key: dependency type to return, corresponds to a key of package.json.
+            (for example, 'dependencies' or 'optionalDependencies')
+        """
+
+        package_json = json.loads(self.osutils.get_text_contents(self.osutils.joinpath(package_dir, 'package.json')))
+        if dependency_key not in package_json.keys():
+            return {}
+
+        dependencies = package_json[dependency_key]
+
+        return dict(
+            [(name, module_path) for (name, module_path) in dependencies.items()
+                if self.is_local_dependency(module_path)]
+        )
+
+    def has_local_dependencies(self, package_dir):
+        """
+        Checks if a NPM project has local dependencies
+
+        :type package_dir: str
+        :param package_dir: path to a NPM project directory (containing package.json)
+        """
+        return len(self.get_local_dependencies(package_dir, 'dependencies')) > 0 or \
+            len(self.get_local_dependencies(package_dir, 'optionalDependencies')) > 0
+
+    def pack_to_tar(self, package_dir):
+        """
+        Runs npm pack to produce a tar containing project sources, which can be used
+        as a target for project dependencies
+
+        :type package_dir: str
+        :param package_dir: path to a NPM project directory (containing package.json)
+        """
+        package_path = "file:{}".format(self.osutils.abspath(package_dir))
+
+        tarfile_name = self.subprocess_npm.run(['pack', '-q', package_path], cwd=self.scratch_dir)
+
+        return self.osutils.joinpath(self.scratch_dir, tarfile_name)
+
+    def update_dependency(self, package_dir, name, module_path, dependency_key='dependencies'):
+        """
+        Updates package.json by rewriting a dependency to point to a specified module path
+
+        :type package_dir: str
+        :param package_dir: path to a NPM project directory (containing package.json)
+
+        :type name: str
+        :param name: the name of the dependency (sub-key in package.json)
+
+        :type module_path: str
+        :param module_path: new destination for the dependency
+
+        :type dependency_key: str
+        :param dependency_key: dependency type to return, corresponds to a key of package.json.
+            (for example, 'dependencies' or 'optionalDependencies')
+        """
+
+        package_json_path = self.osutils.joinpath(package_dir, 'package.json')
+        package_json = json.loads(self.osutils.get_text_contents(package_json_path))
+
+        package_json[dependency_key][name] = module_path
+        package_json_contents = json.dumps(package_json, ensure_ascii=False)
+        self.osutils.write_text_contents(package_json_path, package_json_contents)

aws_lambda_builders/workflows/nodejs_npm/utils.py

@@ -7,7 +7,8 @@
 import tarfile
 import subprocess
 import shutil
-
+import io
+import tempfile
 
 class OSUtils(object):
 
@@ -48,3 +49,17 @@ def abspath(self, path):
 
     def is_windows(self):
         return platform.system().lower() == 'windows'
+
+    def get_text_contents(self, filename, encoding='utf-8'):
+        with io.open(filename, 'r', encoding=encoding) as f:
+            return f.read()
+
+    def write_text_contents(self, filename, contents, encoding='utf-8'):
+        with io.open(filename, 'w', encoding=encoding) as f:
+            f.write(contents)
+
+    def tempdir(self, parent_dir):
+        return tempfile.mkdtemp(dir=parent_dir)
+
+    def is_dir(self, path):
+        return os.path.isdir(path)

aws_lambda_builders/workflows/nodejs_npm/workflow.py

@@ -4,9 +4,10 @@
 from aws_lambda_builders.path_resolver import PathResolver
 from aws_lambda_builders.workflow import BaseWorkflow, Capability
 from aws_lambda_builders.actions import CopySourceAction
-from .actions import NodejsNpmPackAction, NodejsNpmInstallAction, NodejsNpmrcCopyAction, NodejsNpmrcCleanUpAction
+from .actions import NodejsNpmPackAction, NodejsNpmInstallAction, NodejsNpmrcCopyAction, \
+    NodejsNpmrcCleanUpAction, NodejsNpmRewriteLocalDependenciesAction
 from .utils import OSUtils
-from .npm import SubprocessNpm
+from .npm import SubprocessNpm, NpmModulesUtils
 
 
 class NodejsNpmWorkflow(BaseWorkflow):
@@ -62,6 +63,13 @@ def __init__(self,
             npm_pack,
             npm_copy_npmrc,
             CopySourceAction(tar_package_dir, artifacts_dir, excludes=self.EXCLUDED_FILES),
+            NodejsNpmRewriteLocalDependenciesAction(
+                artifacts_dir,
+                source_dir,
+                tar_dest_dir,
+                npm_modules_utils=NpmModulesUtils(osutils, subprocess_npm, scratch_dir),
+                osutils=osutils
+            ),
             npm_install,
             NodejsNpmrcCleanUpAction(artifacts_dir, osutils=osutils)
         ]

tests/functional/workflows/nodejs_npm/test_utils.py

@@ -2,6 +2,7 @@
 import shutil
 import sys
 import tempfile
+import io
 
 from unittest import TestCase
 
@@ -65,7 +66,7 @@ def test_file_exists_checking_if_file_exists_in_a_dir(self):
         self.assertTrue(self.osutils.file_exists(existing_file))
 
         self.assertFalse(self.osutils.file_exists(nonexisting_file))
-   
+
     def test_extract_tarfile_unpacks_a_tar(self):
 
         test_tar = os.path.join(os.path.dirname(__file__), "test_data", "test.tgz")
@@ -127,3 +128,36 @@ def test_popen_can_accept_cwd(self):
         self.assertEqual(p.returncode, 0)
 
         self.assertEqual(out.decode('utf8').strip(), os.path.abspath(testdata_dir))
+
+    def test_reads_file_content(self):
+        scratch_dir = tempfile.mkdtemp()
+        filename = os.path.join(scratch_dir, 'test_write.txt')
+
+        with io.open(filename, 'w', encoding='utf-8') as f:
+            f.write(u'hello')
+
+        content = self.osutils.get_text_contents(filename)
+        self.assertEqual(content, 'hello')
+
+    def test_writes_text_context(self):
+        scratch_dir = tempfile.mkdtemp()
+        filename = os.path.join(scratch_dir, 'test_write.txt')
+        self.osutils.write_text_contents(filename, u'hello')
+        with io.open(filename, 'r', encoding='utf-8') as f:
+            content = f.read()
+            self.assertEqual(content, u'hello')
+
+    def test_should_create_temporary_dir(self):
+        scratch_dir = tempfile.mkdtemp()
+        temp_dir = self.osutils.tempdir(scratch_dir)
+        os.path.isdir(temp_dir)
+
+    def test_is_dir_returns_true_if_dir_exists(self):
+        dir_exists = self.osutils.is_dir(os.path.dirname(__file__))
+
+        self.assertTrue(dir_exists)
+
+    def test_is_dir_returns_false_if_dir_does_not_exist(self):
+        dir_does_not_exist = self.osutils.is_dir(os.path.join(os.path.dirname(__file__), 'some_random_dirname'))
+
+        self.assertFalse(dir_does_not_exist)