Merge branch 'master' of github.com:nficano/python-lambda

nficano · nficano · commit f9546021d708 · 2017-09-07T10:10:05.000-04:00
* 'master' of github.com:nficano/python-lambda:
  updating README and changing md5 to hashlib
  removing print statement
  adding various changes based on review feedback
  Update README.rst
  include subdirectories when building for deploy
  making reference to bucket_name consistent
  forgot to add refs to upload func
  adding function to upload to s3
  Push environment variables from the config file into the actual environment when calling invoke
diff --git a/README.rst b/README.rst
@@ -165,11 +165,22 @@ value for the environment variable at the time of deployment (instead of hard co
 
 This would create environment variables in the lambda instance upon deploy. If your functions don't need environment variables, simply leave this section out of your config.
 
+Uploading to S3
+===============
+You may find that you do not need the toolkit to fully deploy your Lambda or that your code bundle is too large to upload via the API.  You can use the `upload` command to send the bundle to an S3 bucket of your choosing.
+Before doing this, you will need to set the following variables in `config.yaml`:
+```
+role: basic_s3_upload
+bucket_name: 'example-bucket'
+s3_key_prefix: 'path/to/file/'
+```
+Your role must have `s3:PutObject` permission on the bucket/key that you specify for the upload to work properly. Once you have that set, you can execute `lambda upload` to initiate the transfer.
 
 Development
 ===========
 
-Development of this happens on GitHub, patches including tests, documentation are very welcome, as well as bug reports and feature contributions are welcome! Also please open an issue if this tool does not function as you'd expect.
+Development of "python-lambda" is facilitated exclusively on GitHub. Contributions in the form of patches, tests and feature creation and/or requests are very welcome and highly encouraged. Please open an issue if this tool does not function as you'd expect.
+
 
 How to release updates
 ----------------------
@@ -180,10 +191,16 @@ Once complete, execute the following commands:
 
 .. code:: bash
 
-   $ git checkout master
-   $ bumpversion [major|minor|patch]
-   $
-   $ python setup.py sdist bdist_wheel upload
-   $
-   $ bumpversion --no-tag patch
-   $ git push origin master --tags
+    git checkout master
+
+    # Increment the version number and tag the release.
+    bumpversion [major|minor|patch]
+
+    # Upload the distribution to PyPi
+    python setup.py sdist bdist_wheel upload
+
+    # Since master often contains work-in-progress changes, increment the version
+    # to a patch release to prevent inaccurate attribution.
+    bumpversion --no-tag patch
+
+    git push origin master --tags
diff --git a/aws_lambda/__init__.py b/aws_lambda/__init__.py
@@ -4,7 +4,7 @@
 __email__ = 'nficano@gmail.com'
 __version__ = '1.0.1'
 
-from .aws_lambda import deploy, invoke, init, build, cleanup_old_versions
+from .aws_lambda import deploy, invoke, init, build, upload, cleanup_old_versions
 
 # Set default logging handler to avoid "No handler found" warnings.
 import logging
diff --git a/aws_lambda/aws_lambda.py b/aws_lambda/aws_lambda.py
@@ -15,6 +15,7 @@
 import botocore
 import pip
 import yaml
+import hashlib
 
 from .helpers import archive
 from .helpers import mkdir
@@ -94,6 +95,27 @@ def deploy(src, requirements=False, local_package=None):
     else:
         create_function(cfg, path_to_zip_file)
 
+def upload(src, requirements=False, local_package=None):
+    """Uploads a new function to AWS S3.
+
+    :param str src:
+        The path to your Lambda ready project (folder must contain a valid
+        config.yaml and handler module (e.g.: service.py).
+    :param str local_package:
+        The path to a local package with should be included in the deploy as
+        well (and/or is not available on PyPi)
+    """
+    # Load and parse the config file.
+    path_to_config_file = os.path.join(src, 'config.yaml')
+    cfg = read(path_to_config_file, loader=yaml.load)
+
+    # Copy all the pip dependencies required to run your code into a temporary
+    # folder then add the handler file in the root of this directory.
+    # Zip the contents of this folder into a single file and output to the dist
+    # directory.
+    path_to_zip_file = build(src, requirements, local_package)
+
+    upload_s3(cfg, path_to_zip_file)
 
 def invoke(src, alt_event=None, verbose=False):
     """Simulates a call to your function.
@@ -110,6 +132,10 @@ def invoke(src, alt_event=None, verbose=False):
     path_to_config_file = os.path.join(src, 'config.yaml')
     cfg = read(path_to_config_file, loader=yaml.load)
 
+    # Load environment variables from the config file into the actual environment.
+    for key, value in cfg.get('environment_variables').items():
+        os.environ[key] = value
+
     # Load and parse event file.
     if alt_event:
         path_to_event_file = os.path.join(src, alt_event)
@@ -211,8 +237,9 @@ def build(src, requirements=False, local_package=None):
                 continue
             if filename == 'config.yaml':
                 continue
-            print('Bundling: %r' % filename)
-            files.append(os.path.join(src, filename))
+        # TODO: Check subdirectories for '.DS_Store' files
+        print('Bundling: %r' % filename)
+        files.append(os.path.join(src, filename))
 
     # "cd" into `temp_path` directory.
     os.chdir(path_to_temp)
@@ -432,6 +459,39 @@ def update_function(cfg, path_to_zip_file):
 
     client.update_function_configuration(**kwargs)
 
+def upload_s3(cfg, path_to_zip_file):
+    """Upload a function to AWS S3."""
+
+    print('Uploading your new Lambda function')
+    aws_access_key_id = cfg.get('aws_access_key_id')
+    aws_secret_access_key = cfg.get('aws_secret_access_key')
+    account_id = get_account_id(aws_access_key_id, aws_secret_access_key)
+    client = get_client('s3', aws_access_key_id, aws_secret_access_key,
+                        cfg.get('region'))
+    role = get_role_name(account_id, cfg.get('role', 'basic_s3_upload'))
+    byte_stream = b''
+    with open(path_to_zip_file, mode='rb') as fh:
+        byte_stream = fh.read()
+    s3_key_prefix = cfg.get('s3_key_prefix', '/dist')
+    checksum = hashlib.new('md5', byte_stream).hexdigest()
+    timestamp = str(time.time())
+    filename = '{prefix}{checksum}-{ts}.zip'.format(prefix=s3_key_prefix, checksum=checksum, ts=timestamp)
+
+    # Do we prefer development variable over config?
+    buck_name = (
+        os.environ.get('S3_BUCKET_NAME') or cfg.get('bucket_name')
+    )
+    func_name = (
+        os.environ.get('LAMBDA_FUNCTION_NAME') or cfg.get('function_name')
+    )
+    kwargs = {
+        'Bucket': '{}'.format(buck_name),
+        'Key': '{}'.format(filename),
+        'Body': byte_stream
+    }
+
+    client.put_object(**kwargs)
+    print('Finished uploading {} to S3 bucket {}'.format(func_name, buck_name))
 
 def function_exists(cfg, function_name):
     """Check whether a function exists or not"""
diff --git a/aws_lambda/project_templates/config.yaml b/aws_lambda/project_templates/config.yaml
@@ -2,9 +2,14 @@ region: us-east-1
 
 function_name: my_lambda_function
 handler: service.handler
-# role: lambda_basic_execution
 description: My first lambda function
 runtime: python2.7
+# role: lambda_basic_execution
+
+# S3 upload requires appropriate role with s3:PutObject permission
+# (ex. basic_s3_upload), a destination bucket, and the key prefix
+# bucket_name: 'example-bucket'
+# s3_key_prefix: 'path/to/file/'
 
 # if access key and secret are left blank, boto will use the credentials
 # defined in the [default] section of ~/.aws/credentials.
diff --git a/scripts/lambda b/scripts/lambda
@@ -46,7 +46,11 @@ def invoke(event_file, verbose):
 def deploy(use_requirements, local_package):
     aws_lambda.deploy(CURRENT_DIR, use_requirements, local_package)
 
-
+@click.command(help="Upload your lambda to S3.")
+@click.option('--use-requirements', default=False, is_flag=True, help='Install all packages defined in requirements.txt')
+@click.option('--local-package', default=None, help='Install local package as well.', type=click.Path(), multiple=True)
+def upload(use_requirements, local_package):
+    aws_lambda.upload(CURRENT_DIR, use_requirements, local_package)
 
 @click.command(help="Delete old versions of your functions")
 @click.option("--keep-last", type=int, prompt="Please enter the number of recent versions to keep")
@@ -57,6 +61,7 @@ if __name__ == '__main__':
     cli.add_command(init)
     cli.add_command(invoke)
     cli.add_command(deploy)
+    cli.add_command(upload)
     cli.add_command(build)
     cli.add_command(cleanup)
     cli()
