diff --git a/Makefile b/Makefile index 4dd514337..f1c2aa999 100644 --- a/Makefile +++ b/Makefile @@ -73,4 +73,7 @@ test-async: test-general: nosetests tests/tests.py --with-coverage --cover-package=zappa --with-timer -tests: clean test-docs test-handler test-middleware test-placebo test-async test-general +test-utilities: + nosetests tests/tests_utilities.py --with-coverage --cover-package=zappa --with-timer + +tests: clean test-docs test-handler test-middleware test-placebo test-async test-general test-utilities diff --git a/Pipfile b/Pipfile index 28839f094..a5c15c4d7 100644 --- a/Pipfile +++ b/Pipfile @@ -40,8 +40,6 @@ tqdm = "*" troposphere = ">=3.0" Werkzeug = "*" wheel = "*" -wsgi-request-logger = "*" [pipenv] -# Required for 'black' since all of its release tags contain 'b' -allow_prereleases = true +allow_prereleases = false diff --git a/tests/tests.py b/tests/tests.py index 589211684..e66e8ea9e 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -26,9 +26,7 @@ from zappa.cli import ZappaCLI, disable_click_colors, shamelessly_promote from zappa.core import ALB_LAMBDA_ALIAS, ASSUME_POLICY, ATTACH_POLICY, Zappa -from zappa.ext.django_zappa import get_django_wsgi from zappa.letsencrypt import ( - cleanup, create_chained_certificate, create_domain_csr, create_domain_key, @@ -39,21 +37,6 @@ parse_csr, register_account, sign_certificate, - verify_challenge, -) -from zappa.utilities import ( - InvalidAwsLambdaName, - conflicts_with_a_neighbouring_module, - contains_python_files_or_subdirs, - detect_django_settings, - detect_flask_apps, - get_venv_from_python_version, - human_size, - is_valid_bucket_name, - parse_s3_url, - string_to_timestamp, - titlecase_keys, - validate_name, ) from zappa.wsgi import common_log, create_wsgi_request @@ -101,61 +84,6 @@ def test_disable_click_colors(self): disable_click_colors() assert resolve_color_default() is False - @mock.patch("zappa.core.find_packages") - @mock.patch("os.remove") - def test_copy_editable_packages(self, mock_remove, mock_find_packages): - virtual_env = os.environ.get("VIRTUAL_ENV") - if not virtual_env: - return self.skipTest("test_copy_editable_packages must be run in a virtualenv") - - temp_package_dir = tempfile.mkdtemp() - try: - egg_links = [ - os.path.join( - virtual_env, - "lib", - get_venv_from_python_version(), - "site-packages", - "test-copy-editable-packages.egg-link", - ) - ] - egg_path = "/some/other/directory/package" - mock_find_packages.return_value = [ - "package", - "package.subpackage", - "package.another", - ] - temp_egg_link = os.path.join(temp_package_dir, "package-python.egg-link") - - z = Zappa() - mock_open = mock.mock_open(read_data=egg_path.encode("utf-8")) - with mock.patch("zappa.core.open", mock_open), mock.patch("glob.glob") as mock_glob, mock.patch( - "zappa.core.copytree" - ) as mock_copytree: - # we use glob.glob to get the egg-links in the temp packages - # directory - mock_glob.return_value = [temp_egg_link] - - z.copy_editable_packages(egg_links, temp_package_dir) - - # make sure we copied the right directories - mock_copytree.assert_called_with( - os.path.join(egg_path, "package"), - os.path.join(temp_package_dir, "package"), - metadata=False, - symlinks=False, - ) - self.assertEqual(mock_copytree.call_count, 1) - - # make sure it removes the egg-link from the temp packages - # directory - mock_remove.assert_called_with(temp_egg_link) - self.assertEqual(mock_remove.call_count, 1) - finally: - shutil.rmtree(temp_package_dir) - - return - def test_create_lambda_package(self): # mock the pkg_resources.WorkingSet() to include a known package in lambda_packages so that the code # for zipping pre-compiled packages gets called @@ -2028,119 +1956,6 @@ def test_get_all_zones_two_pages(self, client): ) self.assertListEqual(zones["HostedZones"], [{"Id": "zone1"}, {"Id": "zone2"}]) - ## - # Django - ## - - def test_detect_dj(self): - # Sanity - settings_modules = detect_django_settings() - - def test_dj_wsgi(self): - # Sanity - settings_modules = detect_django_settings() - - settings = """ -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -import os -BASE_DIR = os.path.dirname(os.path.dirname(__file__)) - -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/ - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'alskdfjalsdkf=0*%do-ayvy*m2k=vss*$7)j8q!@u0+d^na7mi2(^!l!d' - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - -TEMPLATE_DEBUG = True - -ALLOWED_HOSTS = [] - -# Application definition - -INSTALLED_APPS = ( - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', -) - -MIDDLEWARE_CLASSES = ( - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', -) - -ROOT_URLCONF = 'blah.urls' -WSGI_APPLICATION = 'hackathon_starter.wsgi.application' - -# Database -# https://docs.djangoproject.com/en/1.7/ref/settings/#databases - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), - } -} - -# Internationalization -# https://docs.djangoproject.com/en/1.7/topics/i18n/ - -LANGUAGE_CODE = 'en-us' -TIME_ZONE = 'UTC' -USE_I18N = True -USE_L10N = True -USE_TZ = True - """ - - djts = open("dj_test_settings.py", "w") - djts.write(settings) - djts.close() - - app = get_django_wsgi("dj_test_settings") - try: - os.remove("dj_test_settings.py") - os.remove("dj_test_settings.pyc") - except Exception as e: - pass - - ## - # Util / Misc - ## - - def test_human_units(self): - human_size(1) - human_size(9999999999999) - - def test_string_to_timestamp(self): - boo = string_to_timestamp("asdf") - self.assertTrue(boo == 0) - - yay = string_to_timestamp("1h") - self.assertTrue(type(yay) == int) - self.assertTrue(yay > 0) - - yay = string_to_timestamp("4m") - self.assertTrue(type(yay) == int) - self.assertTrue(yay > 0) - - yay = string_to_timestamp("1mm") - self.assertTrue(type(yay) == int) - self.assertTrue(yay > 0) - - yay = string_to_timestamp("1mm1w1d1h1m1s1ms1us") - self.assertTrue(type(yay) == int) - self.assertTrue(yay > 0) - def test_event_name(self): zappa = Zappa() truncated = zappa.get_event_name( @@ -2234,42 +2049,9 @@ def test_get_scheduled_event_name__truncated__has_name__has_index(self): f"{hashed_lambda_name}-{index}-{event['name']}-{function}", ) - def test_detect_dj(self): - # Sanity - settings_modules = detect_django_settings() - - def test_detect_flask(self): - # Sanity - settings_modules = detect_flask_apps() - def test_shameless(self): shamelessly_promote() - def test_s3_url_parser(self): - remote_bucket, remote_file = parse_s3_url("s3://my-project-config-files/filename.json") - self.assertEqual(remote_bucket, "my-project-config-files") - self.assertEqual(remote_file, "filename.json") - - remote_bucket, remote_file = parse_s3_url("s3://your-bucket/account.key") - self.assertEqual(remote_bucket, "your-bucket") - self.assertEqual(remote_file, "account.key") - - remote_bucket, remote_file = parse_s3_url("s3://my-config-bucket/super-secret-config.json") - self.assertEqual(remote_bucket, "my-config-bucket") - self.assertEqual(remote_file, "super-secret-config.json") - - remote_bucket, remote_file = parse_s3_url("s3://your-secure-bucket/account.key") - self.assertEqual(remote_bucket, "your-secure-bucket") - self.assertEqual(remote_file, "account.key") - - remote_bucket, remote_file = parse_s3_url("s3://your-bucket/subfolder/account.key") - self.assertEqual(remote_bucket, "your-bucket") - self.assertEqual(remote_file, "subfolder/account.key") - - # Sad path - remote_bucket, remote_file = parse_s3_url("/dev/null") - self.assertEqual(remote_bucket, "") - def test_remote_env_package(self): zappa_cli = ZappaCLI() zappa_cli.api_stage = "deprecated_remote_env" @@ -2395,28 +2177,6 @@ def test_slim_handler(self): zappa_cli.remove_local_zip() - def test_validate_name(self): - fname = "tests/name_scenarios.json" - with open(fname, "r") as f: - scenarios = json.load(f) - for scenario in scenarios: - value = scenario["value"] - is_valid = scenario["is_valid"] - if is_valid: - assert validate_name(value) - else: - with self.assertRaises(InvalidAwsLambdaName) as exc: - validate_name(value) - - def test_contains_python_files_or_subdirs(self): - self.assertTrue(contains_python_files_or_subdirs("tests/data")) - self.assertTrue(contains_python_files_or_subdirs("tests/data/test2")) - self.assertFalse(contains_python_files_or_subdirs("tests/data/test1")) - - def test_conflicts_with_a_neighbouring_module(self): - self.assertTrue(conflicts_with_a_neighbouring_module("tests/data/test1")) - self.assertFalse(conflicts_with_a_neighbouring_module("tests/data/test2")) - def test_settings_py_generation(self): zappa_cli = ZappaCLI() zappa_cli.api_stage = "ttt888" @@ -2449,55 +2209,6 @@ def test_only_ascii_env_var_allowed(self): zappa_cli.create_package() self.assertEqual("Environment variable keys must be ascii.", str(context.exception)) - def test_titlecase_keys(self): - raw = { - "hOSt": "github.com", - "ConnECtiOn": "keep-alive", - "UpGRAde-InSecuRE-ReQueSts": "1", - "uSer-AGEnT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36", - "cONtENt-TYPe": "text/html; charset=utf-8", - "aCCEpT": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", - "ACcePT-encoDInG": "gzip, deflate, br", - "AcCEpT-lAnGUagE": "en-US,en;q=0.9", - } - transformed = titlecase_keys(raw) - expected = { - "Host": "github.com", - "Connection": "keep-alive", - "Upgrade-Insecure-Requests": "1", - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36", - "Content-Type": "text/html; charset=utf-8", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", - "Accept-Encoding": "gzip, deflate, br", - "Accept-Language": "en-US,en;q=0.9", - } - self.assertEqual(expected, transformed) - - def test_is_valid_bucket_name(self): - # Bucket names must be at least 3 and no more than 63 characters long. - self.assertFalse(is_valid_bucket_name("ab")) - self.assertFalse(is_valid_bucket_name("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefhijlmn")) - # Bucket names must not contain uppercase characters or underscores. - self.assertFalse(is_valid_bucket_name("aaaBaaa")) - self.assertFalse(is_valid_bucket_name("aaa_aaa")) - # Bucket names must start with a lowercase letter or number. - self.assertFalse(is_valid_bucket_name(".abbbaba")) - self.assertFalse(is_valid_bucket_name("abbaba.")) - self.assertFalse(is_valid_bucket_name("-abbaba")) - self.assertFalse(is_valid_bucket_name("ababab-")) - # Bucket names must be a series of one or more labels. Adjacent labels are separated by a single period (.). - # Each label must start and end with a lowercase letter or a number. - self.assertFalse(is_valid_bucket_name("aaa..bbbb")) - self.assertFalse(is_valid_bucket_name("aaa.-bbb.ccc")) - self.assertFalse(is_valid_bucket_name("aaa-.bbb.ccc")) - # Bucket names must not be formatted as an IP address (for example, 192.168.5.4). - self.assertFalse(is_valid_bucket_name("192.168.5.4")) - self.assertFalse(is_valid_bucket_name("127.0.0.1")) - self.assertFalse(is_valid_bucket_name("255.255.255.255")) - - self.assertTrue(is_valid_bucket_name("valid-formed-s3-bucket-name")) - self.assertTrue(is_valid_bucket_name("worst.bucket.ever")) - # TODO: encountered error when vpc_config["SubnetIds"] or vpc_config["SecurityGroupIds"] is missing # We need to make the code more robust in this case and avoid the KeyError def test_zappa_core_deploy_lambda_alb_missing_cert_arn(self): diff --git a/tests/tests_utilities.py b/tests/tests_utilities.py new file mode 100644 index 000000000..4004ad0a6 --- /dev/null +++ b/tests/tests_utilities.py @@ -0,0 +1,431 @@ +import json +import os +import re +import shutil +import tempfile +import unittest +from typing import Tuple +from unittest import mock + +from zappa.core import Zappa +from zappa.ext.django_zappa import get_django_wsgi +from zappa.utilities import ( + ApacheNCSAFormatter, + InvalidAwsLambdaName, + conflicts_with_a_neighbouring_module, + contains_python_files_or_subdirs, + detect_django_settings, + detect_flask_apps, + get_venv_from_python_version, + human_size, + is_valid_bucket_name, + parse_s3_url, + string_to_timestamp, + titlecase_keys, + validate_name, +) + + +class GeneralUtilitiesTestCase(unittest.TestCase): + def setUp(self): + self.sleep_patch = mock.patch("time.sleep", return_value=None) + # Tests expect us-east-1. + # If the user has set a different region in env variables, we set it aside for now and use us-east-1 + self.users_current_region_name = os.environ.get("AWS_DEFAULT_REGION", None) + os.environ["AWS_DEFAULT_REGION"] = "us-east-1" + if not os.environ.get("PLACEBO_MODE") == "record": + self.sleep_patch.start() + + def tearDown(self): + if not os.environ.get("PLACEBO_MODE") == "record": + self.sleep_patch.stop() + del os.environ["AWS_DEFAULT_REGION"] + if self.users_current_region_name is not None: + # Give the user their AWS region back, we're done testing with us-east-1. + os.environ["AWS_DEFAULT_REGION"] = self.users_current_region_name + + @mock.patch("zappa.core.find_packages") + @mock.patch("os.remove") + def test_copy_editable_packages(self, mock_remove, mock_find_packages): + virtual_env = os.environ.get("VIRTUAL_ENV") + if not virtual_env: + return self.skipTest("test_copy_editable_packages must be run in a virtualenv") + + temp_package_dir = tempfile.mkdtemp() + try: + egg_links = [ + os.path.join( + virtual_env, + "lib", + get_venv_from_python_version(), + "site-packages", + "test-copy-editable-packages.egg-link", + ) + ] + egg_path = "/some/other/directory/package" + mock_find_packages.return_value = [ + "package", + "package.subpackage", + "package.another", + ] + temp_egg_link = os.path.join(temp_package_dir, "package-python.egg-link") + + z = Zappa() + mock_open = mock.mock_open(read_data=egg_path.encode("utf-8")) + with mock.patch("zappa.core.open", mock_open), mock.patch("glob.glob") as mock_glob, mock.patch( + "zappa.core.copytree" + ) as mock_copytree: + # we use glob.glob to get the egg-links in the temp packages + # directory + mock_glob.return_value = [temp_egg_link] + + z.copy_editable_packages(egg_links, temp_package_dir) + + # make sure we copied the right directories + mock_copytree.assert_called_with( + os.path.join(egg_path, "package"), + os.path.join(temp_package_dir, "package"), + metadata=False, + symlinks=False, + ) + self.assertEqual(mock_copytree.call_count, 1) + + # make sure it removes the egg-link from the temp packages + # directory + mock_remove.assert_called_with(temp_egg_link) + self.assertEqual(mock_remove.call_count, 1) + finally: + shutil.rmtree(temp_package_dir) + + return + + def test_detect_dj(self): + # Sanity + settings_modules = detect_django_settings() + + def test_dj_wsgi(self): + # Sanity + settings_modules = detect_django_settings() + + settings = """ +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +import os +BASE_DIR = os.path.dirname(os.path.dirname(__file__)) + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'alskdfjalsdkf=0*%do-ayvy*m2k=vss*$7)j8q!@u0+d^na7mi2(^!l!d' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +TEMPLATE_DEBUG = True + +ALLOWED_HOSTS = [] + +# Application definition + +INSTALLED_APPS = ( + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', +) + +MIDDLEWARE_CLASSES = ( + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +) + +ROOT_URLCONF = 'blah.urls' +WSGI_APPLICATION = 'hackathon_starter.wsgi.application' + +# Database +# https://docs.djangoproject.com/en/1.7/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + } +} + +# Internationalization +# https://docs.djangoproject.com/en/1.7/topics/i18n/ + +LANGUAGE_CODE = 'en-us' +TIME_ZONE = 'UTC' +USE_I18N = True +USE_L10N = True +USE_TZ = True + """ + + djts = open("dj_test_settings.py", "w") + djts.write(settings) + djts.close() + + app = get_django_wsgi("dj_test_settings") + try: + os.remove("dj_test_settings.py") + os.remove("dj_test_settings.pyc") + except Exception as e: + pass + + ## + # Util / Misc + ## + + def test_human_units(self): + human_size(1) + human_size(9999999999999) + + def test_string_to_timestamp(self): + boo = string_to_timestamp("asdf") + self.assertTrue(boo == 0) + + yay = string_to_timestamp("1h") + self.assertTrue(type(yay) == int) + self.assertTrue(yay > 0) + + yay = string_to_timestamp("4m") + self.assertTrue(type(yay) == int) + self.assertTrue(yay > 0) + + yay = string_to_timestamp("1mm") + self.assertTrue(type(yay) == int) + self.assertTrue(yay > 0) + + yay = string_to_timestamp("1mm1w1d1h1m1s1ms1us") + self.assertTrue(type(yay) == int) + self.assertTrue(yay > 0) + + def test_detect_dj(self): + # Sanity + settings_modules = detect_django_settings() + + def test_detect_flask(self): + # Sanity + settings_modules = detect_flask_apps() + + def test_s3_url_parser(self): + remote_bucket, remote_file = parse_s3_url("s3://my-project-config-files/filename.json") + self.assertEqual(remote_bucket, "my-project-config-files") + self.assertEqual(remote_file, "filename.json") + + remote_bucket, remote_file = parse_s3_url("s3://your-bucket/account.key") + self.assertEqual(remote_bucket, "your-bucket") + self.assertEqual(remote_file, "account.key") + + remote_bucket, remote_file = parse_s3_url("s3://my-config-bucket/super-secret-config.json") + self.assertEqual(remote_bucket, "my-config-bucket") + self.assertEqual(remote_file, "super-secret-config.json") + + remote_bucket, remote_file = parse_s3_url("s3://your-secure-bucket/account.key") + self.assertEqual(remote_bucket, "your-secure-bucket") + self.assertEqual(remote_file, "account.key") + + remote_bucket, remote_file = parse_s3_url("s3://your-bucket/subfolder/account.key") + self.assertEqual(remote_bucket, "your-bucket") + self.assertEqual(remote_file, "subfolder/account.key") + + # Sad path + remote_bucket, remote_file = parse_s3_url("/dev/null") + self.assertEqual(remote_bucket, "") + + def test_validate_name(self): + fname = "tests/name_scenarios.json" + with open(fname, "r") as f: + scenarios = json.load(f) + for scenario in scenarios: + value = scenario["value"] + is_valid = scenario["is_valid"] + if is_valid: + assert validate_name(value) + else: + with self.assertRaises(InvalidAwsLambdaName) as exc: + validate_name(value) + + def test_contains_python_files_or_subdirs(self): + self.assertTrue(contains_python_files_or_subdirs("tests/data")) + self.assertTrue(contains_python_files_or_subdirs("tests/data/test2")) + self.assertFalse(contains_python_files_or_subdirs("tests/data/test1")) + + def test_conflicts_with_a_neighbouring_module(self): + self.assertTrue(conflicts_with_a_neighbouring_module("tests/data/test1")) + self.assertFalse(conflicts_with_a_neighbouring_module("tests/data/test2")) + + def test_titlecase_keys(self): + raw = { + "hOSt": "github.com", + "ConnECtiOn": "keep-alive", + "UpGRAde-InSecuRE-ReQueSts": "1", + "uSer-AGEnT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36", + "cONtENt-TYPe": "text/html; charset=utf-8", + "aCCEpT": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", + "ACcePT-encoDInG": "gzip, deflate, br", + "AcCEpT-lAnGUagE": "en-US,en;q=0.9", + } + transformed = titlecase_keys(raw) + expected = { + "Host": "github.com", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36", + "Content-Type": "text/html; charset=utf-8", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", + "Accept-Encoding": "gzip, deflate, br", + "Accept-Language": "en-US,en;q=0.9", + } + self.assertEqual(expected, transformed) + + def test_is_valid_bucket_name(self): + # Bucket names must be at least 3 and no more than 63 characters long. + self.assertFalse(is_valid_bucket_name("ab")) + self.assertFalse(is_valid_bucket_name("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefhijlmn")) + # Bucket names must not contain uppercase characters or underscores. + self.assertFalse(is_valid_bucket_name("aaaBaaa")) + self.assertFalse(is_valid_bucket_name("aaa_aaa")) + # Bucket names must start with a lowercase letter or number. + self.assertFalse(is_valid_bucket_name(".abbbaba")) + self.assertFalse(is_valid_bucket_name("abbaba.")) + self.assertFalse(is_valid_bucket_name("-abbaba")) + self.assertFalse(is_valid_bucket_name("ababab-")) + # Bucket names must be a series of one or more labels. Adjacent labels are separated by a single period (.). + # Each label must start and end with a lowercase letter or a number. + self.assertFalse(is_valid_bucket_name("aaa..bbbb")) + self.assertFalse(is_valid_bucket_name("aaa.-bbb.ccc")) + self.assertFalse(is_valid_bucket_name("aaa-.bbb.ccc")) + # Bucket names must not be formatted as an IP address (for example, 192.168.5.4). + self.assertFalse(is_valid_bucket_name("192.168.5.4")) + self.assertFalse(is_valid_bucket_name("127.0.0.1")) + self.assertFalse(is_valid_bucket_name("255.255.255.255")) + + self.assertTrue(is_valid_bucket_name("valid-formed-s3-bucket-name")) + self.assertTrue(is_valid_bucket_name("worst.bucket.ever")) + + +class ApacheNCSAFormatterTestCase(unittest.TestCase): + def setUp(self): + self.method = "GET" + self.datetime_regex = re.compile( + r"\d+\/(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\/\d{4}:\d{2}:\d{2}:\d{2}\s\+\d{4}" + ) + self.agent = "myagent" + + def _build_expected_format_string(self, status_code, addtional_environ, content_length, **kwargs) -> Tuple[dict, str]: + referer = "me" + logname = "-" + user = "-" + host = "127.0.0.1" + server_protocol = "myprot" + environ = { + "REMOTE_ADDR": host, + "HTTP_USER_AGENT": self.agent, + "HTTP_REFERER": referer, + "SERVER_PROTOCOL": server_protocol, + "PATH_INFO": "/my/path/", + "REQUEST_METHOD": self.method, + } + environ.update(addtional_environ) + query_string = "" + raw_query_string = environ.get("QUERY_STRING", None) + if raw_query_string: + query_string = f"?{raw_query_string}" + request = f"{self.method} {environ['PATH_INFO']}{query_string} {server_protocol}" + + regex_log_entry = f'{host} {logname} {user} [] "{request}" {status_code} {content_length} "{referer}" "{self.agent}"' + rt_us = kwargs.get("rt_us") + if rt_us: + rt_seconds = int(rt_us / 1_000_000) + regex_log_entry = f"{regex_log_entry} {rt_seconds}/{rt_us}" + return environ, regex_log_entry + + def test_with_response_time__true(self): + formatter = ApacheNCSAFormatter(with_response_time=True) + expected = "format_log_with_response_time" + actual = formatter.__name__ + self.assertEqual(actual, expected) + + status_code = 200 + content_length = 10 + rt_us = 15 + environ, expected = self._build_expected_format_string(status_code, {}, content_length, rt_us=15) + actual = formatter(status_code, environ, content_length, rt_us=rt_us) + self.assertRegexpMatches(actual, self.datetime_regex) + # extract and remove matched datetime + result = self.datetime_regex.search(actual) + match_start, match_end = result.span() + replace_text = actual[match_start:match_end] + actual = actual.replace(replace_text, "") + self.assertEqual(actual, expected) + + agent_endstring = f'"{self.agent}"' + self.assertFalse(actual.endswith(agent_endstring)) + + def test_with_response_time__true__with_querystring(self): + formatter = ApacheNCSAFormatter(with_response_time=True) + + status_code = 200 + content_length = 10 + rt_us = 15 + additional_environ = {"QUERY_STRING": "name=hello&data=hello"} + environ, expected = self._build_expected_format_string(status_code, additional_environ, content_length, rt_us=15) + actual = formatter(status_code, environ, content_length, rt_us=rt_us) + self.assertRegexpMatches(actual, self.datetime_regex) + # extract and remove matched datetime + result = self.datetime_regex.search(actual) + match_start, match_end = result.span() + replace_text = actual[match_start:match_end] + actual = actual.replace(replace_text, "") + self.assertEqual(actual, expected) + agent_endstring = f'"{self.agent}"' + self.assertFalse(actual.endswith(agent_endstring)) + + def test_with_response_time__false(self): + formatter = ApacheNCSAFormatter(with_response_time=False) + + expected = "format_log" + actual = formatter.__name__ + self.assertEqual(actual, expected) + + status_code = 200 + content_length = 10 + environ, expected = self._build_expected_format_string(status_code, {}, content_length) + actual = formatter(status_code, environ, content_length) + self.assertRegexpMatches(actual, self.datetime_regex) + # extract and remove matched datetime + result = self.datetime_regex.search(actual) + match_start, match_end = result.span() + replace_text = actual[match_start:match_end] + actual = actual.replace(replace_text, "") + self.assertEqual(actual, expected) + + agent_endstring = f'"{self.agent}"' + self.assertTrue(actual.endswith(agent_endstring)) + + def test_with_response_time__false__with_querystring(self): + formatter = ApacheNCSAFormatter(with_response_time=False) + + status_code = 200 + content_length = 10 + additional_environ = {"QUERY_STRING": "name=hello&data=hello"} + environ, expected = self._build_expected_format_string(status_code, additional_environ, content_length) + actual = formatter(status_code, environ, content_length) + self.assertRegexpMatches(actual, self.datetime_regex) + # extract and remove matched datetime + result = self.datetime_regex.search(actual) + match_start, match_end = result.span() + replace_text = actual[match_start:match_end] + actual = actual.replace(replace_text, "") + self.assertEqual(actual, expected) + agent_endstring = f'"{self.agent}"' + self.assertTrue(actual.endswith(agent_endstring)) diff --git a/zappa/utilities.py b/zappa/utilities.py index 15da07c1c..9a514b2f5 100644 --- a/zappa/utilities.py +++ b/zappa/utilities.py @@ -9,7 +9,7 @@ import shutil import stat import sys -from typing import Any +from typing import Any, Callable from urllib.parse import urlparse import botocore @@ -602,6 +602,81 @@ def merge_headers(event): return multi_headers +class ApacheNCSAFormatters: + """ + NCSA extended/combined Log Format: + "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" + %h: Remote hostname. + %l: Remote logname + %u: Remote user if the request was authenticated. May be bogus if return status (%s) is 401 (unauthorized). + %t: Time the request was received, in the format [18/Sep/2011:19:18:28 -0400]. + The last number indicates the timezone offset from GMT + %r: First line of request. + %>s: Final Status + %b: Size of response in bytes, excluding HTTP headers. + In CLF format, i.e. a '-' rather than a 0 when no bytes are sent. + %{Referer}i:The contents of Referer: header line(s) in the request sent to the server. + %{User-agent}i: The contents of User-agent: header line(s) in the request sent to the server. + + Refer to: + https://httpd.apache.org/docs/current/en/mod/mod_log_config.html + """ + + @staticmethod + def format_log(status_code: int, environ: dict, content_length: int, **kwargs) -> str: + ip_header = kwargs.get("ip_header", None) + if ip_header: + host = environ.get(ip_header, "") + else: + host = environ.get("REMOTE_ADDR", "") + + logname = "-" + user = "-" + now = datetime.datetime.now(datetime.timezone.utc) + display_datetime = now.strftime("%d/%b/%Y:%H:%M:%S %z") + method = environ.get("REQUEST_METHOD", "") + path_info = environ.get("PATH_INFO", "") + query_string = "" + raw_query_string = environ.get("QUERY_STRING", "") + if raw_query_string: + query_string = f"?{raw_query_string}" + server_protocol = environ.get("SERVER_PROTOCOL", "") + request = f"{method} {path_info}{query_string} {server_protocol}" + referer = environ.get("HTTP_REFERER", "") + agent = environ.get("HTTP_USER_AGENT", "") + log_entry = ( + f'{host} {logname} {user} [{display_datetime}] "{request}" {status_code} {content_length} "{referer}" "{agent}"' + ) + return log_entry + + @staticmethod + def format_log_with_response_time(*args, **kwargs) -> str: + """ + Expect that kwargs includes response time in microseconds, 'rt_us'. + Mimics Apache-like access HTTP log where the response time data is enabled + + NCSA extended/combined Log Format: + "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %T/%D" + + %T: The time taken to serve the request, in seconds. + %D: The time taken to serve the request, in microseconds. + """ + response_time_microseconds = kwargs.get("rt_us", None) + log_entry = ApacheNCSAFormatters.format_log(*args, **kwargs) + if response_time_microseconds: + response_time_seconds = int(response_time_microseconds / 1_000_000) + log_entry = f"{log_entry} {response_time_seconds}/{response_time_microseconds}" + return log_entry + + +def ApacheNCSAFormatter(with_response_time: bool = True) -> Callable: + """A factory that returns the wanted formatter""" + if with_response_time: + return ApacheNCSAFormatters.format_log_with_response_time + else: + return ApacheNCSAFormatters.format_log + + def validate_json_serializable(*args: Any, **kwargs: Any) -> None: try: json.dumps((args, kwargs)) diff --git a/zappa/wsgi.py b/zappa/wsgi.py index 3b0729c36..c1889c08f 100644 --- a/zappa/wsgi.py +++ b/zappa/wsgi.py @@ -4,10 +4,9 @@ from io import BytesIO from urllib.parse import urlencode -from requestlogger import ApacheFormatter from werkzeug import urls -from .utilities import merge_headers, titlecase_keys +from .utilities import ApacheNCSAFormatter, merge_headers, titlecase_keys BINARY_METHODS = ["POST", "PUT", "PATCH", "DELETE", "CONNECT", "OPTIONS"] @@ -166,24 +165,15 @@ def common_log(environ, response, response_time=None): logger = logging.getLogger() if response_time: - formatter = ApacheFormatter(with_response_time=True) - try: - log_entry = formatter( - response.status_code, - environ, - len(response.content), - rt_us=response_time, - ) - except TypeError: - # Upstream introduced a very annoying breaking change on the rt_ms/rt_us kwarg. - log_entry = formatter( - response.status_code, - environ, - len(response.content), - rt_ms=response_time, - ) + formatter = ApacheNCSAFormatter(with_response_time=True) + log_entry = formatter( + response.status_code, + environ, + len(response.content), + rt_us=response_time, + ) else: - formatter = ApacheFormatter(with_response_time=False) + formatter = ApacheNCSAFormatter(with_response_time=False) log_entry = formatter(response.status_code, environ, len(response.content)) logger.info(log_entry)