Skip to content

Commit

Permalink
Resolve (zappa#410) Logs are missing query strings (zappa#1165)
Browse files Browse the repository at this point in the history 
* ✅ migrate utilities based tests from `tests.py` to `tests_utilities.py`

* 🔥 remove `wsgi-request-logger` dependency (appears to no longer be maintained) and bring formatter functionality into zappa
✨ add `ApacheNCSAFormatter` to replace wsgi-request-logger functionality.

* 🔧 black is no longer in 'beta', changing allow_prereleases to false.
  • Loading branch information
@monkut @Ian288
monkut authored and Ian288 committed Jul 11, 2023
1 parent 7fe267f commit 45826c2
Show file tree
Hide file tree
Showing 6 changed files with 521 additions and 313 deletions.
5 changes: 4 additions & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,4 +73,7 @@ test-async:
test-general:
nosetests tests/tests.py --with-coverage --cover-package=zappa --with-timer

tests: clean test-docs test-handler test-middleware test-placebo test-async test-general
test-utilities:
nosetests tests/tests_utilities.py --with-coverage --cover-package=zappa --with-timer

tests: clean test-docs test-handler test-middleware test-placebo test-async test-general test-utilities
4 changes: 1 addition & 3 deletions Pipfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,6 @@ tqdm = "*"
troposphere = ">=3.0"
Werkzeug = "*"
wheel = "*"
wsgi-request-logger = "*"

[pipenv]
# Required for 'black' since all of its release tags contain 'b'
allow_prereleases = true
allow_prereleases = false
289 changes: 0 additions & 289 deletions tests/tests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,7 @@

from zappa.cli import ZappaCLI, disable_click_colors, shamelessly_promote
from zappa.core import ALB_LAMBDA_ALIAS, ASSUME_POLICY, ATTACH_POLICY, Zappa
from zappa.ext.django_zappa import get_django_wsgi
from zappa.letsencrypt import (
cleanup,
create_chained_certificate,
create_domain_csr,
create_domain_key,
Expand All @@ -39,21 +37,6 @@
parse_csr,
register_account,
sign_certificate,
verify_challenge,
)
from zappa.utilities import (
InvalidAwsLambdaName,
conflicts_with_a_neighbouring_module,
contains_python_files_or_subdirs,
detect_django_settings,
detect_flask_apps,
get_venv_from_python_version,
human_size,
is_valid_bucket_name,
parse_s3_url,
string_to_timestamp,
titlecase_keys,
validate_name,
)
from zappa.wsgi import common_log, create_wsgi_request

Expand Down Expand Up @@ -101,61 +84,6 @@ def test_disable_click_colors(self):
disable_click_colors()
assert resolve_color_default() is False

@mock.patch("zappa.core.find_packages")
@mock.patch("os.remove")
def test_copy_editable_packages(self, mock_remove, mock_find_packages):
virtual_env = os.environ.get("VIRTUAL_ENV")
if not virtual_env:
return self.skipTest("test_copy_editable_packages must be run in a virtualenv")

temp_package_dir = tempfile.mkdtemp()
try:
egg_links = [
os.path.join(
virtual_env,
"lib",
get_venv_from_python_version(),
"site-packages",
"test-copy-editable-packages.egg-link",
)
]
egg_path = "/some/other/directory/package"
mock_find_packages.return_value = [
"package",
"package.subpackage",
"package.another",
]
temp_egg_link = os.path.join(temp_package_dir, "package-python.egg-link")

z = Zappa()
mock_open = mock.mock_open(read_data=egg_path.encode("utf-8"))
with mock.patch("zappa.core.open", mock_open), mock.patch("glob.glob") as mock_glob, mock.patch(
"zappa.core.copytree"
) as mock_copytree:
# we use glob.glob to get the egg-links in the temp packages
# directory
mock_glob.return_value = [temp_egg_link]

z.copy_editable_packages(egg_links, temp_package_dir)

# make sure we copied the right directories
mock_copytree.assert_called_with(
os.path.join(egg_path, "package"),
os.path.join(temp_package_dir, "package"),
metadata=False,
symlinks=False,
)
self.assertEqual(mock_copytree.call_count, 1)

# make sure it removes the egg-link from the temp packages
# directory
mock_remove.assert_called_with(temp_egg_link)
self.assertEqual(mock_remove.call_count, 1)
finally:
shutil.rmtree(temp_package_dir)

return

def test_create_lambda_package(self):
# mock the pkg_resources.WorkingSet() to include a known package in lambda_packages so that the code
# for zipping pre-compiled packages gets called
Expand Down Expand Up @@ -2028,119 +1956,6 @@ def test_get_all_zones_two_pages(self, client):
)
self.assertListEqual(zones["HostedZones"], [{"Id": "zone1"}, {"Id": "zone2"}])

##
# Django
##

def test_detect_dj(self):
# Sanity
settings_modules = detect_django_settings()

def test_dj_wsgi(self):
# Sanity
settings_modules = detect_django_settings()

settings = """
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'alskdfjalsdkf=0*%do-ayvy*m2k=vss*$7)j8q!@u0+d^na7mi2(^!l!d'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
TEMPLATE_DEBUG = True
ALLOWED_HOSTS = []
# Application definition
INSTALLED_APPS = (
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
)
MIDDLEWARE_CLASSES = (
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
)
ROOT_URLCONF = 'blah.urls'
WSGI_APPLICATION = 'hackathon_starter.wsgi.application'
# Database
# https://docs.djangoproject.com/en/1.7/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
}
}
# Internationalization
# https://docs.djangoproject.com/en/1.7/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
"""

djts = open("dj_test_settings.py", "w")
djts.write(settings)
djts.close()

app = get_django_wsgi("dj_test_settings")
try:
os.remove("dj_test_settings.py")
os.remove("dj_test_settings.pyc")
except Exception as e:
pass

##
# Util / Misc
##

def test_human_units(self):
human_size(1)
human_size(9999999999999)

def test_string_to_timestamp(self):
boo = string_to_timestamp("asdf")
self.assertTrue(boo == 0)

yay = string_to_timestamp("1h")
self.assertTrue(type(yay) == int)
self.assertTrue(yay > 0)

yay = string_to_timestamp("4m")
self.assertTrue(type(yay) == int)
self.assertTrue(yay > 0)

yay = string_to_timestamp("1mm")
self.assertTrue(type(yay) == int)
self.assertTrue(yay > 0)

yay = string_to_timestamp("1mm1w1d1h1m1s1ms1us")
self.assertTrue(type(yay) == int)
self.assertTrue(yay > 0)

def test_event_name(self):
zappa = Zappa()
truncated = zappa.get_event_name(
Expand Down Expand Up @@ -2234,42 +2049,9 @@ def test_get_scheduled_event_name__truncated__has_name__has_index(self):
f"{hashed_lambda_name}-{index}-{event['name']}-{function}",
)

def test_detect_dj(self):
# Sanity
settings_modules = detect_django_settings()

def test_detect_flask(self):
# Sanity
settings_modules = detect_flask_apps()

def test_shameless(self):
shamelessly_promote()

def test_s3_url_parser(self):
remote_bucket, remote_file = parse_s3_url("s3://my-project-config-files/filename.json")
self.assertEqual(remote_bucket, "my-project-config-files")
self.assertEqual(remote_file, "filename.json")

remote_bucket, remote_file = parse_s3_url("s3://your-bucket/account.key")
self.assertEqual(remote_bucket, "your-bucket")
self.assertEqual(remote_file, "account.key")

remote_bucket, remote_file = parse_s3_url("s3://my-config-bucket/super-secret-config.json")
self.assertEqual(remote_bucket, "my-config-bucket")
self.assertEqual(remote_file, "super-secret-config.json")

remote_bucket, remote_file = parse_s3_url("s3://your-secure-bucket/account.key")
self.assertEqual(remote_bucket, "your-secure-bucket")
self.assertEqual(remote_file, "account.key")

remote_bucket, remote_file = parse_s3_url("s3://your-bucket/subfolder/account.key")
self.assertEqual(remote_bucket, "your-bucket")
self.assertEqual(remote_file, "subfolder/account.key")

# Sad path
remote_bucket, remote_file = parse_s3_url("/dev/null")
self.assertEqual(remote_bucket, "")

def test_remote_env_package(self):
zappa_cli = ZappaCLI()
zappa_cli.api_stage = "deprecated_remote_env"
Expand Down Expand Up @@ -2395,28 +2177,6 @@ def test_slim_handler(self):

zappa_cli.remove_local_zip()

def test_validate_name(self):
fname = "tests/name_scenarios.json"
with open(fname, "r") as f:
scenarios = json.load(f)
for scenario in scenarios:
value = scenario["value"]
is_valid = scenario["is_valid"]
if is_valid:
assert validate_name(value)
else:
with self.assertRaises(InvalidAwsLambdaName) as exc:
validate_name(value)

def test_contains_python_files_or_subdirs(self):
self.assertTrue(contains_python_files_or_subdirs("tests/data"))
self.assertTrue(contains_python_files_or_subdirs("tests/data/test2"))
self.assertFalse(contains_python_files_or_subdirs("tests/data/test1"))

def test_conflicts_with_a_neighbouring_module(self):
self.assertTrue(conflicts_with_a_neighbouring_module("tests/data/test1"))
self.assertFalse(conflicts_with_a_neighbouring_module("tests/data/test2"))

def test_settings_py_generation(self):
zappa_cli = ZappaCLI()
zappa_cli.api_stage = "ttt888"
Expand Down Expand Up @@ -2449,55 +2209,6 @@ def test_only_ascii_env_var_allowed(self):
zappa_cli.create_package()
self.assertEqual("Environment variable keys must be ascii.", str(context.exception))

def test_titlecase_keys(self):
raw = {
"hOSt": "github.com",
"ConnECtiOn": "keep-alive",
"UpGRAde-InSecuRE-ReQueSts": "1",
"uSer-AGEnT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36",
"cONtENt-TYPe": "text/html; charset=utf-8",
"aCCEpT": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"ACcePT-encoDInG": "gzip, deflate, br",
"AcCEpT-lAnGUagE": "en-US,en;q=0.9",
}
transformed = titlecase_keys(raw)
expected = {
"Host": "github.com",
"Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36",
"Content-Type": "text/html; charset=utf-8",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"Accept-Encoding": "gzip, deflate, br",
"Accept-Language": "en-US,en;q=0.9",
}
self.assertEqual(expected, transformed)

def test_is_valid_bucket_name(self):
# Bucket names must be at least 3 and no more than 63 characters long.
self.assertFalse(is_valid_bucket_name("ab"))
self.assertFalse(is_valid_bucket_name("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefhijlmn"))
# Bucket names must not contain uppercase characters or underscores.
self.assertFalse(is_valid_bucket_name("aaaBaaa"))
self.assertFalse(is_valid_bucket_name("aaa_aaa"))
# Bucket names must start with a lowercase letter or number.
self.assertFalse(is_valid_bucket_name(".abbbaba"))
self.assertFalse(is_valid_bucket_name("abbaba."))
self.assertFalse(is_valid_bucket_name("-abbaba"))
self.assertFalse(is_valid_bucket_name("ababab-"))
# Bucket names must be a series of one or more labels. Adjacent labels are separated by a single period (.).
# Each label must start and end with a lowercase letter or a number.
self.assertFalse(is_valid_bucket_name("aaa..bbbb"))
self.assertFalse(is_valid_bucket_name("aaa.-bbb.ccc"))
self.assertFalse(is_valid_bucket_name("aaa-.bbb.ccc"))
# Bucket names must not be formatted as an IP address (for example, 192.168.5.4).
self.assertFalse(is_valid_bucket_name("192.168.5.4"))
self.assertFalse(is_valid_bucket_name("127.0.0.1"))
self.assertFalse(is_valid_bucket_name("255.255.255.255"))

self.assertTrue(is_valid_bucket_name("valid-formed-s3-bucket-name"))
self.assertTrue(is_valid_bucket_name("worst.bucket.ever"))

# TODO: encountered error when vpc_config["SubnetIds"] or vpc_config["SecurityGroupIds"] is missing
# We need to make the code more robust in this case and avoid the KeyError
def test_zappa_core_deploy_lambda_alb_missing_cert_arn(self):
Expand Down
Loading

0 comments on commit 45826c2

Please sign in to comment.