From 33840bd7a4290a411a94c97080b07b002219b2ed Mon Sep 17 00:00:00 2001 From: Mike Finch Date: Thu, 28 Jul 2022 16:30:04 -0400 Subject: [PATCH] Support static websites Fixes #13 Closes #42 This change add support for reading index.html files from S3 and serving them as the default content when a directory is requested. Signed-off-by: Elijah Zupancic --- .../00-check-for-required-env.sh | 18 +++++ common/etc/nginx/include/s3gateway.js | 35 ++++++++- common/etc/nginx/nginx.conf | 2 + .../etc/nginx/templates/default.conf.template | 15 +++- docs/getting_started.md | 14 +++- settings.example | 2 + test.sh | 32 +++++--- test/docker-compose.yaml | 2 + test/integration/test_api.sh | 75 ++++++++++++++++--- 9 files changed, 169 insertions(+), 26 deletions(-) diff --git a/common/docker-entrypoint.d/00-check-for-required-env.sh b/common/docker-entrypoint.d/00-check-for-required-env.sh index 2e093499..cfc883a3 100644 --- a/common/docker-entrypoint.d/00-check-for-required-env.sh +++ b/common/docker-entrypoint.d/00-check-for-required-env.sh @@ -50,6 +50,22 @@ if [ "${AWS_SIGS_VERSION}" != "2" ] && [ "${AWS_SIGS_VERSION}" != "4" ]; then failed=1 fi +parseBoolean() { + case "$1" in + TRUE | true | True | YES | Yes | 1) + echo 1 + ;; + *) + echo 0 + ;; + esac +} + +if [ "$(parseBoolean ${ALLOW_DIRECTORY_LIST})" == "1" ] && [ "$(parseBoolean ${PROVIDE_INDEX_PAGE})" == "1" ]; then + >&2 echo "ALLOW_DIRECTORY_LIST and PROVIDE_INDEX_PAGE cannot be both set" + failed=1 +fi + if [ $failed -gt 0 ]; then exit 1 fi @@ -62,3 +78,5 @@ echo "Addressing Style: ${S3_STYLE}" echo "AWS Signatures Version: v${AWS_SIGS_VERSION}" echo "DNS Resolvers: ${DNS_RESOLVERS}" echo "Directory Listing Enabled: ${ALLOW_DIRECTORY_LIST}" +echo "Provide Index Pages Enabled: ${PROVIDE_INDEX_PAGE}" +echo "Append slash for directory enabled: ${APPEND_SLASH_FOR_POSSIBLE_DIRECTORY}" diff --git a/common/etc/nginx/include/s3gateway.js b/common/etc/nginx/include/s3gateway.js index 72024591..de55d317 100644 --- a/common/etc/nginx/include/s3gateway.js +++ b/common/etc/nginx/include/s3gateway.js @@ -32,9 +32,13 @@ var fs = require('fs'); */ var debug = _parseBoolean(process.env['S3_DEBUG']); var allow_listing = _parseBoolean(process.env['ALLOW_DIRECTORY_LIST']) +var provide_index_page = _parseBoolean(process.env['PROVIDE_INDEX_PAGE']) +var append_slash = _parseBoolean(process.env['APPEND_SLASH_FOR_POSSIBLE_DIRECTORY']) var s3_style = process.env['S3_STYLE']; +var INDEX_PAGE = "index.html"; + /** * The current moment as a timestamp. This timestamp will be used across * functions in order for there to be no variations in signatures. @@ -348,11 +352,14 @@ function s3uri(r) { if (allow_listing) { var queryParams = _s3DirQueryParams(uriPath, r.method); if (queryParams.length > 0) { - path = basePath + '?' + queryParams; + path = basePath + '/?' + queryParams; } else { path = basePath + uriPath; } } else { + if (provide_index_page && _isDirectory(uriPath) ) { + uriPath += INDEX_PAGE; + } path = basePath + uriPath; } @@ -374,6 +381,11 @@ function _s3DirQueryParams(uriPath, method) { return ''; } + // return if static website. We don't want to list the files in the directory, we want to append the index page and get the fil. + if (provide_index_page){ + return ''; + } + let path = 'delimiter=%2F' if (uriPath !== '/') { @@ -406,13 +418,25 @@ function redirectToS3(r) { if (isDirectoryListing && r.method === 'GET') { r.internalRedirect("@s3Listing"); - } else if (!isDirectoryListing && uriPath === '/') { - r.internalRedirect("@error404"); + } else if ( provide_index_page == true ) { + r.internalRedirect("@s3"); + } else if ( !allow_listing && !provide_index_page && uriPath == "/" ) { + r.internalRedirect("@error404"); } else { r.internalRedirect("@s3"); } } +function trailslashControl(r) { + if (append_slash) { + var hasExtension = /\/[^.\/]+\.[^.]+$/; + if (!hasExtension.test(r.variables.uri_path) && !_isDirectory(r.variables.uri_path)){ + return r.internalRedirect("@trailslash"); + } + } + r.internalRedirect("@error404"); +} + /** * Create HTTP Authorization header for authenticating with an AWS compatible * v2 API. @@ -431,6 +455,10 @@ function signatureV2(r, bucket, credentials) { * nginx, then in S3 we need to request /?delimiter=/&prefix=dir1/ * Thus, we can't put the path /dir1/ in the string to sign. */ var uri = _isDirectory(r.variables.uri_path) ? '/' : r.variables.uri_path; + // To return index pages + index.html + if (provide_index_page && _isDirectory(r.variables.uri_path)){ + uri = r.variables.uri_path + INDEX_PAGE + } var hmac = mod_hmac.createHmac('sha1', credentials.secretAccessKey); var httpDate = s3date(r); var stringToSign = method + '\n\n\n' + httpDate + '\n' + '/' + bucket + uri; @@ -1003,6 +1031,7 @@ export default { s3auth, s3SecurityToken, s3uri, + trailslashControl, redirectToS3, editAmzHeaders, filterListResponse, diff --git a/common/etc/nginx/nginx.conf b/common/etc/nginx/nginx.conf index b8d39513..551b5cde 100644 --- a/common/etc/nginx/nginx.conf +++ b/common/etc/nginx/nginx.conf @@ -20,6 +20,8 @@ env AWS_SIGS_VERSION; env S3_DEBUG; env S3_STYLE; env ALLOW_DIRECTORY_LIST; +env PROVIDE_INDEX_PAGE; +env APPEND_SLASH_FOR_POSSIBLE_DIRECTORY; env PROXY_CACHE_VALID_OK; env PROXY_CACHE_VALID_NOTFOUND; env PROXY_CACHE_VALID_FORBIDDEN; diff --git a/common/etc/nginx/templates/default.conf.template b/common/etc/nginx/templates/default.conf.template index 98dce0ff..1e2e6a75 100644 --- a/common/etc/nginx/templates/default.conf.template +++ b/common/etc/nginx/templates/default.conf.template @@ -109,7 +109,9 @@ server { proxy_intercept_errors on; # Comment out this line to receive the error messages returned by S3 - error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 420 422 423 424 426 428 429 431 444 449 450 451 500 501 502 503 504 505 506 507 508 509 510 511 =404 @error404; + error_page 400 401 402 403 405 406 407 408 409 410 411 412 413 414 415 416 417 418 420 422 423 424 426 428 429 431 444 449 450 451 500 501 502 503 504 505 506 507 508 509 510 511 =404 @error404; + + error_page 404 @trailslashControl; proxy_pass ${S3_SERVER_PROTO}://storage_urls$s3uri; } @@ -166,6 +168,17 @@ server { return 404; } + location @trailslashControl { + # Checks if requesting a folder without trailing slash, and return 302 + # appending a slash to it when using for static site hosting. + js_content s3gateway.trailslashControl; + } + + location @trailslash { + # 302 to request without slashes + rewrite ^ $scheme://$http_host$request_uri/ redirect; + } + # Provide a hint to the client on 405 errors of the acceptable request methods error_page 405 @error405; location @error405 { diff --git a/docs/getting_started.md b/docs/getting_started.md index 37a67dc8..e519e030 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -13,7 +13,9 @@ The following environment variables are used to configure the gateway when running as a Container or as a Systemd service. -* `ALLOW_DIRECTORY_LIST` - Enable directory listing - either true or false +* `ALLOW_DIRECTORY_LIST` - Flag (true/false) enabling directory listing (default: false) +* `PROVIDE_INDEX_PAGE` - Flag (true/false) which returns the index page if there is one when requesting a directory. Cannot be enabled with `ALLOW_DIRECTORY_LIST`. (default: false) +* `APPEND_SLASH_FOR_POSSIBLE_DIRECTORY` - Flag (true/false) enabling the return a 302 with a `/` appended to the path. This is independent of the behavior selected in `ALLOW_DIRECTORY_LIST` or `PROVIDE_INDEX_PAGE`. (default: false) * `AWS_SIGS_VERSION` - AWS Signatures API version - either 2 or 4 * `DNS_RESOLVERS` - (optional) DNS resolvers (separated by single spaces) to configure NGINX with * `S3_ACCESS_KEY_ID` - Access key @@ -62,6 +64,16 @@ result in log messages like: Another limitation is that when using v2 signatures with HEAD requests, the gateway will not return 200 for valid folders. +### Static Site Hosting + +When `PROVIDE_INDEX_PAGE` environment variable is set to 1, the gateway will +transform `/some/path/` to `/some/path/index.html` when retrieving from S3. +Default of "index.html" can be edited in `s3gateway.js`. +It will also redirect `/some/path` to `/some/path/` when S3 returns 404 on +`/some/path` if `APPEND_SLASH_FOR_POSSIBLE_DIRECTORY` is set. `path` has to +look like a possible directory, it must not start with a `.` and not have an +extension. + ## Running as a Systemd Service A [install script](/standalone_ubuntu_oss_install.sh) for the gateway shows diff --git a/settings.example b/settings.example index 3552982f..23bf87df 100644 --- a/settings.example +++ b/settings.example @@ -9,6 +9,8 @@ S3_STYLE=virtual S3_DEBUG=false AWS_SIGS_VERSION=4 ALLOW_DIRECTORY_LIST=false +PROVIDE_INDEX_PAGE=false +APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=false PROXY_CACHE_VALID_OK=1h PROXY_CACHE_VALID_NOTFOUND=1m PROXY_CACHE_VALID_FORBIDDEN=30s diff --git a/test.sh b/test.sh index 3d79eb19..216fcc2a 100755 --- a/test.sh +++ b/test.sh @@ -110,11 +110,15 @@ integration_test() { printf "\e[1m Integration test suite for v%s signatures\e[22m\n" "$1" printf "\033[34;1m▶\033[0m" printf "\e[1m Integration test suite with ALLOW_DIRECTORY_LIST=%s\e[22m\n" "$2" + printf "\033[34;1m▶\033[0m" + printf "\e[1m Integration test suite with PROVIDE_INDEX_PAGE=%s\e[22m\n" "$3" + printf "\033[34;1m▶\033[0m" + printf "\e[1m Integration test suite with APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=%s\e[22m\n" "$4" # See if Minio is already running, if it isn't then we don't need to build it if [ -z "$(docker ps -q -f name=${test_compose_project}_minio_1)" ]; then p "Building Docker Compose environment" - AWS_SIGS_VERSION=$1 ALLOW_DIRECTORY_LIST=$2 compose up --no-start + COMPOSE_COMPATIBILITY=true AWS_SIGS_VERSION=$1 ALLOW_DIRECTORY_LIST=$2 PROVIDE_INDEX_PAGE=$3 APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=$4 compose up --no-start p "Adding test data to container" echo "Copying contents of ${test_dir}/data to Docker container ${test_compose_project}_minio_1:/" @@ -124,7 +128,7 @@ integration_test() { fi p "Starting Docker Compose Environment" - AWS_SIGS_VERSION=$1 ALLOW_DIRECTORY_LIST=$2 compose up -d + COMPOSE_COMPATIBILITY=true AWS_SIGS_VERSION=$1 ALLOW_DIRECTORY_LIST=$2 PROVIDE_INDEX_PAGE=$3 APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=$4 compose up -d if [ ${wait_for_it_installed} ]; then # Hit minio's health check end point to see if it has started up @@ -145,8 +149,8 @@ integration_test() { fi p "Starting HTTP API tests (v$1 signatures)" - echo " test/integration/test_api.sh \"$test_server\" \"$test_dir\" $1 $2" - bash "${test_dir}/integration/test_api.sh" "$test_server" "$test_dir" "$1" "$2"; + echo " test/integration/test_api.sh \"$test_server\" \"$test_dir\" $1 $2 $3 $4" + bash "${test_dir}/integration/test_api.sh" "$test_server" "$test_dir" "$1" "$2" "$3" "$4"; # We check to see if NGINX is in fact using the correct version of AWS # signatures as it was configured to do. @@ -225,21 +229,31 @@ ${docker_cmd} run \ ### INTEGRATION TESTS p "Testing API with AWS Signature V2 and allow directory listing off" -integration_test 2 0 +integration_test 2 0 0 0 compose stop nginx-s3-gateway # Restart with new config p "Testing API with AWS Signature V2 and allow directory listing on" -integration_test 2 1 +integration_test 2 1 0 0 + +compose stop nginx-s3-gateway # Restart with new config + +p "Testing API with AWS Signature V2 and static site on" +integration_test 2 0 1 0 compose stop nginx-s3-gateway # Restart with new config p "Test API with AWS Signature V4 and allow directory listing off" -integration_test 4 0 +integration_test 4 0 0 0 + +compose stop nginx-s3-gateway # Restart with new config + +p "Test API with AWS Signature V4 and allow directory listing on and appending /" +integration_test 4 1 0 1 compose stop nginx-s3-gateway # Restart with new config -p "Test API with AWS Signature V4 and allow directory listing on" -integration_test 4 1 +p "Test API with AWS Signature V4 and static site on appending /" +integration_test 4 0 1 1 p "All integration tests complete" diff --git a/test/docker-compose.yaml b/test/docker-compose.yaml index 660d2754..4fba31f8 100644 --- a/test/docker-compose.yaml +++ b/test/docker-compose.yaml @@ -23,6 +23,8 @@ services: S3_DEBUG: "true" S3_STYLE: "virtual" ALLOW_DIRECTORY_LIST: + PROVIDE_INDEX_PAGE: + APPEND_SLASH_FOR_POSSIBLE_DIRECTORY: AWS_SIGS_VERSION: PROXY_CACHE_VALID_OK: "1h" PROXY_CACHE_VALID_NOTFOUND: "1m" diff --git a/test/integration/test_api.sh b/test/integration/test_api.sh index bd27409a..41e4ea11 100644 --- a/test/integration/test_api.sh +++ b/test/integration/test_api.sh @@ -24,6 +24,8 @@ test_server=$1 test_dir=$2 signature_version=$3 allow_directory_list=$4 +index_page=$5 +append_slash=$6 test_fail_exit_code=2 no_dep_exit_code=3 checksum_length=32 @@ -62,16 +64,23 @@ assertHttpRequestEquals() { uri="${test_server}/${path}" fi + if [ "${index_page}" == "1" ]; then + # Follow 302 redirect if testing static hosting + extra_arg="-L -v" + else + extra_arg="" + fi + printf " \033[36;1m▲\033[0m " echo "Testing object: ${method} ${path}" if [ "${method}" = "HEAD" ]; then expected_response_code="$3" - actual_response_code="$(${curl_cmd} -s -o /dev/null -w '%{http_code}' --head "${uri}")" + actual_response_code="$(${curl_cmd} -s -o /dev/null -w '%{http_code}' --head "${uri}" ${extra_arg})" if [ "${expected_response_code}" != "${actual_response_code}" ]; then e "Response code didn't match expectation. Request [${method} ${uri}] Expected [${expected_response_code}] Actual [${actual_response_code}]" - e "curl command: ${curl_cmd} -s -o /dev/null -w '%{http_code}' --head '${uri}'" + e "curl command: ${curl_cmd} -s -o /dev/null -w '%{http_code}' --head '${uri}' ${extra_arg}" exit ${test_fail_exit_code} fi elif [ "${method}" = "GET" ]; then @@ -81,21 +90,21 @@ assertHttpRequestEquals() { checksum_output="$(${checksum_cmd} "${body_data_path}")" expected_checksum="${checksum_output:0:${checksum_length}}" - curl_checksum_output="$(${curl_cmd} -s -X "${method}" "${uri}" | ${checksum_cmd})" + curl_checksum_output="$(${curl_cmd} -s -X "${method}" "${uri}" ${extra_arg} | ${checksum_cmd})" s3_file_checksum="${curl_checksum_output:0:${checksum_length}}" if [ "${expected_checksum}" != "${s3_file_checksum}" ]; then e "Checksum doesn't match expectation. Request [${method} ${uri}] Expected [${expected_checksum}] Actual [${s3_file_checksum}]" - e "curl command: ${curl_cmd} -s -X '${method}' '${uri}' | ${checksum_cmd}" + e "curl command: ${curl_cmd} -s -X '${method}' '${uri}' ${extra_arg} | ${checksum_cmd}" exit ${test_fail_exit_code} fi else expected_response_code="$3" - actual_response_code="$(${curl_cmd} -s -o /dev/null -w '%{http_code}' "${uri}")" + actual_response_code="$(${curl_cmd} -s -o /dev/null -w '%{http_code}' "${uri}" ${extra_arg})" if [ "${expected_response_code}" != "${actual_response_code}" ]; then e "Response code didn't match expectation. Request [${method} ${uri}] Expected [${expected_response_code}] Actual [${actual_response_code}]" - e "curl command: ${curl_cmd} -s -o /dev/null -w '%{http_code}' '${uri}'" + e "curl command: ${curl_cmd} -s -o /dev/null -w '%{http_code}' '${uri}' ${extra_arg}" exit ${test_fail_exit_code} fi fi @@ -109,7 +118,7 @@ set +o errexit # Allow curl command to fail with a non-zero exit code for this block because # we want to use it to test to see if the server is actually up. for (( i=1; i<=3; i++ )); do - response="$(${curl_cmd} -s -o /dev/null -w '%{http_code}' --head "${test_server}")" + response="$(${curl_cmd} -v -s -o /dev/null -w '%{http_code}' --head "${test_server}")" if [ "${response}" != "000" ]; then break fi @@ -138,11 +147,17 @@ assertHttpRequestEquals "HEAD" "системы/system.txt" "200" assertHttpRequestEquals "HEAD" "%D1%81%D0%B8%D1%81%D1%82%D0%B5%D0%BC%D1%8B/%25bad%25file%25name%25" "200" # Expected 400s -assertHttpRequestEquals "HEAD" "request with unencoded spaces" "400" +# curl will not send this to server now +# assertHttpRequestEquals "HEAD" "request with unencoded spaces" "400" # Expected 404s -assertHttpRequestEquals "HEAD" "not%20found" "404" -assertHttpRequestEquals "HEAD" "b/c" "404" +if [ "${append_slash}" == "1" ] && [ "${index_page}" == "0" ]; then + assertHttpRequestEquals "HEAD" "not%20found" "302" + assertHttpRequestEquals "HEAD" "b/c" "302" +else + assertHttpRequestEquals "HEAD" "not%20found" "404" + assertHttpRequestEquals "HEAD" "b/c" "404" +fi # Directory HEAD 404s # Unfortunately, the logic here can't be properly encoded into the test. @@ -151,15 +166,35 @@ assertHttpRequestEquals "HEAD" "b/c" "404" # running with v4 signatures. # Now, both of these cases have the exception of HEAD returning 200 on the root # directory. -if [ "${allow_directory_list}" == "1" ]; then +if [ "${allow_directory_list}" == "1" ] || [ "${index_page}" == "1" ]; then assertHttpRequestEquals "HEAD" "/" "200" else assertHttpRequestEquals "HEAD" "/" "404" fi assertHttpRequestEquals "HEAD" "b/" "404" assertHttpRequestEquals "HEAD" "/b/c/" "404" -assertHttpRequestEquals "HEAD" "b//c" "404" assertHttpRequestEquals "HEAD" "/soap" "404" +if [ "${append_slash}" == "1" ] && [ "${index_page}" == "0" ]; then +assertHttpRequestEquals "HEAD" "b//c" "302" +else +assertHttpRequestEquals "HEAD" "b//c" "404" +fi + +if [ "${index_page}" == "1" ]; then +assertHttpRequestEquals "HEAD" "/statichost/" "200" +assertHttpRequestEquals "HEAD" "/nonexistdir/noindexdir/" "404" +assertHttpRequestEquals "HEAD" "/nonexistdir/noindexdir" "404" +assertHttpRequestEquals "HEAD" "/statichost/noindexdir/multipledir/" "200" +assertHttpRequestEquals "HEAD" "/nonexistdir/" "404" +assertHttpRequestEquals "HEAD" "/nonexistdir" "404" + if [ ${append_slash} == "1" ]; then + assertHttpRequestEquals "HEAD" "/statichost" "200" + assertHttpRequestEquals "HEAD" "/statichost/noindexdir/multipledir" "200" + else + assertHttpRequestEquals "HEAD" "/statichost" "404" + assertHttpRequestEquals "HEAD" "/statichost/noindexdir/multipledir" "404" + fi +fi # Verify GET is working assertHttpRequestEquals "GET" "a.txt" "data/bucket-1/a.txt" @@ -173,12 +208,28 @@ assertHttpRequestEquals "GET" "b/クズ箱/ゴミ.txt" "data/bucket-1/b/クズ assertHttpRequestEquals "GET" "системы/system.txt" "data/bucket-1/системы/system.txt" assertHttpRequestEquals "GET" "%D1%81%D0%B8%D1%81%D1%82%D0%B5%D0%BC%D1%8B/%25bad%25file%25name%25" "data/bucket-1/системы/%bad%file%name%" +if [ "${index_page}" == "1" ]; then +assertHttpRequestEquals "GET" "/statichost/" "data/bucket-1/statichost/index.html" +assertHttpRequestEquals "GET" "/statichost/noindexdir/multipledir/" "data/bucket-1/statichost/noindexdir/multipledir/index.html" + if [ "${append_slash}" == "1" ]; then + assertHttpRequestEquals "GET" "/statichost" "data/bucket-1/statichost/index.html" + assertHttpRequestEquals "GET" "/statichost/noindexdir/multipledir" "data/bucket-1/statichost/noindexdir/multipledir/index.html" + fi +fi + if [ "${allow_directory_list}" == "1" ]; then assertHttpRequestEquals "GET" "/" "200" assertHttpRequestEquals "GET" "b/" "200" assertHttpRequestEquals "GET" "/b/c/" "200" assertHttpRequestEquals "GET" "b/クズ箱/" "200" assertHttpRequestEquals "GET" "системы/" "200" + if [ "$append_slash" == "1" ]; then + assertHttpRequestEquals "GET" "b" "302" + else + assertHttpRequestEquals "GET" "b" "404" + fi +elif [ "${index_page}" == "1" ]; then + assertHttpRequestEquals "GET" "/" "data/bucket-1/index.html" else assertHttpRequestEquals "GET" "/" "404" fi