"]
-workspace = "../"
-edition = "2018"
-
-
-
-[dependencies]
-libc = "0.2"
-
-# time
-time = "0.2"
-chrono = "0.4"
-
-# serialization
-serde = "1"
-serde_json = "1"
-serde_derive = "1"
-base64 = "0.13"
-
-#lazy static
-lazy_static = "1"
-
-# fast hashes, crypto hashs
-twox-hash = "1"
-fnv = "*"
-blake2-rfc = "*"
-
-## Crate-specific dependencies
-
-
-clap = "2"
-
-imageflow_core = { path = "../imageflow_core", version = "*" }
-imageflow_types = { path = "../imageflow_types", version = "*" }
-imageflow_helpers = { path = "../imageflow_helpers", version = "*" }
-imageflow_riapi = { path = "../imageflow_riapi", version = "*" }
-imageflow_http_helpers = { path = "../imageflow_http_helpers", version = "*" }
-
-
-rand = "0.6"
-lru-cache = "0.1"
-regex = "1"
-log="0.4"
-env_logger="0.7"
-wait-timeout = "0.2"
-
-bincode = "1"
-staticfile = { git= "https://github.com/onur/staticfile", rev= "9f2ff7201eda648128c92e3f5597c587f0629f51" }
-conduit-mime-types = "0.7"
-router = "=0.5"
-iron = "=0.5.1"
-persistent = "=0.4"
-hyper = { version = "0.14", default-features = false }
-threadpool = "1.8"
-url="1"
-hyper-native-tls="0.3"
-reqwest="0.11"
-http="*"
-
-[[bin]]
-name = "imageflow_server"
-path = "src/main.rs"
-doc = false
-
-[dependencies.mount]
-git = "https://github.com/iron/mount.git"
-rev = "2c3d719be4c158d4ddbd8cdb402fafccdefec58c"
-
-[dependencies.logger]
-git = "https://github.com/iron/logger.git"
-rev = "0daead5fe10c3cd0c4738767c162dc63a59c3fb3"
-
-[features]
-nightly = ["imageflow_core/nightly", "imageflow_helpers/nightly"]
diff --git a/imageflow_server/README.md b/imageflow_server/README.md
deleted file mode 100644
index b7ba6e446..000000000
--- a/imageflow_server/README.md
+++ /dev/null
@@ -1,16 +0,0 @@
-# imageflow_server
-
-If you're compiling, use `cargo run --bin imageflow_server` instead of `imageflow_server`.
-
-Currently we have 4 mount providers:
-* ir4_http - ImageResizer4 compatible querystring API, pulling originals from a remote server
-* ir4_local - ImageResizer4 compatible querystring API, pulling from disk
-* static - static file server
-* permacache_proxy - static file proxy with permanent caching (no invalidation, ever)
-
-* `imageflow_server start --demo`
-* `imageflow_server start --port 80 --data-dir=./imageflow_data --mount /ir4/local/:ir4_local:./img/ --mount /ir4/remote/:ir4_http:http:://remote.com/img/ --mount`
-* `imageflow_server start --port 80 --data-dir=./imageflow_data --mount /js/:static:./js --mount /proxy_asis/:permacache_proxy:http:://remote.com/static/:360`
-* `imageflow_server diagnose --show-compilation-info`
-
-http://localhost:3004/ir4/proxy_unsplash/photo-1422493757035-1e5e03968f95?width=600
\ No newline at end of file
diff --git a/imageflow_server/demo/index.html b/imageflow_server/demo/index.html
deleted file mode 100644
index dd6784d17..000000000
--- a/imageflow_server/demo/index.html
+++ /dev/null
@@ -1,205 +0,0 @@
-
-
- Sample page
-
-
-
- ImageResizer 4 Examples
- Hover over the images to view the querystrings used. All commands can be combined.
-
-
-
-
Resizing using maxwidth and/or maxheight
-
Aspect ratio is always maintained with maxwidth and maxheight. The image is scaled to fit within those bounds.
-
-
-
-
-
Resizing using width and height
- Specifying only one of width or height will behave the same as using maxwidth
- or maxheight. The difference is when you specify both.
- Specifying both width and height will force the image to those exact dimensions, unless the
- image is already smaller (see scale). This is done by adding whitespace to the image. To center and crop instead, use
- &mode=crop. To lose aspect ratio and fill the specified rectangle, use &mode=stretch.
- 
- 
- 
- 
-
- Scaling
- By default, images are not upscaled. If an image is already smaller than width/height/maxwidth/maxheight, it is not resized.
- To upscale images, use ?scale=both. ?scale=downscaleonly is the default.
-
-
- 
-
- You can control the color by setting &bgcolor=color|hex.
-
- Upscaling the canvas is sometimes desired instead of upscaling the image when it is smaller than the requested size. Use ?scale=upscalecanvas to achieve this effect.
- 
-
-
- Cropping
- To enable cropping, you can use &crop=auto, which minimally crops and centers to preserve aspect ratio, or custom cropping.
- &crop=(x1,y1,x2,y2) specifies the rectangle to crop on the image. You can still resize and modify the cropped portion
- using the other commands as normal. Negative coordinates are relative to the bottom-right corner -
- which makes it easy to trim off a 50-pixel border by specifying &crop=(50,50,-50,-50).
-
-  "Cropping out a 200x200 square using ?crop=(10,150,200,350)")
-  "Cropping out a 200x200 square using ?crop=(60,200,250,400)")
- 
- Cropping can also be done against arbitrary scales, which is very useful for jQuery jCrop interfaces.
- Example to crop 10% off each edge: crop=(.1,.1,.9,.9)&cropxunits=1&cropyunits=1
- Example to crop an image relative to the 'final' coordinates, without knowing the original size. Ex. crop=(20,30,400,350)&cropxunits=500&cropxunits=390
- 
&cropxunits=1&cropyunits=1&width=200 "Cropping 10% off each edge using ?crop=(.1,.1,.9,.9)&cropxunits=1&cropyunits=1")
- Rotation
- rotate=90|180|280
- 
-
-  "Using ?rotate=-90&crop=(60,200,250,400)")
-
- 
-
- Flipping
- You can horizontally or vertically flip an image, as well as both. &flip=h|v|both
-  "Using ?flip=v&crop=(60,200,250,400)")
-  "Using ?flip=both&crop=(60,200,250,400)")
-
-  "Using ?flip=h&crop=(60,200,250,400)")
- Source flipping
- Since normal flipping applies after rotation and cropping occur, it can be
- difficult to work with if you are just wanting the source image flipped before the other
- adjustments are applied. To flip the source prior to work, use &sFlip=h|v|both.
- Note how the same crop coordinates return different sections of the image. This is because the source image is flipped before *anything* happens.
- 
-  "Using ?crop=(0,0,100,100)")
-  "Using ?sflip=both&crop=(0,0,100,100)")
- Stretching
- To stretch an image to width and height, use &stretch=fill.
- 
-
-
-
- Output format
- Jpeg compression levels 0-100 (&quality=0-100)
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
-
-
- Transparent GIFs and PNGs
- Transparency is maintained when resizing PNGs
- 
- 
- 
- Transparency is maintained when resizing WebP images and converting PNG to WebP
- 
- 
- Jpegs can be converted to webp
- 
- 
-
- Whitespace cropping
- 
- 
-
-
-
-
-
-
-
-
-
diff --git a/imageflow_server/dockerize.sh b/imageflow_server/dockerize.sh
deleted file mode 100755
index e9a6ed43a..000000000
--- a/imageflow_server/dockerize.sh
+++ /dev/null
@@ -1,147 +0,0 @@
-#!/bin/bash
-set -e
-
-# The purpose of this script is to compile Imageflow locally (or in a CI simulation docker container), then copy it to *another* docker container, and run a basic smoke test.
-# This can help detect incompatibilites and missing basics, like glibc.
-
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-export FROM_IMAGE="imazen/imageflow_base_os"
-export BUILD_IMAGE_NAME="imazen/imageflow_build_ubuntu16"
-export OUTPUT_IMAGE_NAME="local/if_testing"
-export DOCKER_DIR="/home/imageflow"
-
-export SAFE_IMAGE_NAME="$BUILD_IMAGE_NAME"
-SAFE_IMAGE_NAME="${SAFE_IMAGE_NAME//\//_}"
-SAFE_IMAGE_NAME="${SAFE_IMAGE_NAME//:/_}"
-
-
-echo "./dockerize.sh $1 $2 $3"
-echo "help: ./dockerize.sh (debug|quiet[123]|release|clean|valgrind|test|rusttest)+ localbuild|docker server|tool #SMOKE TESTS ONLY"
-
-export OVERRIDE="$1"
-export OVERRIDE="${OVERRIDE:-debugquiet}"
-if [[ "$OVERRIDE" == *"debug"* ]]; then
- export PROFILE=debug
-else
- export PROFILE=release
-fi
-
-export CARGO_TARGET="${CARGO_TARGET:-}"
-
-if [[ -n "$CARGO_TARGET" ]]; then
- export TARGET_DIR="target/${CARGO_TARGET}/"
-else
- export TARGET_DIR="target/"
-fi
-
-
-if [[ "$3" == 'tool' ]]; then
- export BINARY_NAME=imageflow_tool
- export TEST_ENTRYPOINT=(sudo "${DOCKER_DIR}/${BINARY_NAME}" diagnose --self-test)
-else
- export BINARY_NAME=imageflow_server
- export TEST_ENTRYPOINT=(sudo "${DOCKER_DIR}/${BINARY_NAME}" diagnose --smoke-test-core)
-fi
-
-if [[ "$2" == 'docker' ]]; then
-
- TARGET_CPU="${TARGET_CPU:-x86-64}"
- WORKING_DIR="${HOME}/.docker_imageflow_caches/.docker_${SAFE_IMAGE_NAME}_${TARGET_CPU}"
- export BINARY_DIR="${WORKING_DIR}_cache/${TARGET_DIR}${PROFILE}"
-else
- export BINARY_DIR="${SCRIPT_DIR}/../${TARGET_DIR}${PROFILE}"
-fi
-
-
-if [[ -d "$BINARY_DIR" ]]; then
- export BINARY_DIR
- BINARY_DIR="$(readlink -f "$BINARY_DIR")"
-else
- echo "Cannot find $BINARY_DIR"
-fi
-export BINARY_OUT="$BINARY_DIR/$BINARY_NAME"
-export BINARY_COPY="${SCRIPT_DIR}/bin/$BINARY_NAME"
-mkdir -p "${SCRIPT_DIR}/bin/" || true
-mkdir -p "${BINARY_DIR}" || true &>/dev/null
-
-sep_bar(){
- printf "\n=================== %s ======================\n" "$1"
-}
-print_modified_ago(){
- if [[ -f "$1" ]]; then
- printf "(modified %s seconds ago)" "$(( $(date +%s) - $(stat -c%Y "$1") ))"
- fi
-}
-
-sep_bar "Compiling"
-printf "BINARY_OUT=%s " "$BINARY_OUT" && print_modified_ago "$BINARY_OUT" && printf "\n"
-
-export BUILD_QUIETER="${BUILD_QUIETER:-True}"
-export UPLOAD_BUILD=False
-export UPLOAD_DOCS=False
-export IMAGEFLOW_BUILD_OVERRIDE="$OVERRIDE"
-
-if [[ "$2" == 'docker' ]]; then
- ( cd "${SCRIPT_DIR}/../ci" && ./simulate_travis.sh "${BUILD_IMAGE_NAME}" )
-else
- ( "${SCRIPT_DIR}/../build.sh" "${OVERRIDE}" )
-
- #if [[ "$PROFILE" == 'debug' ]]; then
- # ( set -vx && cd "${SCRIPT_DIR}/../${CRATE_NAME}" && cargo build --bin "${BINARY_NAME}" )
- #else
- # ( set -vx && cd "${SCRIPT_DIR}/../${CRATE_NAME}" && cargo build --bin "${BINARY_NAME}" --release )
- #fide
-fi
-
-# Post-compile build info
-"${BINARY_OUT}" --version || ( printf "\nBINARY_OUT=%s " "$BINARY_OUT" && print_modified_ago "$BINARY_OUT" && printf "\n" )
-
-# Generate and build Dockerfile
-sep_bar "Dockerizing"
-(
- cd "$SCRIPT_DIR"
- cp -p "${BINARY_OUT}" .
- printf "\nCreating Dockerfile\n\n"
- printf "FROM %s\n\nEXPOSE 39876\n\nADD %s %s/" "$FROM_IMAGE" "$BINARY_NAME" "$DOCKER_DIR" > Dockerfile
- docker build -t "$OUTPUT_IMAGE_NAME" .
-)
-sep_bar "Smoke testing in Docker"
-docker run --rm "${OUTPUT_IMAGE_NAME}" "${DOCKER_DIR}/${BINARY_NAME}" --version || printf "Failed to run %s --version!\n" "${BINARY_NAME}"
-
-set +e
-
-if docker run --rm "${OUTPUT_IMAGE_NAME}" "${TEST_ENTRYPOINT[@]}"; then
- sep_bar "PASSED"
-else
- sep_bar "FAILED"
- export TEST_FAILED=1
-fi
-set -e
-
-
-
-if [[ "$TEST_FAILED" == '1' ]]; then
- echo "Entering interactive"
- echo "This creates docker containers and doesn't clean them up. Use this to remove all containers (danger!)"
- # shellcheck disable=SC2016
- echo 'docker rm `docker ps -aq`'
-
- docker run -i -t "${OUTPUT_IMAGE_NAME}" /bin/bash
-
- exit 1
-fi
-
-if [[ "$BINARY_NAME" == 'imageflow_server' ]]; then
- docker run -i -t -p 3000:3000 "${OUTPUT_IMAGE_NAME}" sudo "${DOCKER_DIR}/${BINARY_NAME}" start --demo --port 3000 --bind-address 0.0.0.0
-fi
-
-
-#docker push "${IMAGE_NAME}"
-#docker-cloud stack up --name "$TEST_STACK_NAME"
-#docker-cloud stack update "$TEST_STACK_NAME"
-#export STACK_UID= $(docker-cloud stack up --name flow3 -f docker-solo.yaml)
-#printf "%s" "${STACK_UID}"
-#docker-cloud stack redeploy "$TEST_STACK_NAME"
-
-
-
diff --git a/imageflow_server/src/assets/identity.p12 b/imageflow_server/src/assets/identity.p12
deleted file mode 100644
index d16abb8c7..000000000
Binary files a/imageflow_server/src/assets/identity.p12 and /dev/null differ
diff --git a/imageflow_server/src/assets/root-ca.pem b/imageflow_server/src/assets/root-ca.pem
deleted file mode 100644
index 4ec2f5388..000000000
--- a/imageflow_server/src/assets/root-ca.pem
+++ /dev/null
@@ -1,21 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDXTCCAkWgAwIBAgIJAOIvDiVb18eVMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV
-BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
-aWRnaXRzIFB0eSBMdGQwHhcNMTYwODE0MTY1NjExWhcNMjYwODEyMTY1NjExWjBF
-MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50
-ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
-CgKCAQEArVHWFn52Lbl1l59exduZntVSZyDYpzDND+S2LUcO6fRBWhV/1Kzox+2G
-ZptbuMGmfI3iAnb0CFT4uC3kBkQQlXonGATSVyaFTFR+jq/lc0SP+9Bd7SBXieIV
-eIXlY1TvlwIvj3Ntw9zX+scTA4SXxH6M0rKv9gTOub2vCMSHeF16X8DQr4XsZuQr
-7Cp7j1I4aqOJyap5JTl5ijmG8cnu0n+8UcRlBzy99dLWJG0AfI3VRJdWpGTNVZ92
-aFff3RpK3F/WI2gp3qV1ynRAKuvmncGC3LDvYfcc2dgsc1N6Ffq8GIrkgRob6eBc
-klDHp1d023Lwre+VaVDSo1//Y72UFwIDAQABo1AwTjAdBgNVHQ4EFgQUbNOlA6sN
-XyzJjYqciKeId7g3/ZowHwYDVR0jBBgwFoAUbNOlA6sNXyzJjYqciKeId7g3/Zow
-DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAVVaR5QWLZIRR4Dw6TSBn
-BQiLpBSXN6oAxdDw6n4PtwW6CzydaA+creiK6LfwEsiifUfQe9f+T+TBSpdIYtMv
-Z2H2tjlFX8VrjUFvPrvn5c28CuLI0foBgY8XGSkR2YMYzWw2jPEq3Th/KM5Catn3
-AFm3bGKWMtGPR4v+90chEN0jzaAmJYRrVUh9vea27bOCn31Nse6XXQPmSI6Gyncy
-OAPUsvPClF3IjeL1tmBotWqSGn1cYxLo+Lwjk22A9h6vjcNQRyZF2VLVvtwYrNU3
-mwJ6GCLsLHpwW/yjyvn8iEltnJvByM/eeRnfXV6WDObyiZsE/n6DxIRJodQzFqy9
-GA==
------END CERTIFICATE-----
diff --git a/imageflow_server/src/assets/tiny.jpg b/imageflow_server/src/assets/tiny.jpg
deleted file mode 100644
index 71911bf48..000000000
Binary files a/imageflow_server/src/assets/tiny.jpg and /dev/null differ
diff --git a/imageflow_server/src/config.rs b/imageflow_server/src/config.rs
deleted file mode 100644
index 309fa6b90..000000000
--- a/imageflow_server/src/config.rs
+++ /dev/null
@@ -1,67 +0,0 @@
-
-// use serde_toml
-// Deserialize from TOML or from inline struct
-
-// pub struct Hostname
-//
-// pub struct PerRequestLimits{
-// max_pixels_out: Option,
-// max_pixels_in: Option,
-// max_cpu_milliseconds: Option,
-// max_bitmap_ram_bytes: Option
-// }
-//
-// pub struct ContentTypeRestrictions{
-// allow: Option>,
-// deny: Option>,
-// allow_extensions: Option>,
-// deny_extensions: Option>
-// }
-// pub struct SecurityPolicy{
-// per_request_limits: Option,
-// serve_content_types: Option,
-// proxy_content_types: Option,
-// force_image_recoding: Option
-// }
-//
-// pub enum BlobSource{
-// Directory(String),
-// HttpServer(String),
-// //TODO: Azure and S3 blob backend
-// }
-//
-// pub enum InternalCachingStrategy{
-// PubSubAndPermaPyramid,
-// TrackStatsAndPermaPyramid,
-// OpportunistPermaPyramid,
-// PubSubToInvalidate,
-// OpportunistPubSubEtagCheck,
-//
-// }
-// pub struct CacheControlPolicy{
-// //How do we set etag/last modified/expires/maxage?
-// }
-//
-// pub struct BaseConfig{
-// //Security defaults
-// pub security: Option,
-// //May also want to filter by hostnames or ports for heavy multi-tenanting
-// pub cache_control: Option
-// }
-//
-// pub enum Frontend{
-// ImageResizer4Compatible,
-// Flow0
-// }
-// pub struct MountPath {
-// //Where we get originals from
-// pub source: BlobSource,
-// //The virtual path for which we handle sub-requests.
-// pub prefix: String,
-// //Customize security
-// pub security: Option,
-// //May also want to filter by hostnames or ports for heavy multi-tenanting
-// pub cache_control: Option,
-//
-// pub api: Frontend
-// }
\ No newline at end of file
diff --git a/imageflow_server/src/diagnose.rs b/imageflow_server/src/diagnose.rs
deleted file mode 100644
index 4077503b8..000000000
--- a/imageflow_server/src/diagnose.rs
+++ /dev/null
@@ -1,46 +0,0 @@
-use imageflow_helpers::preludes::from_std::*;
-use imageflow_core::clients::stateless;
-use imageflow_core::clients::fluent;
-use crate::s;
-
-const BLUE_PNG32_200X200_B64:&'static str = "iVBORw0KGgoAAAANSUhEUgAAAMgAAADICAYAAACtWK6eAAABiUlEQVR42u3TgRAAQAgAsA/qkaKLK48EIug2h8XP6gesQhAQBAQBQUAQEAQEAUFAEBAEEAQEAUFAEBAEBAFBQBAQBAQRBAQBQUAQEAQEAUFAEBAEBAEEAUFAEBAEBAFBQBAQBAQBQQBBQBAQBAQBQUAQEAQEAUEAQUAQEAQEAUFAEBAEBAFBQBBAEBAEBAFBQBAQBAQBQUAQQBAQBAQBQUAQEAQEAUFAEBAEEAQEAUFAEBAEBAFBQBAQBAQRBAQBQUAQEAQEAUFAEBAEBAEEAUFAEBAEBAFBQBAQBAQBQQQBQUAQEAQEAUFAEBAEBAFBAEFAEBAEBAFBQBAQBAQBQUAQQBAQBAQBQUAQEAQEAUFAEEAQEAQEAUFAEBAEBAFBQBAQBBAEBAFBQBAQBAQBQUAQEAQQBAQBQUAQEAQEAUFAEBAEBAEEAUFAEBAEBAFBQBAQBAQBQQQBQUAQEAQEAUFAEBAEBAFBAEFAEBAEBAFBQBAQBAQBQUAQQUAQEAQEAUFAEBAEBIGLBkZ+sahOjkyUAAAAAElFTkSuQmCC";
-
-
-fn smoke_jpeg_to_jpeg() {
- let framewise = fluent::fluently()
- .decode(0)
- .constrain_within(Some(40), Some(40), Some(s::ResampleHints::with(None, Some(25f32))))
- .encode(1, s::EncoderPreset::libjpeg_turbo()).builder().to_framewise();
-
- let bytes = include_bytes!("assets/tiny.jpg");
-
- let req = stateless::BuildRequest{
- export_graphs_to: None,
- inputs: vec![stateless::BuildInput{bytes, io_id: 0}],
- framewise
- };
- let _ = stateless::LibClient{}.build(req).unwrap();
-}
-
-
-
-fn smoke_png_to_png() {
- let framewise = fluent::fluently()
- .decode(0)
- .constrain_within(Some(40), Some(40), Some(s::ResampleHints::with(None, Some(25f32))))
- .encode(1, s::EncoderPreset::libpng32()).builder().to_framewise();
-
- let bytes = base64::decode(BLUE_PNG32_200X200_B64).unwrap();
-
- let req = stateless::BuildRequest{
- export_graphs_to: None,
- inputs: vec![stateless::BuildInput{bytes: &bytes, io_id: 0}],
- framewise: framewise
- };
- let _ = stateless::LibClient{}.build(req).unwrap();
-}
-
-pub fn smoke_test_core() {
- smoke_png_to_png();
- smoke_jpeg_to_jpeg();
-}
diff --git a/imageflow_server/src/disk_cache.rs b/imageflow_server/src/disk_cache.rs
deleted file mode 100644
index 1c1fbb0cf..000000000
--- a/imageflow_server/src/disk_cache.rs
+++ /dev/null
@@ -1,301 +0,0 @@
-/// This is a naive local 'caching' implementation of a key/value blob store
-/// Each pair gets 1 file
-/// Hash collisions are improbable - we use blake2 256, faster than SHA-3. 32-byte hashes
-/// Write only
-/// Append-only log for transitioning to more complex system
-/// Staging folder - files are renamed into final locations
-/// soft and hard count and byte limit - NO DELETION
-use std::path::*;
-use std::io;
-use std;
-use std::io::prelude::*;
-use std::fs::{create_dir_all, File};
-use std::sync::atomic::{AtomicBool, Ordering};
-use self::rand::RngCore;
-// TODO:
-// Cleanup staging folders automatically (failed renames)
-// Implement write-only log
-// Implement transactional filesystem 'counters' to track total count/size
-// Implement write failure when limits are reached
-
-// It *is* possible to implement FIFO cache eviction, but is FIFO worth it? (random sampling of the write log, staging folders to drop handles, etc)
-
-extern crate rand;
-extern crate imageflow_helpers;
-use self::imageflow_helpers as hlp;
-
-
-fn create_dir_all_helpful>(path: P) -> io::Result<()> {
- match create_dir_all(&path) {
- Ok(v) => Ok(v),
- Err(e) => {
- //panic!("Failed to create dir {:?} {:?}", path.as_ref(), e);
- Err(e)
- }
- }
-}
-
-
-//
-///// Cache of the state of the cache folder - can be invalidated by failure.
-//struct CacheCache{
-//
-//}
-
-//Since we have a fixed number of folders, known at creation time, let's deterministically order them
-//and use a bitset or something.
-//We can use RwLock on a BitVec or a Vec of AtomicBools (64kb vs 8kb, but maybe we just collapse for storage?)
-
-#[derive(Debug)]
-pub struct CacheFolder{
- root: PathBuf,
- root_confirmed: AtomicBool,
- meta_dir: PathBuf,
- staging_dir: PathBuf,
- meta_layout_confirmed: AtomicBool,
- write_log: PathBuf,
- consumption_log: PathBuf,
- consumption_summary: PathBuf,
- folder_bits: u8,
- folders_from_hash: u32,
- bits_format: &'static str,
- write_layout: FolderLayout
-}
-
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-pub enum FolderLayout {
- /// 64 tier 1 folders, each with a 'files' subdirectory. Optimal range 0 to 8,000 entries or so. Suggested max 51k.
- Tiny,
- /// 64 x 64, each leaf with a 'files' subdirectory. Optimal range ~8,000 to ~500,000 entries. Suggested max 3 million.
- Normal,
- /// 64 x 64 x 16. Optimal for 500k to 8 million entries. Suggested max 50 million.
- Huge
-}
-
-impl CacheFolder{
- pub fn new(root: &Path, write_layout: FolderLayout) -> CacheFolder{
- CacheFolder{
- meta_layout_confirmed: AtomicBool::default(),
- root: root.to_owned(),
- root_confirmed: AtomicBool::default(),
- meta_dir: root.join(Path::new("meta")),
- staging_dir: root.join(Path::new("staging")),
- write_log: root.join(Path::new("meta")).join(Path::new("write_log")),
- consumption_log: root.join(Path::new("meta")).join(Path::new("consumption_log")),
- consumption_summary: root.join(Path::new("meta")).join(Path::new("consumption_summary")),
- folder_bits: match write_layout{
- FolderLayout::Tiny => 6,
- FolderLayout::Normal => 12,
- FolderLayout::Huge => 16,
- },
- bits_format: match write_layout{
- FolderLayout::Tiny => "{250-256:02x}/files/{0-256:064x}",
- FolderLayout::Normal => "{250-256:02x}/{244-250:02x}/files/{0-256:064x}",
- FolderLayout::Huge => "{250-256:02x}/{244-250:02x}/{240-244:02x}/{0-256:064x}",
- },
- folders_from_hash: match write_layout{
- FolderLayout::Tiny => 64 * 2,
- FolderLayout::Normal => 64 * 64 * 2 + 64,
- FolderLayout::Huge => 64 * 64 * 16 + 64 * 64 + 64,
- },
- write_layout: write_layout
- }
- }
-
- pub fn entry(&self, hash: &[u8;32]) -> CacheEntry {
- CacheEntry {
- path: self.root.join(hlp::hashing::normalize_slashes(hlp::hashing::bits_format(hash, self.bits_format))),
- hash: *hash,
- parent: self
- }
- }
-// PUT A README IN THE CACHE ROOT! DO IT!
-
-
- fn ensure_root(&self) -> io::Result<()>{
- if !self.root_confirmed.load(Ordering::Relaxed) &&
- !self.root.as_path().is_dir(){
- create_dir_all_helpful(&self.root)?;
- self.root_confirmed.store(true, Ordering::Relaxed);
- }
- Ok(())
- }
-
- fn ensure_meta_layout_confirmed(&self) -> io::Result<()>{
- if !self.meta_layout_confirmed.load(Ordering::SeqCst){
- let path = self.meta_dir.join(Path::new(match self.write_layout{
- FolderLayout::Huge => "huge",
- FolderLayout::Tiny => "tiny",
- FolderLayout::Normal => "normal"
- }));
- if !self.meta_layout_confirmed.load(Ordering::SeqCst) && !path.exists() {
- create_dir_all_helpful(&self.meta_dir)?;
- File::create(path)?;
- self.meta_layout_confirmed.store(true, Ordering::SeqCst);
- }
- }
- Ok(())
- }
-
- ///TODO: we could optimize directory existence checks with an 8, 512, or 8kb BitVec, easily persisted.
- /// We would need 'fast path' that falls back to 'careful path' when any of those caches get out of sync
- fn prepare_for(&self, entry: &CacheEntry) -> io::Result<()> {
- self.ensure_root().unwrap();
- self.ensure_meta_layout_confirmed().unwrap();
- let dir = entry.path.as_path().parent().expect("Every cache path should have a parent dir; this did not!");
- if !dir.exists(){
- create_dir_all_helpful(dir)?;
- }
- Ok(())
- }
-
- fn acquire_staging_location(&self, hash: &[u8;32]) -> io::Result{
- if !self.staging_dir.as_path().exists(){
- create_dir_all_helpful(self.staging_dir.as_path())?;
- }
-
- let slot_id = hlp::timeywimey::time_bucket(60 * 60 * 2, 6);
-
- //six slots, each used for 2 hours.
- let subdir = self.staging_dir.join(Path::new(&format!("{}", slot_id)));
- if !subdir.exists() {
- create_dir_all_helpful(subdir.as_path())?;
- }
-
- let staging_path = format!("{:064x}_{:016x}_incoming", hlp::hashing::HexableBytes(hash), rand::thread_rng().next_u64());
- Ok(subdir.join(Path::new(&staging_path)))
- }
-}
-
-pub struct CacheEntry<'a>{
- //Path shall always have a valid parent.
- hash: [u8;32],
- path: PathBuf,
- parent: &'a CacheFolder
-}
-
-impl<'a> CacheEntry<'a>{
- pub fn prepare_dir(&self) -> io::Result<()>{
- self.parent.prepare_for(self)
- }
- pub fn exists(&self) -> bool{
- self.path.as_path().is_file()
- }
-
- // static NEXT_FLUENT_NODE_ID: AtomicU64 = ATOMIC_U64_INIT;
-
-
- // We have to write to a different file, first. Then we fs::rename() to overwrite
- //
- pub fn write(&self, bytes: &[u8]) -> io::Result<()> {
- self.prepare_dir()?;
- let temp_path = self.parent.acquire_staging_location(&self.hash)?;
- use ::std::fs::OpenOptions;
- {
- let mut f = OpenOptions::new().write(true).create_new(true).open(&temp_path)?;
- f.write_all(bytes)?;
- }
- std::fs::rename(&temp_path, &self.path)?;
- Ok(())
- }
-
- pub fn read(&self) -> io::Result>{
- hlp::filesystem::read_file_bytes(&self.path)
- }
-
-
-}
-
-// If one migrates from one FolderLayout to another, or is moving off of a old cache directory, then multiple queries make sense
-// Check for meta/tiny, meta/normal, meta/huge presence to auto-populate
-//struct CacheReader{
-// folders: Vec
-//}
-//
-//impl CacheReader{
-//
-//}
-//
-//
-//
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-//We should log each 32-byte hash saved to a file. This would be far faster than a directory listing later.
-//128 entries per block is quite efficient. We could handle 10 million in 320mb.
-
-//But if we don't require aligning to block boundaries, another 11 to 16 bytes would be quite handy.
-
-//We could log size for another 8 bytes (or 4 or 5 if we are ok with a 2 or 512GB limit)
-//Dimensions are u16xu16 for the formats we care about. at least 1 byte to cover all mime types
-//And another 2-3 bytes for useful metadata about the image??
-// I.e, 43-48 bytes per record.
-
-//We can append to file pretty reliably, but reads will get torn at the end https://stackoverflow.com/questions/1154446/is-file-append-atomic-in-unix
-
-
-
-
-
-
-//
-//hashes are 32 bytes?
-//
-//Cache write
-//Create 3 parent directories
-//
-//
-//
-//
-//There is no cleanup
-//64 x 64 x 1 nested directories by default. Optional 64x64x16 + 1. with read through
-//
-//
-//
-//
-//
-//
-//Allow read-through to
-//There is no index
-//
-//
-//Fetch by hash - blake2d 256bit
-//
-//blake2-rfc = "0.2.17" or https://github.com/RustCrypto/hashes
-//base 32?
-//
-//
-//Filesystem
-//Disable 8.3 on windows
-//Disable all metadata (as much as possible)
-//only list unsorted
-//https://stackoverflow.com/questions/197162/ntfs-performance-and-large-volumes-of-files-and-directories
-
-
-
-
-
-
-
-
diff --git a/imageflow_server/src/lib.rs b/imageflow_server/src/lib.rs
deleted file mode 100644
index 692c2b434..000000000
--- a/imageflow_server/src/lib.rs
+++ /dev/null
@@ -1,757 +0,0 @@
-
-extern crate iron;
-extern crate persistent;
-extern crate router;
-extern crate logger;
-
-extern crate bincode;
-extern crate mount;
-
-use staticfile::Static;
-
-
-#[macro_use] extern crate serde_derive;
-
-extern crate staticfile;
-extern crate hyper;
-
-extern crate time;
-#[macro_use] extern crate lazy_static;
-extern crate regex;
-
-extern crate hyper_native_tls;
-
-use hyper_native_tls::NativeTlsServer;
-
-use std::sync::atomic::AtomicUsize;
-
-
-extern crate conduit_mime_types as mime_types;
-
-use regex::Regex;
-
-extern crate imageflow_helpers;
-extern crate imageflow_core;
-extern crate imageflow_types as s;
-extern crate imageflow_riapi;
-extern crate reqwest;
-
-use ::imageflow_helpers as hlp;
-use imageflow_http_helpers::FetchConfig;
-use imageflow_helpers::preludes::from_std::*;
-use imageflow_core::clients::stateless;
-
-
-pub mod disk_cache;
-pub mod resizer;
-pub mod diagnose;
-
-mod requested_path;
-extern crate url;
-
-use crate::disk_cache::{CacheFolder, FolderLayout};
-use logger::Logger;
-
-pub mod preludes {
- pub use super::{MountedEngine, MountLocation, StartServerConfig, ServerError};
- pub use super::disk_cache::FolderLayout;
-}
-
-
-use iron::mime::*;
-use iron::prelude::*;
-use iron::status;
-use router::Router;
-
-
-
-
-
-use imageflow_helpers::timeywimey::precise_time_ns;
-
-#[cfg_attr(feature = "cargo-clippy", allow(useless_attribute))]
-#[allow(unused_imports)]
-#[macro_use] extern crate log;
-extern crate env_logger;
-
-
-#[derive(Debug)]
-struct SharedData {
- source_cache: CacheFolder,
- output_cache: CacheFolder,
- requests_received: AtomicUsize,
- //detailed_errors: bool
-}
-
-impl iron::typemap::Key for SharedData { type Value = SharedData; }
-
-
-// Todo: consider lru_cache crate
-
-#[derive(Debug)]
-pub enum ServerError {
- HyperError(hyper::Error),
- ReqwestError(reqwest::Error),
- IoError(std::io::Error),
- DiskCacheReadIoError(std::io::Error),
- DiskCacheWriteIoError(std::io::Error),
- UpstreamResponseError(hyper::StatusCode),
- UpstreamResponseErrorWithBytes((hyper::StatusCode, Vec)),
- UpstreamHyperError(hyper::Error),
- UpstreamReqwestError(reqwest::Error),
- UpstreamIoError(std::io::Error),
- BuildFailure(stateless::BuildFailure),
- LayoutSizingError(::imageflow_riapi::sizing::LayoutError)
-}
-
-impl From for ServerError {
- fn from(e: stateless::BuildFailure) -> ServerError {
- ServerError::BuildFailure(e)
- }
-}
-
-impl From for ServerError {
- fn from(e: reqwest::Error) -> ServerError {
- ServerError::ReqwestError(e)
- }
-}
-impl From for ServerError {
- fn from(e: hyper::Error) -> ServerError {
- ServerError::HyperError(e)
- }
-}
-impl From<::imageflow_http_helpers::FetchError> for ServerError {
- fn from(e: ::imageflow_http_helpers::FetchError) -> ServerError {
- match e{
- ::imageflow_http_helpers::FetchError::HyperError(e) => ServerError::HyperError(e),
- ::imageflow_http_helpers::FetchError::IoError(e) => ServerError::IoError(e),
- ::imageflow_http_helpers::FetchError::UpstreamResponseError(e) => ServerError::UpstreamResponseError(e),
- ::imageflow_http_helpers::FetchError::UpstreamResponseErrorWithResponse{status, ..}=> ServerError::UpstreamResponseError(status),
- ::imageflow_http_helpers::FetchError::ReqwestError(e) => ServerError::ReqwestError(e)
- }
-
- }
-}
-
-impl From for ServerError {
- fn from(e: std::io::Error) -> ServerError {
- ServerError::IoError(e)
- }
-}
-
-struct FetchedResponse {
- bytes: Vec,
- perf: AcquirePerf,
- content_type: reqwest::header::HeaderValue,
-}
-
-fn fetch_bytes(url: &str, config: Option) -> std::result::Result {
- let start = precise_time_ns();
- let result = ::imageflow_http_helpers::fetch(url, config);
- let downloaded = precise_time_ns();
-
- match result{
- Ok(r) => {
- if r.code.is_success() {
- Ok(FetchedResponse {
- bytes: r.bytes,
- content_type: r.content_type,
- perf: AcquirePerf { fetch_ns: downloaded - start, ..Default::default() }
- })
- }else if r.bytes.len() > 0{
- Err(ServerError::UpstreamResponseErrorWithBytes((r.code, r.bytes)))
- } else {
- Err(ServerError::UpstreamResponseError(r.code))
- }
- },
- Err(e) => Err(error_upstream(e.into()))
- }
-}
-
-fn error_upstream(from: ServerError) -> ServerError {
- match from {
- ServerError::HyperError(e) => ServerError::UpstreamHyperError(e),
- ServerError::ReqwestError(e) => ServerError::UpstreamReqwestError(e),
- ServerError::IoError(e) => ServerError::UpstreamIoError(e),
- e => e,
- }
-}
-
-
-
-
-#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
-struct CachedResponse {
- bytes: Vec,
- content_type: String,
-}
-
-// Additional ways this can fail (compared to fetch_bytes)
-// Parent directories are deleted from cache between .exists() and cache writes
-// Permissions issues
-// Cached file is deleted between .exists(0 and .read()
-// Write fails due to out-of-space
-// rename fails (it should overwrite, for eventual consistency, but ... filesystems)
-fn fetch_bytes_using_cache_by_url(cache: &CacheFolder, url: &str) -> std::result::Result<(Vec, AcquirePerf), ServerError> {
- let hash = hlp::hashing::hash_256(url.as_bytes());
- let entry = cache.entry(&hash);
- if entry.exists() {
- let start = precise_time_ns();
- match entry.read() {
- Ok(vec) => {
- let end = precise_time_ns();
- Ok((vec, AcquirePerf { cache_read_ns: end - start, ..Default::default() }))
- },
- Err(e) => Err(ServerError::DiskCacheReadIoError(e))
- }
- } else {
- let result = fetch_bytes(url, Some(FetchConfig{ custom_ca_trust_file: None, read_error_body: Some(true) }));
- if let Ok(FetchedResponse { bytes, perf, .. }) = result {
- let start = precise_time_ns();
- match entry.write(&bytes) {
- Ok(()) => {
- let end = precise_time_ns();
- Ok((bytes, AcquirePerf { cache_write_ns: end - start, ..perf }))
- },
- Err(e) => Err(ServerError::DiskCacheWriteIoError(e))
- }
- } else {
- Err(result.map_err(error_upstream).err().unwrap())
- }
- }
-}
-
-fn fetch_response_using_cache_by_url(cache: &CacheFolder, url: &str) -> std::result::Result<(CachedResponse, AcquirePerf), ServerError> {
- let hash = hlp::hashing::hash_256(url.as_bytes()); //TODO: version this
- let entry = cache.entry(&hash);
- if entry.exists() {
- let start = precise_time_ns();
- match entry.read() {
- Ok(vec) => {
- let end = precise_time_ns();
- let cached: CachedResponse = bincode::deserialize(&vec).unwrap();
- Ok((cached, AcquirePerf { cache_read_ns: end - start, ..Default::default() }))
- },
- Err(e) => Err(ServerError::DiskCacheReadIoError(e))
- }
- } else {
- let result = fetch_bytes(url, None);
- if let Ok(fetched) = result {
- let start = precise_time_ns();
- let bytes = bincode::serialize(&fetched.bytes).unwrap();
- match entry.write(&bytes) {
- Ok(()) => {
- let end = precise_time_ns();
- Ok((CachedResponse { bytes: fetched.bytes, content_type: format!("{}", fetched.content_type.to_str().unwrap()) }, AcquirePerf { cache_write_ns: end - start, ..fetched.perf }))
- },
- Err(e) => Err(ServerError::DiskCacheWriteIoError(e))
- }
- } else {
- Err(result.map_err(error_upstream).err().unwrap())
- }
- }
-}
-
-fn fetch_bytes_from_disk(url: &Path) -> std::result::Result<(Vec, AcquirePerf), ServerError> {
- let start = precise_time_ns();
- let vec = hlp::filesystem::read_file_bytes(url)?;
- let end = precise_time_ns();
- Ok((vec, AcquirePerf { cache_read_ns: end - start, ..Default::default() }))
-}
-
-#[derive(Default, Copy, Clone, Debug)]
-struct AcquirePerf {
- fetch_ns: u64,
- cache_read_ns: u64,
- cache_write_ns: u64
-}
-
-
-impl AcquirePerf {
- fn total(&self) -> u64{
- self.fetch_ns + self.cache_read_ns + self.cache_write_ns
- }
-}
-
-struct RequestPerf {
- acquire: AcquirePerf,
- get_image_info_ns: u64,
- execute_ns: u64,
-}
-
-impl RequestPerf {
- fn short(&self) -> String {
- format!("execute {:.2}ms getinfo {:.2}ms fetch-through: {:.2}ms",
- self.execute_ns as f64 / 1_000_000.0f64
- , self.get_image_info_ns as f64 / 1_000_000.0f64,
- (self.acquire.total() as f64) / 1_000_000.0f64)
- }
-}
-
-
-fn execute_using(bytes_provider: F2, framewise_generator: F)
- -> std::result::Result<(stateless::BuildOutput, RequestPerf), ServerError>
- where F: Fn(s::ImageInfo) -> std::result::Result,
- F2: Fn() -> std::result::Result<(Vec, AcquirePerf), ServerError>,
-{
- let (original_bytes, acquire_perf) = bytes_provider()?;
- let mut client = stateless::LibClient {};
- let start_get_info = precise_time_ns();
- let info = client.get_image_info(&original_bytes)?;
- let start_execute = precise_time_ns();
-
- let result: stateless::BuildSuccess = client.build(stateless::BuildRequest {
- framewise: framewise_generator(info)?,
- inputs: vec![stateless::BuildInput {
- io_id: 0,
- bytes: &original_bytes,
- }],
- export_graphs_to: None,
- })?;
- let end_execute = precise_time_ns();
- Ok((result.outputs.into_iter().next().unwrap(),
- RequestPerf {
- acquire: acquire_perf,
- get_image_info_ns: start_execute - start_get_info,
- execute_ns: end_execute - start_execute,
- }))
-}
-
-fn respond_using(debug_info: &A, bytes_provider: F2, framewise_generator: F)
- -> IronResult
- where F: Fn(s::ImageInfo) -> std::result::Result,
- F2: Fn() -> std::result::Result<(Vec, AcquirePerf), ServerError>,
- A: std::fmt::Debug
-{
- //TODO: support process=, cache=, etc? pass-through by default?
- match execute_using(bytes_provider, framewise_generator) {
- Ok((output, perf)) => {
- let mime = output.mime_type
- .parse::()
- .unwrap_or_else(|_| Mime::from_str("application/octet-stream").unwrap());
- let mut res = Response::with((mime, status::Ok, output.bytes));
-
-
-
- res.headers.set_raw("X-Imageflow-Perf", vec![perf.short().into_bytes()]);
- Ok(res)
- }
- Err(e) => respond_with_server_error(&debug_info, e, true)
- }
-}
-
-fn respond_with_server_error(debug_info: &A, e: ServerError, detailed_errors: bool) -> IronResult where A: std::fmt::Debug {
- match e {
- ServerError::UpstreamResponseError(reqwest::StatusCode::NOT_FOUND) => {
- let bytes = if detailed_errors {
- b"Remote file not found (upstream server responded with 404)".to_vec()
- }else {
- format!("Remote file not found (upstream server responded with 404 to {:?})", debug_info).into_bytes()
- };
-
- Ok(Response::with((Mime::from_str("text/plain").unwrap(),
- status::NotFound,
- bytes)))
- },
- e => {
- let bytes = if detailed_errors {
- format!("Internal Server Error\nInfo:{:?}\nError:{:?}", debug_info, e).into_bytes()
- }else{
- b"Internal Server Error".to_vec()
- };
- Ok(Response::with((Mime::from_str("text/plain").unwrap(),
- status::InternalServerError,
- bytes)))
- // TODO: get a bit more specific with the error codes
- }
- }
-}
-
-
-fn ir4_http_respond(shared: &SharedData, url: &str, framewise_generator: F) -> IronResult
- where F: Fn(s::ImageInfo) -> std::result::Result
-{
- respond_using(&url, || fetch_bytes_using_cache_by_url(&shared.source_cache, url).map_err(error_upstream), framewise_generator)
-}
-
-fn ir4_http_respond_uncached(_shared: &SharedData, url: &str, framewise_generator: F) -> IronResult
- where F: Fn(s::ImageInfo) -> std::result::Result
-{
- respond_using(&url, || {
- fetch_bytes( url, None).map_err(error_upstream).map(|r|
- (r.bytes, r.perf))
- }, framewise_generator)
-}
-
-
-fn ir4_framewise(_info: &s::ImageInfo, url: &url::Url) -> std::result::Result {
- let t = ::imageflow_riapi::ir4::Ir4Translate{
- i: ::imageflow_riapi::ir4::Ir4Command::Url(url.as_str().to_owned()),
- decode_id: Some(0),
- encode_id: Some(1),
- watermarks: None
- };
- t.translate().map_err( ServerError::LayoutSizingError).and_then(|r: ::imageflow_riapi::ir4::Ir4Result| Ok(s::Framewise::Steps(r.steps.unwrap())))
-}
-
-
-type EngineHandler = fn(req: &mut Request, engine_data: &T, mount: &MountLocation) -> IronResult;
-type EngineSetup = fn(mount: &MountLocation) -> Result<(T, EngineHandler), String>;
-
-
-fn ir4_local_respond(_: &SharedData, source: &Path, framewise_generator: F) -> IronResult
- where F: Fn(s::ImageInfo) -> std::result::Result
-{
- respond_using(&source, || fetch_bytes_from_disk(source), framewise_generator)
-}
-
-fn ir4_local_handler(req: &mut Request, local_path: &PathBuf, _: &MountLocation) -> IronResult {
- let requested_path = requested_path::RequestedPath::new(local_path, req);
-
- let url: url::Url = req.url.clone().into();
- let shared = req.get::>().unwrap();
-
- if requested_path.path.exists() {
- return ir4_local_respond(&shared, requested_path.path.as_path(), move |info: s::ImageInfo| {
- ir4_framewise(&info, &url)
- });
- }
-
- let _ = writeln!(&mut std::io::stderr(), "404 {:?} using local path {:?} and base {:?}", &url.path(), requested_path.path.as_path(), local_path);
- //writeln!(&mut std::io::stdout(), "404 {:?} using local path {:?}", &url.path(), original );
-
- Ok(Response::with((Mime::from_str("text/plain").unwrap(),
- status::NotFound,
- b"File not found".to_vec())))
-}
-
-fn static_handler(_: &mut Request, _: &Static, _: &MountLocation) -> IronResult {
-
- Ok(Response::with((Mime::from_str("text/plain").unwrap(),
- status::InternalServerError,
- b"Do not use".to_vec())))
-}
-
-fn ir4_local_setup(mount: &MountLocation) -> Result<(PathBuf, EngineHandler), String> {
- if mount.engine_args.len() < 1 {
- Err("ir4_local requires at least one argument - the path to the physical folder it is serving".to_owned())
- } else {
- //TODO: validate path
- let local_dir = Path::new(&mount.engine_args[0]).canonicalize().map_err(|e| format!("{:?} for {:?}", e, &mount.engine_args[0]))?;
- Ok((local_dir, ir4_local_handler))
- }
-}
-
-fn static_setup(mount: &MountLocation) -> Result<(Static, EngineHandler), String> {
- if mount.engine_args.len() < 1 {
- Err("static requires at least one argument - the path to the physical folder it is serving".to_owned())
- } else {
- //TODO: validate path
- let path = Path::new(&mount.engine_args[0]).canonicalize().map_err(|e| format!("{:?}", e))?;
- let h = if mount.engine_args.len() > 1 {
- panic!("Static file cache headers not yet supported") //(we must compile staticfile with the 'cache' feature enabled)
-// let mins = mount.engine_args[1].parse::().expect("second argument to static must be the number of minutes to browser cache for");
-// Static::new(path).cache(Duration::minutes(mins))
- } else {
- Static::new(path)
- };
- Ok((h, static_handler))
- }
-}
-
-//Function is passed as generic trait (generic over 2nd arg), thus &String
-#[cfg_attr(feature = "cargo-clippy", allow(ptr_arg))]
-fn permacache_proxy_handler(req: &mut Request, base_url: &String, _: &MountLocation) -> IronResult {
- let url: url::Url = req.url.clone().into();
- let shared = req.get::>().unwrap();
- //TODO: Ensure the combined url is canonical (or, at least, lacks ..)
- let remote_url = format!("{}{}{}", base_url, &url.path()[1..], req.url.query().unwrap_or(""));
-
- match fetch_response_using_cache_by_url(&shared.source_cache, &remote_url) {
- Ok((output, _)) => {
- let mime = output.content_type
- .parse::()
- .unwrap_or_else(|_|Mime::from_str("application/octet-stream").unwrap());
-
- Ok(Response::with((mime, status::Ok, output.bytes)))
- }
- Err(e) => respond_with_server_error(&remote_url, e, true)
- }
-}
-lazy_static! {
- static ref MIME_TYPES: mime_types::Types = mime_types::Types::new().unwrap();
-}
-
-//Function is passed as generic trait (generic over 2nd arg), thus &String
-#[cfg_attr(feature = "cargo-clippy", allow(ptr_arg))]
-fn permacache_proxy_handler_guess_types(req: &mut Request, base_url: &String, _: &MountLocation) -> IronResult {
-
- let url: url::Url = req.url.clone().into();
-
- let shared = req.get::>().unwrap();
- //TODO: Ensure the combined url is canonical (or, at least, lacks ..)
- let remote_url = format!("{}{}{}", base_url, &url.path()[1..], req.url.query().unwrap_or(""));
- match fetch_bytes_using_cache_by_url(&shared.source_cache, &remote_url) {
- Ok((bytes, _)) => {
-
- let part_path = Path::new(&url.path()[1..]);
- let mime_str = MIME_TYPES.mime_for_path(part_path);
- let mime:Mime = mime_str.parse().unwrap();
-
-// let mime = output.content_type
-// .parse::()
-// .unwrap_or(Mime::from_str("application/octet-stream").unwrap());
-
- Ok(Response::with((mime, status::Ok, bytes)))
- }
- Err(e) => respond_with_server_error(&remote_url, e, true)
- }
-}
-
-//Function is passed as generic trait (generic over 2nd arg), thus &String
-#[cfg_attr(feature = "cargo-clippy", allow(ptr_arg))]
-fn ir4_http_handler(req: &mut Request, base_url: &String, _: &MountLocation) -> IronResult {
- let url: url::Url = req.url.clone().into();
- let shared = req.get::>().unwrap();
- //TODO: Ensure the combined url is canonical (or, at least, lacks ..)
- let remote_url = format!("{}{}", base_url, &url.path()[1..]);
-
- ir4_http_respond(&shared, &remote_url, move |info: s::ImageInfo| {
- ir4_framewise(&info, &url)
- })
-}
-
-#[cfg_attr(feature = "cargo-clippy", allow(ptr_arg))]
-fn ir4_proxy_uncached_handler(req: &mut Request, base_url: &String, _: &MountLocation) -> IronResult {
- let url: url::Url = req.url.clone().into();
- let shared = req.get::>().unwrap();
- //TODO: Ensure the combined url is canonical (or, at least, lacks ..)
- let remote_url = format!("{}{}", base_url, &url.path()[1..]);
-
- ir4_http_respond_uncached(&shared, &remote_url, move |info: s::ImageInfo| {
- ir4_framewise(&info, &url)
- })
-}
-
-fn ir4_http_setup(mount: &MountLocation) -> Result<(String, EngineHandler), String> {
- if mount.engine_args.len() < 1 {
- Err("ir4_http requires at least one argument - the base url to suffix paths to".to_owned())
- } else {
- Ok((mount.engine_args[0].to_owned(), ir4_http_handler))
- }
-}
-
-fn ir4_http_uncached_setup(mount: &MountLocation) -> Result<(String, EngineHandler), String> {
- if mount.engine_args.len() < 1 {
- Err("ir4_proxy_uncached requires at least one argument - the base url to suffix paths to".to_owned())
- } else {
- Ok((mount.engine_args[0].to_owned(), ir4_proxy_uncached_handler))
- }
-}
-
-fn permacache_proxy_setup(mount: &MountLocation) -> Result<(String, EngineHandler), String> {
- if mount.engine_args.len() < 1 {
- Err("permacache_proxy requires at least one argument - the base url to suffix paths to".to_owned())
- } else {
- Ok((mount.engine_args[0].to_owned(), permacache_proxy_handler))
- }
-}
-fn permacache_proxy_guess_content_types_setup(mount: &MountLocation) -> Result<(String, EngineHandler), String> {
- if mount.engine_args.len() < 1 {
- Err("permacache_proxy_guess_content_types requires at least one argument - the base url to suffix paths to".to_owned())
- } else {
- Ok((mount.engine_args[0].to_owned(), permacache_proxy_handler_guess_types))
- }
-}
-
-fn mount(mount: MountLocation, mou: &mut mount::Mount, setup: EngineSetup) -> Result<(), String>
- where T: Send, T: Sync, T: 'static {
- let (data, handler) = setup(&mount)?;
-
- let prefix = mount.prefix.clone();
- mou.mount(&prefix, move |r: &mut Request| { handler(r, &data, &mount) });
- Ok(())
-}
-
-
-pub fn serve(c: StartServerConfig) {
- env_logger::init();
-
- let shared_data = SharedData {
- source_cache: CacheFolder::new(c.data_dir.join(Path::new("source_cache")).as_path(), c.default_cache_layout.unwrap_or(FolderLayout::Normal)),
- output_cache: CacheFolder::new(c.data_dir.join(Path::new("output_cache")).as_path(), c.default_cache_layout.unwrap_or(FolderLayout::Normal)),
- requests_received: AtomicUsize::new(0) //NOT YET USED
- };
-
- let mut mou = mount::Mount::new();
- let mut router = Router::new();
-
- // Mount prefix (external) (url|relative path)
- // pass through static files (whitelisted??)
-
- for m in c.mounts {
- let copy = m.clone();
- let mount_result = match m.engine {
- //MountedEngine::Ir4Https => "ir4_https",
- MountedEngine::Ir4Http => mount(m, &mut mou, ir4_http_setup),
- MountedEngine::Ir4ProxyUncached => mount(m, &mut mou, ir4_http_uncached_setup),
- MountedEngine::Ir4Local => mount(m, &mut mou, ir4_local_setup),
- MountedEngine::PermacacheProxy => mount(m, &mut mou, permacache_proxy_setup),
- MountedEngine::PermacacheProxyGuessContentTypes => mount(m, &mut mou, permacache_proxy_guess_content_types_setup),
- MountedEngine::Static => {
- mou.mount(&m.prefix, static_setup(&m).expect("Failed to mount static directory").0);
- Ok(())
- },
- };
- if let Err(e) = mount_result{
-
- panic!("Failed to mount {} using engine {} ({:?})\n({:?})\nCurrent dir: {:?}", ©.prefix, ©.engine.to_id(), ©, e, std::env::current_dir())
- }
- }
-
- if c.integration_test {
- router.get("/test/shutdown", move |_: &mut Request| -> IronResult {
- println!("Stopping server due to GET /test/shutdown");
- std::process::exit(0);
-
- // Ok(Response::with((Mime::from_str("text/plain").unwrap(),
- // status::InternalServerError,
- // bytes)))
- }, "test-shutdown");
- }
-
- router.get("/imageflow.ready", move |_: &mut Request| -> IronResult {
- Ok(Response::with((Mime::from_str("text/plain").unwrap(),
- status::Ok,
- "Imageflow Server is ready to start accepting requests.")))
- }, "imageflow-ready");
-
- router.get("/imageflow.health", move |_: &mut Request| -> IronResult {
- Ok(Response::with((Mime::from_str("text/plain").unwrap(),
- status::Ok,
- "Imageflow Server is healthy.")))
- }, "imageflow-health");
-
- mou.mount("/", router);
-
- let mut chain = Chain::new(mou);
-
- chain.link(persistent::Read::::both(shared_data));
-
- let (logger_before, logger_after) = Logger::new(None);
-
- // Link logger_before as your first before middleware.
- chain.link_before(logger_before);
-
- // Link logger_after as your *last* after middleware.
- chain.link_after(logger_after);
-
- //let ssl = NativeTlsServer::new("identity.p12", "mypass").unwrap();
-
-
- println!("Listening on {}", c.bind_addr.as_str());
- if c.cert.is_some() {
- let pwd = c.cert_pwd.unwrap_or_default();
- let ssl = NativeTlsServer::new(c.cert.unwrap(), &pwd).unwrap();
-
- Iron::new(chain).https(c.bind_addr.as_str(), ssl).unwrap();
- }else{
- Iron::new(chain).http(c.bind_addr.as_str()).unwrap();
- }
-}
-
-
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-pub enum MountedEngine {
- Ir4Local,
- Ir4Http,
- Ir4ProxyUncached,
- PermacacheProxy,
- PermacacheProxyGuessContentTypes,
- Static,
- //Ir4Https
-}
-
-impl MountedEngine {
- pub fn to_id(&self) -> &'static str {
- match *self {
- //MountedEngine::Ir4Https => "ir4_https",
- MountedEngine::Ir4Http => "ir4_http",
- MountedEngine::Ir4ProxyUncached => "ir4_proxy_uncached",
- MountedEngine::Ir4Local => "ir4_local",
- MountedEngine::PermacacheProxy => "permacache_proxy",
- MountedEngine::PermacacheProxyGuessContentTypes => "permacache_proxy_guess_content_types",
- MountedEngine::Static => "static"
- }
- }
- pub fn from_id(s: &str) -> Option {
- match s {
- "ir4_local" => Some(MountedEngine::Ir4Local),
- "ir4_http" => Some(MountedEngine::Ir4Http),
- "ir4_proxy_uncached" => Some(MountedEngine::Ir4ProxyUncached),
- "permacache_proxy" => Some(MountedEngine::PermacacheProxy),
- "permacache_proxy_guess_content_types" => Some(MountedEngine::PermacacheProxyGuessContentTypes),
- "static" => Some(MountedEngine::Static),
- //"ir4_https" => Some(MountedEngine::Ir4Https),
- _ => None
- }
- }
-
- pub fn id_values() -> &'static [&'static str] {
- static ID_VALUES: [&'static str; 5] = ["ir4_local", "ir4_http", "permacache_proxy", "static", "permacache_proxy_guess_content_types"/* "ir4_https"*/];
-
- &ID_VALUES
- }
-}
-
-trait Engine {
- fn mount(self, mount: MountLocation, router: &mut Router) -> Result<(), String>;
-}
-
-#[derive(Debug, Clone, PartialEq)]
-pub struct MountLocation {
- pub prefix: String,
- pub engine: MountedEngine,
- pub engine_args: Vec,
- //TODO: HTTPS
-}
-
-impl MountLocation {
- pub fn parse(prefix: String, engine_name: String, args: Vec) -> std::result::Result {
- lazy_static! {
- static ref RE: Regex = Regex::new(r"\A(/[a-zA-Z0-9-_]+?)+?/\z").unwrap();
- }
- if !RE.is_match(&prefix) {
- return Err("mount points must be valid paths with leading and trailing slashes, like /img/logos/. Between slashes, [a-zA-Z0-9-_] may be used".to_owned());
- }
- let engine = MountedEngine::from_id(engine_name.as_str());
-
- if engine.is_none() {
- return Err(format!("Valid engine names include {:?}. Provided {}", MountedEngine::id_values(), engine_name.as_str()));
- }
-
- Ok(MountLocation {
- prefix: prefix,
- engine: engine.unwrap(),
- engine_args: args
- })
- }
-}
-
-
-#[derive(Debug, Clone, PartialEq)]
-pub struct StartServerConfig {
- pub data_dir: PathBuf,
- pub bind_addr: String,
- pub mounts: Vec,
- pub default_cache_layout: Option,
- pub integration_test: bool,
- pub cert: Option,
- pub cert_pwd: Option
-}
-
-
-#[test]
-fn test_file_macro_for_this_build(){
- assert!(file!().starts_with(env!("CARGO_PKG_NAME")))
-}
diff --git a/imageflow_server/src/main.rs b/imageflow_server/src/main.rs
deleted file mode 100644
index ab68a7694..000000000
--- a/imageflow_server/src/main.rs
+++ /dev/null
@@ -1,200 +0,0 @@
-extern crate clap;
-extern crate imageflow_server;
-
-
-use clap::{App, Arg, SubCommand, AppSettings};
-use imageflow_server::preludes::*;
-use std::path::{Path, PathBuf};
-
-use std::net::ToSocketAddrs;
-
-extern crate imageflow_types as s;
-
-fn main() {
- let exit_code = main_with_exit_code();
- std::process::exit(exit_code);
-}
-
-
-fn parse_mount(s: &str) -> std::result::Result{
- //Escape ::
- let mut parts = s.replace("::","||||||").split(':').map(|s| s.replace("||||||",":")).collect::>();
- if parts.len() < 2 {
- Err(format!("--mount prefix:engine:args Mount value must contain at least prefix:engine - received {:?} ({:?})", s, &parts))
- }else{
- MountLocation::parse(parts.remove(0), parts.remove(0), parts)
- }
-}
-
-fn main_with_exit_code() -> i32 {
- let version = s::version::one_line_version();
- let app = App::new("imageflow_server").version(version.as_ref())
- .setting(AppSettings::VersionlessSubcommands).setting(AppSettings::SubcommandRequiredElseHelp)
- .subcommand(
- SubCommand::with_name("diagnose").setting(AppSettings::ArgRequiredElseHelp)
- .about("Diagnostic utilities")
- .arg(
- Arg::with_name("show-compilation-info").long("show-compilation-info")
- .help("Show all the information stored in this executable about the environment in which it was compiled.")
- ).arg(
- Arg::with_name("call-panic").long("call-panic")
- .help("Triggers a Rust panic (so you can observe failure/backtrace behavior)")
- ).arg(
- Arg::with_name("smoke-test-core").long("smoke-test-core")
- .help("Smoke test a few tiny image processing operations"))
- )
- .subcommand(
- SubCommand::with_name("start")
- .about("Start HTTP server").setting(AppSettings::ArgRequiredElseHelp)
- .arg(Arg::with_name("demo").long("demo").conflicts_with("mount").required_unless("mount")
- .help("Start demo server (on localhost:39876 by default) with mounts /ir4/proxy/unsplash -> http://images.unsplash.com/"))
- .arg(
- Arg::with_name("mount").long("mount").takes_value(true).empty_values(false).multiple(true).required_unless("demo")
- .validator(|f| parse_mount(&f).map(|_| ()))
- .help("Serve images from the given location using the provided API, e.g --mount \"/prefix/:ir4_local:./{}\" --mount \"/extern/:ir4_http:http:://domain.com/\" --mount \"/extern/:ir4_proxy_uncached:http:://domain.com/\"\n Escape colons by doubling, e.g. http:// -> http:://")
- )
- .arg(Arg::with_name("bind-address").long("bind-address").takes_value(true).required(false).default_value("localhost")
- .help("The IPv4 or IPv6 address to bind to (or the hostname, like localhost). 0.0.0.0 binds to all addresses."
- ))
- .arg(Arg::with_name("port").long("port").short("-p").takes_value(true).default_value("39876").required(false).help("Set the port that the server will listen on"))
- .arg(Arg::with_name("cert").long("certificate").takes_value(true).required(false).help("Path to a valid PKCS12 certificate (enables https)"))
- .arg(Arg::with_name("cert-pwd").long("certificate-password").takes_value(true).required(false).help("Password to the PKCS12 certificate"))
-
-
- .arg(Arg::with_name("data-dir").long("data-dir").takes_value(true).required_unless("demo")
- .validator(|f| if Path::new(&f).is_dir() { Ok(()) } else { Err(format!("The specified data-dir {} must be an existing directory. ", f)) })
- .help("An existing directory for logging and caching"))
- .arg(Arg::with_name("integration-test").long("integration-test").hidden(true).help("Never use this outside of an integration test. Exposes an HTTP endpoint to kill the server."))
-
-
- );
-
-
-
- let matches = app.get_matches();
-
- if let Some(matches) = matches.subcommand_matches("diagnose") {
- let m: &clap::ArgMatches = matches;
-
- if m.is_present("show-compilation-info") {
- println!("{}\n{}\n",
- s::version::one_line_version(),
- s::version::all_build_info_pairs());
- return 0;
- }
- if m.is_present("call-panic") {
- panic!("Panicking on command");
- }
- if m.is_present("smoke-test-core") {
- ::imageflow_server::diagnose::smoke_test_core();
- return 0;
- }
- }
- if let Some(matches) = matches.subcommand_matches("start") {
- let m: &clap::ArgMatches = matches;
-
-
- let port = matches.value_of("port").map(|s| s.parse::().expect("Port must be a valid 16-bit positive integer") ).unwrap_or(39_876);
- let integration_test = matches.is_present("integration-test");
- let data_dir = m.value_of("data-dir").map(PathBuf::from);
- let cert = m.value_of("cert").map(PathBuf::from);
- if let Some(ref p) = cert{
- if !p.is_file(){
- println!("The provided certificate file does not exist: {:?}", &cert);
- std::process::exit(64);
- }
- }
- let bind = m.value_of("bind-address").map(|s| s.to_owned()).expect("bind address required");
-
- let combined = format!("{}:{}", bind, port);
-
- {
- let socket_addr_iter = combined.to_socket_addrs();
- if socket_addr_iter.is_err() || socket_addr_iter.unwrap().next().is_none() {
- println!("Invalid value for --bind-address. {} failed to parse.", &combined);
- std::process::exit(64);
- }
- }
-
- if m.is_present("demo"){
- //TODO: fetch an examples directory, with javascript/html/css and images, and mount that
-
-
- // If not provided, ./imageflow_data is created and used
-
- let alt_data_dir = Path::new(".").join("imageflow_data");
-
-
- let demo_commit = s::version::get_build_env_value("GIT_COMMIT").unwrap();
-
- let mut mounts = vec![
- MountLocation {
- engine: MountedEngine::Ir4Http,
- prefix: "/ir4/proxy_unsplash/".to_owned(),
- engine_args: vec!["http://images.unsplash.com/".to_owned()]
- },
- MountLocation {
- engine: MountedEngine::PermacacheProxyGuessContentTypes,
- prefix: "/proxied_demo/".to_owned(),
- engine_args: vec![format!("https://raw.githubusercontent.com/imazen/imageflow/{}/imageflow_server/demo/", demo_commit)]
- },
- MountLocation {
- engine: MountedEngine::Ir4Http,
- prefix: "/demo_images/".to_owned(),
- engine_args: vec!["http://resizer-images.s3.amazonaws.com/".to_owned()]
- },
- MountLocation {
- engine: MountedEngine::Ir4Http,
- prefix: "/website_images/".to_owned(),
- engine_args: vec!["http://resizer-web.s3.amazonaws.com/".to_owned()]
- },
- MountLocation {
- engine: MountedEngine::Ir4ProxyUncached,
- prefix: "/demo_images_uncached/".to_owned(),
- engine_args: vec!["http://resizer-images.s3.amazonaws.com/".to_owned()]
- }
- ];
- let local_demo_folder = Path::new(env!("CARGO_MANIFEST_DIR")).join("demo");
- if local_demo_folder.exists() {
- mounts.push(MountLocation {
- engine: MountedEngine::Static,
- prefix: "/src_demo/".to_owned(),
- engine_args: vec![local_demo_folder.as_path().to_str().unwrap().to_owned()]
- });
-
- println!("Open your browser to http://{}/src_demo/index.html", &combined);
- }else{
- println!("Open your browser to http://{}/proxied_demo/index.html", &combined);
-
- }
-
- println!("{}",&version);
- ::imageflow_server::serve(StartServerConfig {
- bind_addr: combined,
- data_dir: data_dir.unwrap_or_else(|| { if !alt_data_dir.exists() { std::fs::create_dir_all(&alt_data_dir).unwrap(); } alt_data_dir }),
- default_cache_layout: Some(FolderLayout::Tiny),
- integration_test: integration_test,
- mounts: mounts,
- cert: cert,
- cert_pwd: m.value_of("cert-pwd").map(|s| s.into()),
- });
- }else {
- let mounts = m.values_of_lossy("mount").expect("at least one --mount required").into_iter().map(|s| parse_mount(&s).expect("validator not working - bug in clap?")).collect::>();
-
- println!("{}",&version);
- ::imageflow_server::serve(StartServerConfig {
- bind_addr: combined,
- data_dir: data_dir.expect("data-dir required"),
- mounts: mounts,
- default_cache_layout: Some(FolderLayout::Normal),
- integration_test: integration_test,
- cert: cert,
- cert_pwd: m.value_of("cert-pwd").map(|s| s.into()),
- });
- }
- return 0;
- }
-
- 64
-}
-
diff --git a/imageflow_server/src/mem_cache.rs b/imageflow_server/src/mem_cache.rs
deleted file mode 100644
index e69de29bb..000000000
diff --git a/imageflow_server/src/requested_path.rs b/imageflow_server/src/requested_path.rs
deleted file mode 100644
index 56a0b106f..000000000
--- a/imageflow_server/src/requested_path.rs
+++ /dev/null
@@ -1,63 +0,0 @@
-/* The MIT License (MIT)
-
-Copyright (c) 2014 iron
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE. */
-
-
-use iron::Request;
-use std::iter::FromIterator;
-use std::path::{Component, PathBuf, Path};
-use std::convert::AsRef;
-use url::percent_encoding::percent_decode;
-
-pub struct RequestedPath {
- pub path: PathBuf,
-}
-
-#[inline]
-fn decode_percents(string: &&str) -> String {
- percent_decode(string.as_bytes()).decode_utf8().unwrap().into_owned()
-}
-
-fn normalize_path(path: &Path) -> PathBuf {
- path.components().fold(PathBuf::new(), |mut result, p| {
- match p {
- Component::Normal(x) => {
- result.push(x);
- result
- }
- Component::ParentDir => {
- result.pop();
- result
- },
- _ => result
- }
- })
-}
-
-impl RequestedPath {
- pub fn new>(root_path: P, request: &Request) -> RequestedPath {
- let decoded_req_path = PathBuf::from_iter(request.url.path().iter().map(decode_percents));
- let mut result = root_path.as_ref().to_path_buf();
- result.extend(&normalize_path(&decoded_req_path));
- RequestedPath { path: result }
- }
-
-}
diff --git a/imageflow_server/src/resizer/mod.rs b/imageflow_server/src/resizer/mod.rs
deleted file mode 100644
index e69de29bb..000000000
diff --git a/imageflow_server/tests/test_ir4.rs b/imageflow_server/tests/test_ir4.rs
deleted file mode 100644
index b5ce4cf26..000000000
--- a/imageflow_server/tests/test_ir4.rs
+++ /dev/null
@@ -1,458 +0,0 @@
-
-extern crate imageflow_helpers;
-extern crate imageflow_core as fc;
-extern crate imageflow_types as s;
-use imageflow_helpers::preludes::from_std::*;
-extern crate hyper;
-
-
-extern crate wait_timeout;
-use wait_timeout::ChildExt;
-use std::time::Duration;
-use std::process::{Command, Stdio, Output};
-use std::net::{TcpListener};
-
-#[macro_use]
-extern crate lazy_static;
-
-use std::sync::Mutex;
-
-use imageflow_helpers::process_testing::*;
-use crate::fc::test_helpers::process_testing::ProcTestContextExtras;
-use ::imageflow_http_helpers::{fetch, fetch_bytes,get_status_code_for, FetchError, FetchConfig};
-
-use std::collections::vec_deque::VecDeque;
-use reqwest::StatusCode;
-
-lazy_static! {
- static ref RECENT_PORTS: Mutex> = Mutex::new(VecDeque::new());
-}
-
-fn assert_valid_image(url: &str) {
- match fetch(url, Some(FetchConfig{ custom_ca_trust_file: None, read_error_body: Some(true)})){
- Ok(v) => {
- if !v.code.is_success(){
- panic!("Error {:?} for {}", v.code, &url);
- }
- fc::clients::stateless::LibClient {}.get_image_info(&v.bytes).expect("Image response should be valid");
- },
- Err(e) => { panic!("{:?} for {}", &e, &url); }
- }
-}
-
-fn assert_ok(url: &str) {
- match fetch(url, Some(FetchConfig{ custom_ca_trust_file: None, read_error_body: Some(true)})){
- Ok(response) => {
- if !response.code.is_success(){
- panic!("Error {:?} for {}", response.code, &url);
- }
- },
- Err(e) => { panic!("{:?} for {}", &e, &url); }
- }
-}
-//fn write_env_vars(path: &Path){
-// let mut f = File::create(&path).unwrap();
-// for (k,v) in std::env::vars(){
-// write!(f, "{}={}\n", k, v).unwrap();
-// }
-//}
-
-fn build_dirs() -> Vec{
- let target_triple = crate::s::version::get_build_env_value("TARGET").expect("TARGET triple required");
- let profile = crate::s::version::get_build_env_value("PROFILE").expect("PROFILE (debug/release) required");
-
-
- let target_dir = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap().join("target");
-
- let a = target_dir.join(target_triple).join(profile);
- let b = target_dir.join(profile);
- vec![a,b]
-}
-#[cfg(windows)]
-fn binary_ext() -> &'static str{
- "exe"
-}
-#[cfg(not(windows))]
-fn binary_ext() -> &'static str{
- ""
-}
-
-fn locate_binary(name: &str) -> Option {
- for dir in build_dirs() {
- let file_path = dir.join(name).with_extension(binary_ext());
-
- if file_path.exists() {
- return Some(dir.join(name))
- }
- }
- None
-}
-
-
-fn server_path() -> PathBuf {
- match locate_binary("imageflow_server"){
- Some(v) => v,
- None => {
- panic!("Failed to locate imageflow_server binary in {:?}", build_dirs());
- }
- }
-}
-
-fn enqueue_unique_port(q: &mut VecDeque, count: usize) -> u16{
- let listener = TcpListener::bind("127.0.0.1:0").unwrap();
- let port = listener.local_addr().unwrap().port();
- if !q.contains(&port) {
- q.push_back(port);
- if count <= 1{
- port
- }else {
- enqueue_unique_port(q, count - 1)
- }
- } else {
- enqueue_unique_port(q, count)
- }
-}
-
-fn fetch_next_port(q: &mut VecDeque) -> u16 {
- if q.len() < 1 {
- let _ = enqueue_unique_port(q, 25); // pre-check ports 25 at a time.
- }
- q.pop_front().unwrap()
-}
-
-fn get_next_port() -> u16 {
- let mut q = RECENT_PORTS.lock().unwrap();
- fetch_next_port(&mut q)
-}
-
-
-struct ServerInstance{
- port: u16,
- protocol: Proto,
- #[allow(dead_code)]
- trust_ca_file: Option,
- #[allow(dead_code)]
- cert: Option
-
-}
-
-type CallbackResult = std::result::Result<(), ::imageflow_http_helpers::FetchError>;
-
-#[derive(Debug,PartialEq,Eq,Copy,Clone)]
-enum Proto{
- Http,
- Https
-}
-impl ServerInstance{
-
- fn hello(&self) -> Result {
- get_status_code_for(&self.url_for("/hello/are/you/running?"))
- }
-
- fn url_for(&self, rel_path: &str) -> String{
- if self.protocol == Proto::Https{
- format!("https://localhost:{}{}", self.port,rel_path)
- }else{
- format!("http://localhost:{}{}", self.port,rel_path)
- }
-
- }
-
- fn get_status(&self, rel_path: &str) -> Result {
- get_status_code_for(&self.url_for(rel_path))
- }
-
- fn request_stop(&self) -> Result {
- get_status_code_for(&self.url_for("/test/shutdown"))
- }
-
- fn run(c: &ProcTestContext, protocol: Proto, args: Vec<&str>, callback: F) -> (ProcOutput, CallbackResult)
- where F: Fn(&ServerInstance) -> CallbackResult {
- let assets = Path::new(env!("CARGO_MANIFEST_DIR")).join(Path::new("src")).join(Path::new("assets"));
- let cert_path = assets.join(Path::new("identity.p12"));
- let ca_path = assets.join(Path::new("root-ca.pem"));
-
-
- let instance = ServerInstance {
- port: get_next_port(),
- protocol,
- trust_ca_file: Some(ca_path),
- cert: Some(cert_path.clone())
- };
- // NOTE --bind=localhost::{} (two colons) causes a generic "error:",exit code 1, and no other output. This is bad UX.
- let test_arg = "--integration-test";
- let port_arg = format!("--port={}", instance.port);
-
- let mut all_args = args.clone();
- if protocol == Proto::Https{
- all_args.insert(0, "mypass");
- all_args.insert(0, "--certificate-password");
- all_args.insert(0, cert_path.to_str().unwrap());
- all_args.insert(0, "--certificate");
- }
- all_args.insert(0, test_arg);
- all_args.insert(0, &port_arg);
- all_args.insert(0, "start");
-
- c.execute_callback(all_args, false,
- |_child: &mut std::process::Child| -> std::result::Result<(), ::imageflow_http_helpers::FetchError> {
-
- ::std::thread::sleep(::std::time::Duration::from_millis(500));
- // Server may not be running
- instance.hello()?;
-
- let r = callback(&instance);
-
- let _ = instance.request_stop();
- r
- })
- //po.expect_status_code(Some(0));
- }
-}
-
-
-// ports 36,000 to 39,999 seem the safest.
-#[test]
-fn run_server_test_i4(){
-
- //write_env_vars(&Path::new("env.txt"));
-
- let context = ProcTestContext::create_timestamp_subdir_within(std::env::current_dir().unwrap().join("server_tests"), Some(server_path()));
-
- {
- let c = context.subfolder_context("basics");
- c.exec("diagnose --show-compilation-info").expect_status_code(Some(0));
- c.exec("--version").expect_status_code(Some(0));
- c.exec("-V").expect_status_code(Some(0));
-
- //TODO: test diagnose --call-panic (xplat hard)
-
- //Test incorrect args
- c.execute(vec!["demo"], false, |_child: &mut std::process::Child| {
- }).expect_status_code(Some(1));
-
- }
-
- {
- let c = context.subfolder_context("demo"); //stuck on port 39876
- c.subfolder_context("demo");
- let (_po, callback_result) = ServerInstance::run(&c, Proto::Http, vec!["--demo", "--data-dir=."], | server | {
- assert_ok(&server.url_for("/imageflow.ready"));
- assert_ok(&server.url_for("/imageflow.health"));
-
- fetch_bytes(&server.url_for("/ir4/proxy_unsplash/photo-1422493757035-1e5e03968f95?width=100"))?;
- //TODO: Find a way to test upstream 404 and 403 errors
- // assert_eq!(server.get_status("/demo_images/notthere.jpg")?, http::StatusCode::NOT_FOUND);
-
- let url = server.url_for("/proxied_demo/index.html");
- match fetch(&url, Some(FetchConfig{ custom_ca_trust_file: None, read_error_body: Some(true)})){
- Ok(_) => {},
- Err(e) => { panic!("{:?} for {}", &e, &url); }
- }
-
- assert_valid_image(&server.url_for("/demo_images/example-028-whitespace.jpg?width=600&trim.threshold=80&trim.percentpadding=0.5"));
-
-
- Ok(())
- });
-
- //po.expect_status_code(Some(0));
-
- callback_result.unwrap();
- }
- {
- let c = context.subfolder_context("proxy");
- c.subfolder_context("proxy");
- let (_po, callback_result) = ServerInstance::run(&c, Proto::Http, vec!["--data-dir=.", "--mount","/extern/:ir4_http:http:://images.unsplash.com/"], | server | {
- fetch_bytes(&server.url_for("/extern/photo-1422493757035-1e5e03968f95?width=100"))?;
- Ok(())
- });
-
- //po.expect_status_code(Some(0));
-
- callback_result.unwrap();
- }
- {
- let c = context.subfolder_context("mount_local"); //stuck on port 39876
- c.create_blank_image_here("eh", 100,100, s::EncoderPreset::libpng32());
- let a = c.subfolder_context("a"); //stuck on port 39876
- a.create_blank_image_here("eh2", 100,100, s::EncoderPreset::libpng32());
-
- let mut params = vec!["--data-dir=.", "--mount=/local/:ir4_local:./",
- "--mount=/local_1/:ir4_local:./a",
- "--mount=/local_2/:ir4_local:./a/",
- "--mount=/local_3/:ir4_local:a"];
- if std::path::MAIN_SEPARATOR == '\\'{
- params.push(r"--mount=/local_4/:ir4_local:.\a");
- params.push(r"--mount=/local_5/:ir4_local:.\a/");
- params.push(r"--mount=/local_6/:ir4_local:.\a\");
- }
-
- let last_mount = params.len() - 2;
-
- let (_, callback_result) = ServerInstance::run(&c, Proto::Http, params , | server | {
- assert_valid_image(&server.url_for("/local/eh.png?width=100"));
-
- for ix in 1..last_mount + 1{
- let url = format!("/local_{ix}/eh2.png?w=1", ix=ix);
- println!("Testing {}", &url);
- assert_valid_image(&server.url_for(&url));
- }
-
-
- assert_eq!(server.get_status("/local/notthere.jpg")?, StatusCode::NOT_FOUND);
- assert_eq!(server.get_status("/notrouted")?, StatusCode::NOT_FOUND);
- Ok(())
- });
- //po.expect_status_code(Some(0));
-
- callback_result.unwrap();
- }
-
- // we can't currently test https server support. We *should* be able to, on linux - but ... nope.
- //test_https(context);
-}
-
-#[allow(dead_code)]
-#[cfg(not(any(target_os = "windows", target_os = "macos")))]
-fn test_https(context: &ProcTestContext){
- {
- let c = context.subfolder_context("https_demo"); //stuck on port 39876
- c.subfolder_context("demo");
- let (_, callback_result) = ServerInstance::run(&c, Proto::Https, vec!["--demo", "--data-dir=."], | server | {
- let url = server.url_for("/ir4/proxy_unsplash/photo-1422493757035-1e5e03968f95?width=100");
- let bytes = fetch(&url, Some(FetchConfig{custom_ca_trust_file: server.trust_ca_file.clone(), read_error_body: Some(true) })).expect(&url).bytes;
- let _ = fc::clients::stateless::LibClient {}.get_image_info(&bytes).expect("Image response should be valid");
-
- //assert_eq!(server.get_status("/ir4/proxy_unsplash/notthere.jpg")?, http::StatusCode::NOT_FOUND);
- Ok(())
- });
-
- //po.expect_status_code(Some(0));
-
- callback_result.unwrap();
- }
-}
-
-#[allow(dead_code)]
-#[cfg(any(target_os = "windows", target_os = "macos"))]
-fn test_https(_context: ProcTestContext){}
-
-#[test]
-fn run_server_test_ir4_heavy(){
- let context = ProcTestContext::create_timestamp_subdir_within(std::env::current_dir().unwrap().join("server_tests_heavy"), Some(server_path()));
- {
- let c = context.subfolder_context("mount_local_test"); //stuck on port 39876
- c.exec("diagnose --show-compilation-info").expect_status_code(Some(0));
- c.create_blank_image_here("eh", 100,100, s::EncoderPreset::libpng32());
-
- let params = vec!["--data-dir=.", "--mount=/local/:ir4_local:./"];
- let (_, callback_result) = ServerInstance::run(&c, Proto::Http, params , | server | {
- for _ in 1..20{
- assert_valid_image(&server.url_for("/local/eh.png?width=100"));
- }
- Ok(())
- });
- callback_result.unwrap();
- }
-}
-
-trait ProcTestContextHttp{
- fn execute_callback(&self, args_vec: Vec<&str>, valgrind_on_signal_death: bool, callback: F) -> (ProcOutput, T)
-where F: Fn(&mut std::process::Child) -> T;
-
-}
-impl ProcTestContextHttp for ProcTestContext{
-
- ///
- /// Pass false for valgrind_on_signal_death if your callback might kill the child
- fn execute_callback(&self, args_vec: Vec<&str>, valgrind_on_signal_death: bool, callback: F) -> (ProcOutput, T)
- where F: Fn(&mut std::process::Child) -> T {
- //TODO: serialize in a safer way - this isn't correct
- let full_invocation = format!("{} {}", &self.bin_location().to_str().unwrap(), args_vec.join(" "));
-
- let dir = self.working_dir();
- let exe = self.bin_location();
-
- let valgrind_copy_result = self.create_valgrind_suppressions();
- let _ = writeln!(&mut std::io::stderr(),
- "Executing from folder {} with valgrind_suppressions {:?}\n{}",
- dir.to_str().unwrap(),
- valgrind_copy_result,
- full_invocation);
- // change working dir to dir
- let mut cmd = Command::new(exe);
- cmd.args(args_vec.as_slice()).current_dir(dir).env("RUST_BACKTRACE", "1");
-
-
- //cmd.stderr(Stdio::piped()).stdout(Stdio::piped());
- cmd.stderr(Stdio::inherit()).stdout(Stdio::inherit());
-
-
- let mut child = cmd.spawn().expect("Failed to start?");
-
-
- let result = callback(&mut child);
-
-
- //child.kill().unwrap();
- let timeout = Some(Duration::from_secs(1));
-
- let (status_code, output) = match timeout {
- Some(timeout) => {
- match child.wait_timeout(timeout).unwrap() {
- Some(status) => (status.code(), None),
- None => {
- // child hasn't exited yet
- child.kill().unwrap();
- (child.wait().unwrap().code(), None)
- }
- }
- }
- None => {
- let output: Output = child.wait_with_output().unwrap();
- (output.status.code(), Some(output))
- }
- };
-
- let _ = writeln!(&mut std::io::stderr(),
- "exit code {:?}", status_code);
-
- // Double check we dumped output on segfault
- if status_code == None {
- if let Some(ref out) = output {
- std::io::stderr().write_all(&out.stderr).unwrap();
- std::io::stdout().write_all(&out.stdout).unwrap();
- }
- let _ = writeln!(&mut std::io::stderr(),
- "exit code {:?}", status_code);
- }
- // Killed by signal.
- // 11 Segmentation fault
- // 4 illegal instruction 6 abort 8 floating point error
- if status_code == None && valgrind_on_signal_death {
- if std::env::var("VALGRIND_RUNNING").is_ok() {
- let _ = writeln!(&mut std::io::stderr(),
- "VALGRIND_RUNNING defined; skipping valgrind pass");
- } else {
- //ALLOW TO FAIL; valgrind may not be present
- let _ = writeln!(&mut std::io::stderr(),
- "Starting valgrind from within self-test:");
- let mut cmd = Command::new("valgrind");
- cmd.arg("-q").arg("--error-exitcode=9").arg(exe);
- cmd.args(args_vec.as_slice()).current_dir(dir).env("RUST_BACKTRACE", "1").env("VALGRIND_RUNNING", "1");
-
- let _ = writeln!(&mut std::io::stderr(),
- "{:?}", cmd);
-
- let _ = cmd.status(); //.expect("Failed to start valgrind?");
- }
- }
-
- match output {
- Some(out) => (ProcOutput::from(out), result),
- None => (ProcOutput::from_code(status_code), result)
- }
- }
-
-}
diff --git a/imageflow_server/try.sh b/imageflow_server/try.sh
deleted file mode 100755
index a3b8e225f..000000000
--- a/imageflow_server/try.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-set -e
-
-
-firefox http://localhost:3000 & cargo run --bin imageflow_server start & wait