Migration to Pillow and huge performance improvements

.travis.yml

@@ -2,7 +2,7 @@ sudo: required
 dist: xenial
 language: python
 install:
-  - sudo apt-get install graphicsmagick libav-tools -y
+  - sudo apt-get install libav-tools -y
   - pip3 install -r requirements.txt
   - pip3 install tox
   - python setup.py develop
@@ -12,7 +12,6 @@ python:
   - "3.6"
 script:
   - tox
-  - gm -version
   - cd example && prosopopee
 notifications:
   irc: "chat.freenode.net#prosopopee"

docs/configuration.rst

@@ -68,7 +68,7 @@ Global settings can be put in your root `settings.yaml`, under the `settings` ke
 GM
 ~~
 
-Currently a `gm` setting key allows to customize the default GraphicsMagick's behaviour. It looks like::
+Currently a `gm` setting key allows to customize the default behaviour for thumbnail creation. It looks like::
 
   title: Gallery
   settings:
@@ -79,15 +79,15 @@ Currently a `gm` setting key allows to customize the default GraphicsMagick's be
       resize: 50%
       progressive: True
 
-The meaning of the currently supported GraphicsMagick's settings is as follows:
+The meaning of the currently supported settings is as follows:
 
  * `quality` allows to customize the compression level of thumbnails (between 0 and 100)
  * `auto-orient` changes the orientation of pictures so they are upright (based on corresponding EXIF tags if present)
  * `strip` removes all profiles and text attributes from the image (good for privacy, slightly reduce file size)
  * `resize` can be used to resize the full-size version of pictures. By default, input image size is preserved
  * `progressive` converts classic baseline JPEG files to progressive JPEG, and interlaces PNG/GIF files (improves the page loading impression, slightly reduces file size)
 
-Any GraphicsMagick setting can be customized on a per-image basis (either `cover` or `image`, see below).
+Any of thumbnail creation settings can be customized on a per-image basis (either `cover` or `image`, see below).
 
 Video converter
 ~~~~~~~~~~~~~~~

docs/install.rst

@@ -11,12 +11,7 @@ We need Python, pip and virtualenv::
 
     apt-get install python3-pip python3-virtualenv
 
-and graphicsmagick library for building the gallery::
-
-    # graphicsmagick requires to have the 5.3.1 version of gcc-5-base
-    apt-get install graphicsmagick
-
-A video converter like ffmpeg::
+And a video converter like ffmpeg::
 
     apt-get install ffmpeg
 
@@ -35,11 +30,7 @@ We need Brew::
 
   /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
 
-and graphicsmagick library for building the gallery::
-
-  brew install graphicsmagick 
-
-A video converter like ffmpeg::  
+And a video converter like ffmpeg::
 
   brew install ffmpeg
 

prosopopee/autogen.py

@@ -31,6 +31,8 @@
 {% endfor %}
 """
 
+logger = logging.getLogger("prosopopee." + __name__)
+
 types = ("*.JPG", "*.jpg", "*.JPEG", "*.jpeg", "*.png", "*.PNG")
 
 
@@ -51,19 +53,19 @@ def build_template(folder, force):
     gallery_settings = load_settings(folder)
 
     if "static" in gallery_settings:
-        logging.info("Skipped: Nothing to do in %s gallery", folder)
+        logger.info("Skipped: Nothing to do in %s gallery", folder)
         return
 
     if any(req not in gallery_settings for req in ["title", "date", "cover"]):
-        logging.error(
+        logger.error(
             "You need configure first, the title, date and cover in %s/settings.yaml "
             "to use autogen",
             folder,
         )
         sys.exit(1)
 
     if "sections" in gallery_settings and force is not True:
-        logging.info("Skipped: %s gallery is already generated", folder)
+        logger.info("Skipped: %s gallery is already generated", folder)
         return
 
     for files in types:
@@ -77,7 +79,7 @@ def build_template(folder, force):
     )
     settings = open(Path(".").joinpath(folder, "settings.yaml").abspath(), "w")
     settings.write(msg)
-    logging.info("Generation: %s gallery", folder)
+    logger.info("Generation: %s gallery", folder)
 
 
 def autogen(folder=None, force=False):

prosopopee/cache.py

@@ -1,21 +1,15 @@
-import os
 import json
+import logging
+import os
+
+from multiprocessing import Manager
+
+from .utils import remove_superficial_options
 
-CACHE_VERSION = 2
+CACHE_VERSION = 3
 
 
-def remove_superficial_options(options):
-    cleaned_options = options.copy()
-    del cleaned_options["name"]
-    if "text" in cleaned_options:
-        del cleaned_options["text"]
-    if "type" in cleaned_options:
-        del cleaned_options["type"]
-    if "size" in cleaned_options:
-        del cleaned_options["size"]
-    if "float" in cleaned_options:
-        del cleaned_options["float"]
-    return cleaned_options
+logger = logging.getLogger("prosopopee." + __name__)
 
 
 class Cache:
@@ -28,28 +22,46 @@ def __init__(self, json):
         # This wonderfully stupid behavior has been fixed in 3.4 (which nobody uses)
         self.json = json
         if os.path.exists(os.path.join(os.getcwd(), ".prosopopee_cache")):
-            self.cache = json.load(open(self.cache_file_path, "r"))
+            cache = json.load(open(self.cache_file_path, "r"))
         else:
-            self.cache = {"version": CACHE_VERSION}
+            cache = {"version": CACHE_VERSION}
 
-        if "version" not in self.cache or self.cache["version"] != CACHE_VERSION:
+        if "version" not in cache or cache["version"] != CACHE_VERSION:
             print("info: cache format as changed, prune cache")
-            self.cache = {"version": CACHE_VERSION}
+            cache = {"version": CACHE_VERSION}
+
+        self.cache = Manager().dict(cache)
 
     def needs_to_be_generated(self, source, target, options):
         if not os.path.exists(target):
+            logger.debug("%s does not exist. Requesting generation...", target)
             return True
 
         if target not in self.cache:
+            logger.debug("%s not in cache. Requesting generation...", target)
             return True
 
         cached_picture = self.cache[target]
 
-        if cached_picture["size"] != os.path.getsize(source) or cached_picture[
-            "options"
-        ] != remove_superficial_options(options):
+        if cached_picture["size"] != os.path.getsize(source):
+            logger.debug(
+                "%s has different size than in cache. Requesting generation...", target
+            )
+            return True
+
+        options = remove_superficial_options(options)
+        # json.dumps() transforms tuples into list, so to be able to compare options
+        # same transformation needs to be done on runtime dict.
+        options = json.loads(json.dumps(options))
+
+        if cached_picture["options"] != options:
+            logger.debug(
+                "%s has different options than in cache. Requesting generation...",
+                target,
+            )
             return True
 
+        logger.debug("(%s) Skipping cached thumbnail %s", source, target)
         return False
 
     def cache_picture(self, source, target, options):
@@ -59,7 +71,7 @@ def cache_picture(self, source, target, options):
         }
 
     def cache_dump(self):
-        self.json.dump(self.cache, open(self.cache_file_path, "w"))
+        self.json.dump(dict(self.cache), open(self.cache_file_path, "w"))
 
 
 CACHE = Cache(json=json)

prosopopee/image.py

@@ -0,0 +1,106 @@
+import imagesize
+import logging
+import re
+import sys
+
+from json import dumps as json_dumps
+from pathlib import Path
+from zlib import crc32
+
+from .utils import remove_superficial_options
+
+
+logger = logging.getLogger("prosopopee." + __name__)
+
+
+class ImageCommon:
+    @property
+    def ratio(self):
+        # For when BaseImage.ratio is called before BaseImage.copy() is.
+        if not self.size:
+            self.size = imagesize.get(self.filepath)
+        width, height = self.size
+        return width / height
+
+
+class Thumbnail(ImageCommon):
+    def __init__(self, base_filepath, base_id, size):
+        self.filepath = self.__filepath(base_filepath, base_id, size)
+        self.size = size
+
+    def __filepath(self, base_filepath, base_id, size):
+        p = Path(base_filepath)
+        width, height = size
+        suffix = "-{base_id}-{width}x{height}{suffix}".format(
+            base_id=base_id,
+            width=width if width else "",
+            height=height if height else "",
+            suffix=p.suffix,
+        )
+
+        return p.parent / (p.stem + suffix)
+
+
+class BaseImage(ImageCommon):
+    re_rsz = re.compile(r"^(\d+)%$")
+
+    def __init__(self, options, global_options):
+        self.copysize = None
+        self.thumbnails = dict()
+        self.options = global_options.copy()
+        self.options.update(options)
+        self.filepath = self.options["name"]
+        self.resize = self.options.get("resize")
+        self.options = remove_superficial_options(self.options)
+        self.chksum_opt = crc32(
+            bytes(json_dumps(self.options, sort_keys=True), "utf-8")
+        )
+
+    def copy(self):
+        if not self.copysize:
+            width, height = imagesize.get(self.filepath)
+
+            if self.resize:
+                match = BaseImage.re_rsz.match(str(self.resize))
+                if not match:
+                    logger.error(
+                        "(%s) specified resize setting is not a percentage",
+                        self.filepath,
+                    )
+                    sys.exit(1)
+                percentage = int(match.group(1))
+                width, height = width * percentage // 100, height * percentage // 100
+
+            self.copysize = width, height
+
+        return self.thumbnail(self.copysize)
+
+    def thumbnail(self, size):
+        thumbnail = Thumbnail(self.filepath, self.chksum_opt, size)
+        return self.thumbnails.setdefault(thumbnail.filepath, thumbnail).filepath.name
+
+
+# TODO: add support for looking into parent directories (name: ../other_gallery/pic.jpg)
+class ImageFactory:
+    base_imgs = dict()
+    global_options = dict()
+
+    @classmethod
+    def get(cls, path, image):
+        if not isinstance(image, dict):
+            image = {"name": image}
+
+        if "name" not in image:
+            logger.error(
+                'At least one image in "%s" does not have a `name` property, please add the '
+                "filename of the image to a `name` property.",
+                path + "/settings.yaml",
+            )
+            sys.exit(1)
+
+        im = image.copy()
+        # To resolve paths with .. in them, we need to resolve the path first and then
+        # find the relative path to the source (current) directory.
+        im["name"] = Path(path).joinpath(im["name"]).resolve().relative_to(Path.cwd())
+        img = BaseImage(im, cls.global_options)
+        return cls.base_imgs.setdefault(img.filepath / str(img.chksum_opt), img)