diff --git a/.travis.yml b/.travis.yml index 41f79fb8..38052d7d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ sudo: required dist: xenial language: python install: - - sudo apt-get install graphicsmagick libav-tools -y + - sudo apt-get install libav-tools -y - pip3 install -r requirements.txt - pip3 install tox - python setup.py develop @@ -12,7 +12,6 @@ python: - "3.6" script: - tox - - gm -version - cd example && prosopopee notifications: irc: "chat.freenode.net#prosopopee" diff --git a/docs/configuration.rst b/docs/configuration.rst index 24e15b2a..5fc27cc9 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -68,7 +68,7 @@ Global settings can be put in your root `settings.yaml`, under the `settings` ke GM ~~ -Currently a `gm` setting key allows to customize the default GraphicsMagick's behaviour. It looks like:: +Currently a `gm` setting key allows to customize the default behaviour for thumbnail creation. It looks like:: title: Gallery settings: @@ -79,7 +79,7 @@ Currently a `gm` setting key allows to customize the default GraphicsMagick's be resize: 50% progressive: True -The meaning of the currently supported GraphicsMagick's settings is as follows: +The meaning of the currently supported settings is as follows: * `quality` allows to customize the compression level of thumbnails (between 0 and 100) * `auto-orient` changes the orientation of pictures so they are upright (based on corresponding EXIF tags if present) @@ -87,7 +87,7 @@ The meaning of the currently supported GraphicsMagick's settings is as follows: * `resize` can be used to resize the full-size version of pictures. By default, input image size is preserved * `progressive` converts classic baseline JPEG files to progressive JPEG, and interlaces PNG/GIF files (improves the page loading impression, slightly reduces file size) -Any GraphicsMagick setting can be customized on a per-image basis (either `cover` or `image`, see below). +Any of thumbnail creation settings can be customized on a per-image basis (either `cover` or `image`, see below). Video converter ~~~~~~~~~~~~~~~ diff --git a/docs/install.rst b/docs/install.rst index 13159e76..49102b34 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -11,12 +11,7 @@ We need Python, pip and virtualenv:: apt-get install python3-pip python3-virtualenv -and graphicsmagick library for building the gallery:: - - # graphicsmagick requires to have the 5.3.1 version of gcc-5-base - apt-get install graphicsmagick - -A video converter like ffmpeg:: +And a video converter like ffmpeg:: apt-get install ffmpeg @@ -35,11 +30,7 @@ We need Brew:: /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" -and graphicsmagick library for building the gallery:: - - brew install graphicsmagick - -A video converter like ffmpeg:: +And a video converter like ffmpeg:: brew install ffmpeg diff --git a/prosopopee/autogen.py b/prosopopee/autogen.py index 11f05375..6a394d24 100644 --- a/prosopopee/autogen.py +++ b/prosopopee/autogen.py @@ -31,6 +31,8 @@ {% endfor %} """ +logger = logging.getLogger("prosopopee." + __name__) + types = ("*.JPG", "*.jpg", "*.JPEG", "*.jpeg", "*.png", "*.PNG") @@ -51,11 +53,11 @@ def build_template(folder, force): gallery_settings = load_settings(folder) if "static" in gallery_settings: - logging.info("Skipped: Nothing to do in %s gallery", folder) + logger.info("Skipped: Nothing to do in %s gallery", folder) return if any(req not in gallery_settings for req in ["title", "date", "cover"]): - logging.error( + logger.error( "You need configure first, the title, date and cover in %s/settings.yaml " "to use autogen", folder, @@ -63,7 +65,7 @@ def build_template(folder, force): sys.exit(1) if "sections" in gallery_settings and force is not True: - logging.info("Skipped: %s gallery is already generated", folder) + logger.info("Skipped: %s gallery is already generated", folder) return for files in types: @@ -77,7 +79,7 @@ def build_template(folder, force): ) settings = open(Path(".").joinpath(folder, "settings.yaml").abspath(), "w") settings.write(msg) - logging.info("Generation: %s gallery", folder) + logger.info("Generation: %s gallery", folder) def autogen(folder=None, force=False): diff --git a/prosopopee/cache.py b/prosopopee/cache.py index d7f6c731..60e341ac 100644 --- a/prosopopee/cache.py +++ b/prosopopee/cache.py @@ -1,21 +1,15 @@ -import os import json +import logging +import os + +from multiprocessing import Manager + +from .utils import remove_superficial_options -CACHE_VERSION = 2 +CACHE_VERSION = 3 -def remove_superficial_options(options): - cleaned_options = options.copy() - del cleaned_options["name"] - if "text" in cleaned_options: - del cleaned_options["text"] - if "type" in cleaned_options: - del cleaned_options["type"] - if "size" in cleaned_options: - del cleaned_options["size"] - if "float" in cleaned_options: - del cleaned_options["float"] - return cleaned_options +logger = logging.getLogger("prosopopee." + __name__) class Cache: @@ -28,28 +22,46 @@ def __init__(self, json): # This wonderfully stupid behavior has been fixed in 3.4 (which nobody uses) self.json = json if os.path.exists(os.path.join(os.getcwd(), ".prosopopee_cache")): - self.cache = json.load(open(self.cache_file_path, "r")) + cache = json.load(open(self.cache_file_path, "r")) else: - self.cache = {"version": CACHE_VERSION} + cache = {"version": CACHE_VERSION} - if "version" not in self.cache or self.cache["version"] != CACHE_VERSION: + if "version" not in cache or cache["version"] != CACHE_VERSION: print("info: cache format as changed, prune cache") - self.cache = {"version": CACHE_VERSION} + cache = {"version": CACHE_VERSION} + + self.cache = Manager().dict(cache) def needs_to_be_generated(self, source, target, options): if not os.path.exists(target): + logger.debug("%s does not exist. Requesting generation...", target) return True if target not in self.cache: + logger.debug("%s not in cache. Requesting generation...", target) return True cached_picture = self.cache[target] - if cached_picture["size"] != os.path.getsize(source) or cached_picture[ - "options" - ] != remove_superficial_options(options): + if cached_picture["size"] != os.path.getsize(source): + logger.debug( + "%s has different size than in cache. Requesting generation...", target + ) + return True + + options = remove_superficial_options(options) + # json.dumps() transforms tuples into list, so to be able to compare options + # same transformation needs to be done on runtime dict. + options = json.loads(json.dumps(options)) + + if cached_picture["options"] != options: + logger.debug( + "%s has different options than in cache. Requesting generation...", + target, + ) return True + logger.debug("(%s) Skipping cached thumbnail %s", source, target) return False def cache_picture(self, source, target, options): @@ -59,7 +71,7 @@ def cache_picture(self, source, target, options): } def cache_dump(self): - self.json.dump(self.cache, open(self.cache_file_path, "w")) + self.json.dump(dict(self.cache), open(self.cache_file_path, "w")) CACHE = Cache(json=json) diff --git a/prosopopee/image.py b/prosopopee/image.py new file mode 100644 index 00000000..65bfee32 --- /dev/null +++ b/prosopopee/image.py @@ -0,0 +1,106 @@ +import imagesize +import logging +import re +import sys + +from json import dumps as json_dumps +from pathlib import Path +from zlib import crc32 + +from .utils import remove_superficial_options + + +logger = logging.getLogger("prosopopee." + __name__) + + +class ImageCommon: + @property + def ratio(self): + # For when BaseImage.ratio is called before BaseImage.copy() is. + if not self.size: + self.size = imagesize.get(self.filepath) + width, height = self.size + return width / height + + +class Thumbnail(ImageCommon): + def __init__(self, base_filepath, base_id, size): + self.filepath = self.__filepath(base_filepath, base_id, size) + self.size = size + + def __filepath(self, base_filepath, base_id, size): + p = Path(base_filepath) + width, height = size + suffix = "-{base_id}-{width}x{height}{suffix}".format( + base_id=base_id, + width=width if width else "", + height=height if height else "", + suffix=p.suffix, + ) + + return p.parent / (p.stem + suffix) + + +class BaseImage(ImageCommon): + re_rsz = re.compile(r"^(\d+)%$") + + def __init__(self, options, global_options): + self.copysize = None + self.thumbnails = dict() + self.options = global_options.copy() + self.options.update(options) + self.filepath = self.options["name"] + self.resize = self.options.get("resize") + self.options = remove_superficial_options(self.options) + self.chksum_opt = crc32( + bytes(json_dumps(self.options, sort_keys=True), "utf-8") + ) + + def copy(self): + if not self.copysize: + width, height = imagesize.get(self.filepath) + + if self.resize: + match = BaseImage.re_rsz.match(str(self.resize)) + if not match: + logger.error( + "(%s) specified resize setting is not a percentage", + self.filepath, + ) + sys.exit(1) + percentage = int(match.group(1)) + width, height = width * percentage // 100, height * percentage // 100 + + self.copysize = width, height + + return self.thumbnail(self.copysize) + + def thumbnail(self, size): + thumbnail = Thumbnail(self.filepath, self.chksum_opt, size) + return self.thumbnails.setdefault(thumbnail.filepath, thumbnail).filepath.name + + +# TODO: add support for looking into parent directories (name: ../other_gallery/pic.jpg) +class ImageFactory: + base_imgs = dict() + global_options = dict() + + @classmethod + def get(cls, path, image): + if not isinstance(image, dict): + image = {"name": image} + + if "name" not in image: + logger.error( + 'At least one image in "%s" does not have a `name` property, please add the ' + "filename of the image to a `name` property.", + path + "/settings.yaml", + ) + sys.exit(1) + + im = image.copy() + # To resolve paths with .. in them, we need to resolve the path first and then + # find the relative path to the source (current) directory. + im["name"] = Path(path).joinpath(im["name"]).resolve().relative_to(Path.cwd()) + img = BaseImage(im, cls.global_options) + return cls.base_imgs.setdefault(img.filepath / str(img.chksum_opt), img) diff --git a/prosopopee/prosopopee.py b/prosopopee/prosopopee.py index dc56ba18..2522e439 100644 --- a/prosopopee/prosopopee.py +++ b/prosopopee/prosopopee.py @@ -8,9 +8,12 @@ import subprocess import sys import http.server +import struct from babel.core import default_locale from babel.dates import format_date +from multiprocessing import Pool +from PIL import Image, ImageOps, JpegImagePlugin, ImageFile from path import Path @@ -20,6 +23,7 @@ from .utils import encrypt, rfc822, load_settings, CustomFormatter from .autogen import autogen from .__init__ import __version__ +from .image import ImageFactory def loglevel(string): @@ -38,12 +42,20 @@ def loglevel(string): parser.add_argument("--version", action="version", version="%(prog)s " + __version__) parser.add_argument( "--log-level", - default=logging.NOTSET, + default=logging.DEBUG, type=loglevel, help="Configure the logging level", ) subparser = parser.add_subparsers(dest="cmd") -subparser.add_parser("build", help="Generate static site") +parser_build = subparser.add_parser("build", help="Generate static site") +parser_build.add_argument( + "-j", + "--jobs", + default=None, + type=int, + help="Specifies number of jobs (thumbnail generations) to run simultaneously. Default: number " + "of threads available on the system", +) subparser.add_parser("test", help="Verify all your yaml data") subparser.add_parser("preview", help="Start preview webserver on port 9000") subparser.add_parser("deploy", help="Deploy your website") @@ -107,13 +119,16 @@ def loglevel(string): } +ImageFactory.global_options = SETTINGS["gm"] + + class Video: base_dir = Path() target_dir = Path() def __init__(self, options): if SETTINGS["ffmpeg"] is False: - logging.error( + logger.error( "I couldn't find a binary to convert video and I ask to do so + abort" ) sys.exit(1) @@ -133,7 +148,7 @@ def ffmpeg(self, source, target, options): if not options.get("resize"): target = target + "." + options["extension"] if not CACHE.needs_to_be_generated(source, target, options): - logging.info("Skipped: %s is already generated", source) + logger.info("Skipped: %s is already generated", source) return ffmpeg_switches = { @@ -151,7 +166,7 @@ def ffmpeg(self, source, target, options): "other": "%s" % options["other"], } - logging.info("Generation: %s", source) + logger.info("Generation: %s", source) if options.get("resize"): command = ( @@ -168,7 +183,7 @@ def ffmpeg(self, source, target, options): print(command) if os.system(command) != 0: - logging.error("%s command failed", ffmpeg_switches["binary"]) + logger.error("%s command failed", ffmpeg_switches["binary"]) sys.exit(1) CACHE.cache_picture(source, target, options) @@ -225,7 +240,7 @@ class Audio: def __init__(self, options): if SETTINGS["ffmpeg"] is False: - logging.error( + logger.error( "I couldn't find a binary to convert audio and I ask to do so + abort" ) sys.exit(1) @@ -244,7 +259,7 @@ def name(self): def ffmpeg(self, source, target, options): target = target + "." + options["extension"] if not CACHE.needs_to_be_generated(source, target, options): - logging.info("Skipped: %s is already generated", source) + logger.info("Skipped: %s is already generated", source) return ffmpeg_switches = { @@ -255,14 +270,14 @@ def ffmpeg(self, source, target, options): "audio": "-c:a %s" % options["audio"], } - logging.info("Generation: %s", source) + logger.info("Generation: %s", source) command = "{binary} {loglevel} -i '{source}' {audio} -y '{target}'".format( **ffmpeg_switches ) print(command) if os.system(command) != 0: - logging.error("%s command failed", ffmpeg_switches["binary"]) + logger.error("%s command failed", ffmpeg_switches["binary"]) sys.exit(1) CACHE.cache_picture(source, target, options) @@ -281,118 +296,6 @@ def __repr__(self): return self.name -class Image: - base_dir = Path() - target_dir = Path() - - def __init__(self, options): - # assuming string - if not isinstance(options, dict): - options = {"name": options} - - self.options = SETTINGS[ - "gm" - ].copy() # used for caching, if it's modified -> regenerate - self.options.update(options) - - @property - def name(self): - return self.options["name"] - - def convert(self, source, target, options): - if not CACHE.needs_to_be_generated(source, target, options): - logging.info("Skipped: %s is already generated", source) - return - - if DEFAULTS["test"]: - return - - gm_switches = { - "source": source, - "target": target, - "auto-orient": "-auto-orient" if options["auto-orient"] else "", - "strip": "-strip" if options["strip"] else "", - "quality": "-quality %s" % options["quality"] - if "quality" in options - else "-define jpeg:preserve-settings", - "resize": "-resize %s" % options["resize"] - if options.get("resize", None) is not None - else "", - "progressive": "-interlace Line" - if options.get("progressive", None) is True - else "", - } - - command = ( - "gm convert '{source}' {auto-orient} {strip} {progressive} {quality} {resize} " - "'{target}'" - ).format(**gm_switches) - logging.info("Generation: %s", source) - - print(command) - if os.system(command) != 0: - logging.error("gm command failed") - sys.exit(1) - - CACHE.cache_picture(source, target, options) - - def copy(self): - source, target = self.base_dir.joinpath(self.name), self.target_dir.joinpath( - self.name - ) - - # XXX doing this DOESN'T improve perf at all (or something like 0.1%) - # if os.path.exists(target) and os.path.getsize(source) == os.path.getsize(target): - # print "Skipped %s since the file hasn't been modified based on file size" % source - # return "" - if not DEFAULTS["test"]: - options = self.options.copy() - - if not options["auto-orient"] and not options["strip"]: - shutil.copyfile(source, target) - print(("%s%s%s" % (source, "->", target))) - else: - # Do not consider quality settings here, since we aim to copy the input image - # better to preserve input encoding setting - del options["quality"] - self.convert(source, target, options) - - return "" - - def generate_thumbnail(self, gm_geometry): - thumbnail_name = ( - ".".join(self.name.split(".")[:-1]) - + "-" - + gm_geometry - + "." - + self.name.split(".")[-1] - ) - if not DEFAULTS["test"]: - source, target = ( - self.base_dir.joinpath(self.name), - self.target_dir.joinpath(thumbnail_name), - ) - - options = self.options.copy() - options.update({"resize": gm_geometry}) - - self.convert(source, target, options) - - return thumbnail_name - - @property - def ratio(self): - command = "gm identify -format %w,%h" - command_list = command.split() - command_list.append(str(self.base_dir.joinpath(self.name))) - out = subprocess.check_output(command_list) - width, height = out.decode("utf-8").split(",") - return float(width) / int(height) - - def __repr__(self): - return self.name - - class TCPServerV4(socketserver.TCPServer): allow_reuse_address = True @@ -415,26 +318,20 @@ def get_settings(): else: conv_video = "ffmpeg" - if os.system("which gm > /dev/null") != 0: - logging.error( - "I can't locate the gm binary + please install the 'graphicsmagick' package." - ) - sys.exit(1) - if os.system("which " + conv_video + " > /dev/null") != 0: if conv_video == "ffmpeg" and os.system("which avconv > /dev/null") == 0: SETTINGS["ffmpeg"]["binary"] = "avconv" - logging.warning( + logger.warning( "Video: I couldn't locate ffmpeg but I could find avconv, " "switching to avconv for video conversion" ) else: - logging.warning( + logger.warning( "Video: I can't locate the %s binary, please install the '%s' package.", conv_video, conv_video, ) - logging.warning( + logger.warning( "Video: I won't be able to encode video and I will stop if I " "encounter a video to convert" ) @@ -443,7 +340,7 @@ def get_settings(): if ( settings["rss"] or settings["share"] or settings["settings"].get("og") ) and not settings.get("url"): - logging.warning( + logger.warning( "warning: If you want the rss, OpenGraph and/or the social network share to work, " "you need to specify the website url in root settings" ) @@ -477,7 +374,7 @@ def get_gallery_templates( ) if not theme_path: - logging.error( + logger.error( "'%s' is not an existing theme + available themes are '%s'", theme_path, available_themes, @@ -548,7 +445,7 @@ def process_directory( return gallery_cover if gallery_settings.get("sections", False): - logging.error( + logger.error( "The gallery in %s can't have both sections and subgalleries", gallery_name.joinpath("settings.yaml"), ) @@ -587,7 +484,7 @@ def process_directory( def create_cover(gallery_name, gallery_settings, gallery_path): if not gallery_settings.get("cover"): - logging.error( + logger.error( "You should specify a path to a cover picture in %s", gallery_name.joinpath("settings.yaml"), ) @@ -595,15 +492,13 @@ def create_cover(gallery_name, gallery_settings, gallery_path): if isinstance(gallery_settings["cover"], dict): cover_image_path = gallery_path.joinpath(gallery_settings["cover"]["name"]) - cover_image_url = gallery_name.joinpath(gallery_settings["cover"]["name"]) cover_image_type = gallery_settings["cover"]["type"] else: cover_image_path = gallery_path.joinpath(gallery_settings["cover"]) - cover_image_url = gallery_name.joinpath(gallery_settings["cover"]) cover_image_type = "image" if not cover_image_path.exists(): - logging.error( + logger.error( "File for %s cover image doesn't exist at %s", gallery_name, cover_image_path, @@ -612,12 +507,13 @@ def create_cover(gallery_name, gallery_settings, gallery_path): gallery_cover = { "title": gallery_settings["title"], - "link": gallery_name + "/", + "link": gallery_path, + "name": gallery_name + "/", "sub_title": gallery_settings.get("sub_title", ""), "date": gallery_settings.get("date", ""), "tags": gallery_settings.get("tags", ""), "cover_type": cover_image_type, - "cover": cover_image_url, + "cover": gallery_settings["cover"], } return gallery_cover @@ -626,10 +522,6 @@ def build_gallery(settings, gallery_settings, gallery_path, template): gallery_index_template = template.get_template("gallery-index.html") page_template = template.get_template("page.html") - # this should probably be a factory - Image.base_dir = Path(".").joinpath(gallery_path) - Image.target_dir = Path(".").joinpath("build", gallery_path) - Video.base_dir = Path(".").joinpath(gallery_path) Video.target_dir = Path(".").joinpath("build", gallery_path) @@ -647,7 +539,7 @@ def build_gallery(settings, gallery_settings, gallery_path, template): html = template_to_render.render( settings=settings, gallery=gallery_settings, - Image=Image, + Image=ImageFactory, Video=Video, Audio=Audio, link=gallery_path, @@ -677,9 +569,6 @@ def build_gallery(settings, gallery_settings, gallery_path, template): "light", gallery_light_path, date_locale=settings["settings"].get("date_locale") ) - Image.base_dir = Path(".").joinpath(gallery_path) - Image.target_dir = Path(".").joinpath("build", gallery_path) - Video.base_dir = Path(".").joinpath(gallery_path) Video.target_dir = Path(".").joinpath("build", gallery_path) @@ -691,7 +580,7 @@ def build_gallery(settings, gallery_settings, gallery_path, template): html = light_template_to_render.render( settings=settings, gallery=gallery_settings, - Image=Image, + Image=ImageFactory, Video=Video, Audio=Audio, link=gallery_light_path, @@ -731,10 +620,6 @@ def build_index( sorted([x for x in galleries_cover if x != {}], key=lambda x: x["date"]) ) - # this should probably be a factory - Image.base_dir = Path(".").joinpath(gallery_path) - Image.target_dir = Path(".").joinpath("build", gallery_path) - Video.base_dir = Path(".").joinpath(gallery_path) Video.target_dir = Path(".").joinpath("build", gallery_path) @@ -742,7 +627,7 @@ def build_index( settings=settings, galleries=galleries_cover, sub_index=sub_index, - Image=Image, + Image=ImageFactory, Video=Video, ).encode("Utf-8") @@ -755,12 +640,171 @@ def build_index( open(Path("build").joinpath(gallery_path, "index.html"), "wb").write(html) +def image_params(img, options): + format = img.format + + params = {"format": format} + if "progressive" in options: + params["progressive"] = options["progressive"] + if "quality" in options: + params["quality"] = options["quality"] + if "dpi" in img.info: + params["dpi"] = img.info["dpi"] + if format == "JPEG" or format == "MPO": + params["subsampling"] = JpegImagePlugin.get_sampling(img) + + exif = img.getexif() + if exif: + params["exif"] = exif + + return params + + +def noncached_images(base): + img = Image.open(base.filepath) + params = image_params(img, base.options) + + if params.get("exif") and base.options.get("strip", False): + del params["exif"] + + for thumbnail in base.thumbnails.values(): + filepath = Path("build") / thumbnail.filepath + + if CACHE.needs_to_be_generated(base.filepath, str(filepath), params): + return base + + +def render_thumbnails(base): + logger.debug("(%s) Rendering thumbnails", base.filepath) + + img = Image.open(base.filepath) + params = image_params(img, base.options) + + exif = params.get("exif") + + # Re-orient if requested and if Orientation EXIF metadata stored in 0x0112 states that + # it's not upright. + if exif and base.options.get("auto-orient", False) and exif.get(0x0112, 1) != 1: + orientation = exif.get(0x0112) + + logger.debug( + "(%s) Orientation EXIF tag set to %d: rotating thumbnails", + base.filepath, + orientation, + ) + + try: + img = ImageOps.exif_transpose(img) + except (TypeError, struct.error) as e: + # Work-around for Pillow < 7.2.0 because of broken handling of some exif metadata + # Fixed with https://github.com/python-pillow/Pillow/pull/4637 + # Work-around for Pillow < 7.0.0 not handling StripByteCounts being of type long + # Fixed with https://github.com/python-pillow/Pillow/pull/4626 + method = { + 2: Image.FLIP_LEFT_RIGHT, + 3: Image.ROTATE_180, + 4: Image.FLIP_TOP_BOTTOM, + 5: Image.TRANSPOSE, + 6: Image.ROTATE_270, + 7: Image.TRANSVERSE, + 8: Image.ROTATE_90, + }.get(orientation) + img = img.transpose(method) + if not base.options.get("strip", False): + logger.warning( + "(%s) Original image contains EXIF metadata that Pillow < %s cannot " + "handle. Consider upgrading to a newer release. The image will be " + "forcefully stripped of its EXIF metadata as a work-around.", + base.filepath, + "7.2.0" if isinstance(e, TypeError) else "7.0.0", + ) + del params["exif"] + + if params.get("exif") and base.options.get("strip", False): + del params["exif"] + + for thumbnail in base.thumbnails.values(): + filepath = Path("build") / thumbnail.filepath + + if not CACHE.needs_to_be_generated(base.filepath, str(filepath), params): + continue + + # Needed because im.thumbnail replaces the original image + im = img.copy() + + width, height = thumbnail.size + + if not width or not height: + # im.thumbnail() needs both valid values in the size tuple. + # Moreover, the function creates a thumbnail whose dimensions are + # within the provided size. + # When only one dimension is specified, the other should thus be + # outrageously big so that the thumbnail dimension will always be + # of the specified value. + IGNORE_DIM = 65596 + height = height if height is not None else IGNORE_DIM + width = width if width is not None else IGNORE_DIM + im.thumbnail((width, height), Image.LANCZOS) + + logger.debug( + "(%s) Creating thumbnail %s: size=%s", + base.filepath, + filepath, + thumbnail.size, + ) + try: + im.save(filepath, **params) + except OSError as e: + # Work-around for: + # https://github.com/python-pillow/Pillow/issues/148 + logger.warning( + '(%s) Failed to save "%s". This usually happens when progressive is set to True and' + " quality set to a too high value (> 90). Please lower quality or disable" + ' progressive, globally or for "%s". As a work-around, increase buffer size. This' + " might result in side-effects.\n" + 'The original error is "%s"', + base.filepath, + filepath, + base.filepath, + e, + ) + width, height = (width, height) if width and height else thumbnail.size + ImageFile.MAXBLOCK = max( + ImageFile.MAXBLOCK, + (4 * width * height) + len(im.info.get("icc_profile", "")) + 10, + ) + im.save(filepath, **params) + except TypeError as e: + # Work-around for Pillow < 7.2.0 because of broken handling of some exif metadata + # Fixed with https://github.com/python-pillow/Pillow/pull/4637 + logger.warning( + "(%s) Original image contains EXIF metadata that Pillow < 7.2.0 cannot handle. " + "Consider upgrading to a newer release. The image will be forcefully stripped of " + "its EXIF metadata as a work-around.\n" + 'The original error is "%s"', + base.filepath, + e, + ) + del params["exif"] + im.save(filepath, **params) + + logger.debug( + "(%s) Done creating thumbnail %s: size=%s", + base.filepath, + filepath, + thumbnail.size, + ) + CACHE.cache_picture(base.filepath, str(filepath), params) + + +logger = logging.getLogger("prosopopee") + + def main(): args = parser.parse_args() handler = logging.StreamHandler() handler.setFormatter(CustomFormatter()) - logger = logging.getLogger() logger.addHandler(handler) logger.setLevel(args.log_level) @@ -774,7 +818,7 @@ def main(): includes = [x for x in settings["include"] if Path(".").joinpath(x).exists()] if not galleries_dirs: - logging.error( + logger.error( "I can't find at least one directory with a settings.yaml in the current " "working directory (NOT the settings.yaml in your current directory, but one " "INSIDE A DIRECTORY in your current directory), you don't have any gallery?" @@ -786,7 +830,7 @@ def main(): if args.cmd == "preview": if not Path("build").exists(): - logging.error("Please build the website before launch preview") + logger.error("Please build the website before launch preview") sys.exit(1) os.chdir("build") @@ -802,12 +846,12 @@ def main(): if args.cmd == "deploy": if os.system("which rsync > /dev/null") != 0: - logging.error( + logger.error( "I can't locate the rsync + please install the 'rsync' package." ) sys.exit(1) if not Path("build").exists(): - logging.error("Please build the website before launch deployment") + logger.error("Please build the website before launch deployment") sys.exit(1) r_dest = settings["settings"]["deploy"]["dest"] @@ -827,7 +871,7 @@ def main(): else: r_cmd = "rsync -avz --progress %s build/* %s" % (r_others, r_dest) if os.system(r_cmd) != 0: - logging.error("deployment failed") + logger.error("deployment failed") sys.exit(1) return @@ -851,6 +895,8 @@ def main(): ) settings["custom_css"] = True + logger.info("Building galleries...") + for gallery in galleries_dirs: front_page_galleries_cover.append( process_directory(gallery.normpath(), settings, templates) @@ -862,7 +908,7 @@ def main(): if srcdir != "": os.makedirs(dstdir, exist_ok=True) d = shutil.copy2(i, dstdir) - logging.warning("copied", d) + logger.warning("copied", d) if settings["rss"]: feed_template = templates.get_template("feed.xml") @@ -880,10 +926,38 @@ def main(): open(Path("build").joinpath("feed.xml"), "wb").write(xml) build_index(settings, front_page_galleries_cover, templates) - CACHE.cache_dump() if DEFAULTS["test"] is True: - logging.info("Success: HTML file building without error") + logger.info("Success: HTML file building without error") + sys.exit(0) + + # If prosopopee is started without any argument, 'build' is assumed but the jobs parameter + # is not part of the namespace, so set its default to None (or 'number of available CPU + # treads') + jobs = args.jobs if args.cmd else None + + with Pool(jobs) as pool: + # Pool splits the iterable into pre-defined chunks which are then assigned to processes. + # There is no other scheduling in play after that. This is an issue when chunks are + # outrageously unbalanced in terms of CPU time which happens when most galleries are + # already built and thus hit the cache but not some, in which case, only a few processes + # will run and not the full CPU power will be used, wasting time. + # In order to optimize this, a first very quick run through the list of images is done + # to list only those which actually need to be generated. + # In the following implementation, the first pool.map function will be slightly + # unbalanced because some images will hit the cache directly while some won't at all, + # iterating over all the thumbnails it needs to create. But it is **much** less + # unbalanced than sending to render_thumbnails all images even those which would hit the + # cache. After the first pool.map, the second will only contain images with at least one + # thumbnail to create. + logger.info("Generating list of thumbnails to create...") + base_imgs = pool.map(noncached_images, ImageFactory.base_imgs.values()) + base_imgs = [img for img in base_imgs if img] + if base_imgs: + logger.info("Generating thumbnails...") + pool.map(render_thumbnails, base_imgs) + + CACHE.cache_dump() if __name__ == "__main__": diff --git a/prosopopee/themes/exposure/templates/index.html b/prosopopee/themes/exposure/templates/index.html index 5f41099d..38ded55b 100644 --- a/prosopopee/themes/exposure/templates/index.html +++ b/prosopopee/themes/exposure/templates/index.html @@ -26,13 +26,13 @@ {% block content %}
{% for galleries_line in galleries|reverse|batch(3)|reverse %} - {% if loop.index is divisibleby 3 or loop.index is divisibleby 2 %} + {% if galleries_line|length > 1 %} {% set no_big_gallery_cover = true %} {% endif %}
{% for gallery in galleries_line|reverse %}
- +

{{ gallery.title }}

{% if gallery.sub_title %}

{{ gallery.sub_title }}

{% endif %} @@ -51,12 +51,11 @@

{{ gallery.title }}

{% else %} - {% set cover = Image(gallery.cover) %} - {{ cover.copy() }} + {% set cover = Image.get(gallery.link, gallery.cover) %} {% if no_big_gallery_cover %} -
+
{% else %} -
+
{% endif %} {% endif %}
{% endfor %} diff --git a/prosopopee/themes/light/templates/opengraph.html b/prosopopee/themes/light/templates/opengraph.html index 26f3a568..8811ab69 100644 --- a/prosopopee/themes/light/templates/opengraph.html +++ b/prosopopee/themes/light/templates/opengraph.html @@ -7,9 +7,8 @@ {% endif %} {% if gallery.cover %} -{% set cover = Image(gallery.cover) %} -{{ cover.copy() }} - +{% set cover = Image.get(link + "/" + pathstatic, gallery.cover) %} + {% endif %} {% if gallery.description %} diff --git a/prosopopee/themes/light/templates/sections/author.html b/prosopopee/themes/light/templates/sections/author.html index cdf20b60..6ffa4d04 100644 --- a/prosopopee/themes/light/templates/sections/author.html +++ b/prosopopee/themes/light/templates/sections/author.html @@ -3,10 +3,9 @@ {% else %} {% set pathstatic = "." %} {% endif %} -{% set image = Image(section.image) %} -{{ image.copy() }} +{% set image = Image.get(link + "/" + pathstatic, section.image) %}
- +
Story by

{{ section.name }}

diff --git a/prosopopee/themes/light/templates/sections/bordered-picture.html b/prosopopee/themes/light/templates/sections/bordered-picture.html index 53474ed8..bb5a0bb8 100644 --- a/prosopopee/themes/light/templates/sections/bordered-picture.html +++ b/prosopopee/themes/light/templates/sections/bordered-picture.html @@ -8,8 +8,7 @@ {% set format = settings.ffmpeg.extension %} {{ video.copy() }} {% else %} -{% set image = Image(section.image) %} -{{ image.copy() }} +{% set image = Image.get(link + "/" + pathstatic, section.image) %} {% endif %} {% set caption = section.text %} {% if video %} @@ -20,7 +19,7 @@ {% else %}
- {% if caption %}{{ caption }}{% endif %} + {% if caption %}{{ caption }}{% endif %}
{% endif %} diff --git a/prosopopee/themes/light/templates/sections/full-picture.html b/prosopopee/themes/light/templates/sections/full-picture.html index a503fb98..1ad4e60d 100644 --- a/prosopopee/themes/light/templates/sections/full-picture.html +++ b/prosopopee/themes/light/templates/sections/full-picture.html @@ -8,8 +8,7 @@ {% set format = settings.ffmpeg.extension %} {{ video.copy() }} {% else %} -{% set image = Image(section.image) %} -{{ image.copy() }} +{% set image = Image.get(link + "/" + pathstatic, section.image) %} {% endif %}
@@ -18,7 +17,7 @@ {% else %} - + {% endif %}
{% if section.text %} diff --git a/prosopopee/themes/light/templates/sections/panorama.html b/prosopopee/themes/light/templates/sections/panorama.html index 224ecd3e..10a2a338 100644 --- a/prosopopee/themes/light/templates/sections/panorama.html +++ b/prosopopee/themes/light/templates/sections/panorama.html @@ -3,8 +3,7 @@ {% else %} {% set pathstatic = "." %} {% endif %} -{% set image = Image(section.image) %} -{{ image.copy() }} +{% set image = Image.get(link + "/" + pathstatic, section.image) %}
- +
diff --git a/prosopopee/themes/light/templates/sections/paragraph.html b/prosopopee/themes/light/templates/sections/paragraph.html index e17e68a7..0131945e 100644 --- a/prosopopee/themes/light/templates/sections/paragraph.html +++ b/prosopopee/themes/light/templates/sections/paragraph.html @@ -9,15 +9,14 @@

{{ section.title }}

{% endif %}

{% if section.image %} - {% set image = Image(section.image) %} + {% set image = Image.get(link + "/" + pathstatic, section.image) %} {% if section.image.float %} {% set float = section.image.float %} {% else %} {% set float = 'left' %} {% endif %} - {{ image.copy() }} - + {% endif %} {{ section.text }} diff --git a/prosopopee/themes/light/templates/sections/pictures-group.html b/prosopopee/themes/light/templates/sections/pictures-group.html index 8e61cee6..e37b8623 100644 --- a/prosopopee/themes/light/templates/sections/pictures-group.html +++ b/prosopopee/themes/light/templates/sections/pictures-group.html @@ -11,8 +11,7 @@ {% set format = settings.ffmpeg.extension %} {{ video.copy() }} {% else %} - {% set image = Image(image) %} - {{ image.copy() }} + {% set image = Image.get(link + "/" + pathstatic, image) %} {% endif %} {% set caption = image.text %}

@@ -22,7 +21,7 @@ {% set video = "" %} {% else %} - {% if caption %}{{ caption }}{% endif %} + {% if caption %}{{ caption }}{% endif %} {% endif %}
{% endfor %} diff --git a/prosopopee/themes/material/templates/index.html b/prosopopee/themes/material/templates/index.html index 2f82803a..65ff1bed 100644 --- a/prosopopee/themes/material/templates/index.html +++ b/prosopopee/themes/material/templates/index.html @@ -34,9 +34,8 @@

{{ gallery.title }}

{% else %} - {% set cover = Image(gallery.cover) %} - {{ cover.copy() }} -
+ {% set cover = Image.get(gallery.link, gallery.cover) %} +
{% endif %}
{% endfor %} diff --git a/prosopopee/themes/material/templates/opengraph.html b/prosopopee/themes/material/templates/opengraph.html index 6632a209..4e4db177 100644 --- a/prosopopee/themes/material/templates/opengraph.html +++ b/prosopopee/themes/material/templates/opengraph.html @@ -1,8 +1,7 @@ {% set absolute_url = settings.url + "/" + link + "/" %} -{% set cover = Image(gallery.cover) %} -{{ cover.copy() }} +{% set cover = Image.get(gallery.link, gallery.cover) %} - + {% if gallery.description %} diff --git a/prosopopee/themes/material/templates/sections/author.html b/prosopopee/themes/material/templates/sections/author.html index 83833650..719fb3ec 100644 --- a/prosopopee/themes/material/templates/sections/author.html +++ b/prosopopee/themes/material/templates/sections/author.html @@ -1,5 +1,4 @@ -{% set image = Image(section.image) %} -{{ image.copy() }} +{% set image = Image.get(link, section.image) %}
@@ -8,7 +7,7 @@
- +
diff --git a/prosopopee/themes/material/templates/sections/bordered-picture.html b/prosopopee/themes/material/templates/sections/bordered-picture.html index 9ae25476..eb010425 100644 --- a/prosopopee/themes/material/templates/sections/bordered-picture.html +++ b/prosopopee/themes/material/templates/sections/bordered-picture.html @@ -2,9 +2,8 @@ {% set video = Video(section.image) %} {% set format = settings.ffmpeg.extension %} {% else %} -{% set image = Image(section.image) %} +{% set image = Image.get(link, section.image) %} {% set caption = section.text %} -{{ image.copy() }} {% endif %} {% if section.background %}
@@ -16,13 +15,13 @@ {% else %} - - + {% if caption %}
{{ caption }}
diff --git a/prosopopee/themes/material/templates/sections/full-picture.html b/prosopopee/themes/material/templates/sections/full-picture.html index e90a2b95..0307b5f8 100644 --- a/prosopopee/themes/material/templates/sections/full-picture.html +++ b/prosopopee/themes/material/templates/sections/full-picture.html @@ -8,9 +8,8 @@ {% else %} - {% set image = Image(section.image) %} - {{ image.copy() }} - + {% set image = Image.get(link, section.image) %} + {% endif %}
diff --git a/prosopopee/themes/material/templates/sections/panorama.html b/prosopopee/themes/material/templates/sections/panorama.html index 641e207e..731d6e17 100644 --- a/prosopopee/themes/material/templates/sections/panorama.html +++ b/prosopopee/themes/material/templates/sections/panorama.html @@ -1,5 +1,4 @@ -{% set image = Image(section.image) %} -{{ image.copy() }} +{% set image = Image.get(link, section.image) %}
- +
diff --git a/prosopopee/themes/material/templates/sections/paragraph.html b/prosopopee/themes/material/templates/sections/paragraph.html index 7ce952b1..cffcf9f0 100644 --- a/prosopopee/themes/material/templates/sections/paragraph.html +++ b/prosopopee/themes/material/templates/sections/paragraph.html @@ -12,7 +12,7 @@

{{ section.title }}

{% if section.image %} - {% set image = Image(section.image) %} + {% set image = Image.get(link, section.image) %} {% if section.image.float %} {% set float = section.image.float %} {% else %} @@ -23,15 +23,14 @@

{{ section.title }}

{% else %} {% set image_resize = '250px' %} {% endif %} - {{ image.copy() }} - - + + {% endif %} diff --git a/prosopopee/themes/material/templates/sections/pictures-group.html b/prosopopee/themes/material/templates/sections/pictures-group.html index 85ae8e46..c5c142e7 100644 --- a/prosopopee/themes/material/templates/sections/pictures-group.html +++ b/prosopopee/themes/material/templates/sections/pictures-group.html @@ -12,8 +12,7 @@ {{ video.copy() }} {% set ratio = video.ratio %} {% else %} - {% set image = Image(image) %} - {{ image.copy() }} + {% set image = Image.get(link, image) %} {% set ratio = image.ratio %} {% endif %}
@@ -28,13 +27,13 @@
{{ caption }}
{% endif %} {% else %} - - + {% if caption %}
{{ caption }}
diff --git a/prosopopee/utils.py b/prosopopee/utils.py index 848c2af9..3b823d7a 100644 --- a/prosopopee/utils.py +++ b/prosopopee/utils.py @@ -10,6 +10,32 @@ import ruamel.yaml as yaml +logger = logging.getLogger("prosopopee." + __name__) + + +def remove_superficial_options(options): + cleaned_options = options.copy() + if "name" in cleaned_options: + del cleaned_options["name"] + if "exif" in cleaned_options: + del cleaned_options["exif"] + if "text" in cleaned_options: + del cleaned_options["text"] + if "type" in cleaned_options: + del cleaned_options["type"] + if "size" in cleaned_options: + del cleaned_options["size"] + if "float" in cleaned_options: + del cleaned_options["float"] + # "resize" only applies to image.copy() in templates, no need to propagate it to the cache since + # the actual size of the "copy" thumbnail is part of the filename and will trigger a + # regeneration if changed (thus "resize" setting is appropriately watched without regenerating + # non-copy thumbnails). + if "resize" in cleaned_options: + del cleaned_options["resize"] + return cleaned_options + + class CustomFormatter(logging.Formatter): """Logging Formatter to add colors""" @@ -74,32 +100,32 @@ def load_settings(folder): msg = "There is something wrong in %s/settings.yaml" % folder if isinstance(exc, yaml.error.MarkedYAMLError): msg = msg + str(exc.context_mark) - logging.error(msg) + logger.error(msg) sys.exit(1) except ValueError: - logging.error( + logger.error( "Incorrect data format, should be YYYY-MM-DD in %s/settings.yaml", folder ) sys.exit(1) except Exception as exc: - logging.exception(exc) + logger.exception(exc) sys.exit(1) if gallery_settings is None: - logging.error("The %s/settings.yaml file is empty", folder) + logger.error("The %s/settings.yaml file is empty", folder) sys.exit(1) elif not isinstance(gallery_settings, dict): - logging.error("%s/settings.yaml should be a dict", folder) + logger.error("%s/settings.yaml should be a dict", folder) sys.exit(1) elif "title" not in gallery_settings: - logging.error("You should specify a title in %s/settings.yaml", folder) + logger.error("You should specify a title in %s/settings.yaml", folder) sys.exit(1) if gallery_settings.get("date"): try: datetime.strptime(str(gallery_settings.get("date")), "%Y-%m-%d") except ValueError: - logging.error( + logger.error( "Incorrect data format, should be YYYY-MM-DD in %s/settings.yaml", folder, ) diff --git a/requirements.txt b/requirements.txt index cdc42afe..d73c8126 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ path.py ruamel.yaml future pillow>=6 +imagesize