Feature Request: management command to download remote assets

[geoffbeier]

When I take my app live, I usually like to store all the static js and css it needs with all its other static files and serve them myself. As I was reading code to see how iommi Styles work, it occurred to me that I'd like a script to download them and stash them in my static tree.

I wrote a quick management command to do that. It doesn't depend on anything other than iommi and the standard library.

Once I figure out how to just attach a script to a github issue, I'll add it here.

If you'd be interested in having it as part of iommi, I'd be happy to clean it up a little more and add some tests for it, then submit it as a PR instead of an issue.

[geoffbeier]

Github won't let me attach a python file. Here it is in a code fence instead.

import logging
import os
from typing import List, Tuple, Optional
from urllib.error import URLError, HTTPError
from urllib.parse import urlparse, unquote
from urllib.request import urlopen
import hashlib
import base64

from django.core.management.base import BaseCommand, CommandError

logger = logging.getLogger(__name__)


def get_style_assets(style) -> dict:
    """Extract assets from an iommi style"""
    try:
        return style.root["assets"]
    except (KeyError, AttributeError):
        return {}


def get_remote_url_and_integrity(asset) -> Tuple[Optional[str], Optional[str]]:
    """
    Extract remote URL and integrity hash from an asset object.
    Returns a tuple of (url, integrity_hash) or (None, None) if not remote.
    """
    try:
        attrs = asset.iommi_namespace.attrs

        # Check for href attribute (e.g., stylesheets)
        if hasattr(attrs, "href"):
            return attrs.href, None

        # Check for src attribute (e.g., scripts)
        if hasattr(attrs, "src") and type(attrs.src) is str:
            integrity = getattr(attrs, "integrity", None)
            return attrs.src, integrity

        return None, None

    except AttributeError as e:
        logger.error(f"AttributeError: {e}")
        return None, None


def remote_resources_for_style(style) -> List[Tuple[str, Optional[str]]]:
    """
    Get all remote resources from a style.
    Returns a list of (url, integrity) tuples.
    """
    assets = get_style_assets(style)
    logger.debug(f"Processing assets for {style}")
    resources = []

    for name, asset in assets.items():
        logger.debug(f"Processing asset {name}")
        url, integrity = get_remote_url_and_integrity(asset)
        if url:
            resources.append((url, integrity))

    return resources


def verify_integrity(content: bytes, integrity: str) -> bool:
    """Verify content matches the specified integrity hash"""
    try:
        algo, expected_hash = integrity.split("-", 1)
        expected_hash = base64.b64decode(expected_hash)

        if algo == "sha384":
            hasher = hashlib.sha384()
        elif algo == "sha256":
            hasher = hashlib.sha256()
        else:
            raise ValueError(f"Unsupported hash algorithm: {algo}")

        hasher.update(content)
        return hasher.digest() == expected_hash
    except Exception as e:
        logger.error(f"Error verifying integrity: {e}")
        return False


class Command(BaseCommand):
    help = "Downloads remote assets (CSS, JS) for a specified iommi style"

    def add_arguments(self, parser):
        parser.add_argument(
            "styles",
            nargs="+",
            type=str,
            help="Name(s) of the iommi style(s) to download assets for",
        )
        parser.add_argument(
            "--destination",
            type=str,
            default="static/iommi/styles",
            help="Destination directory for downloaded assets (relative to Django project root). Each style's assets will be saved into its own subdirectory",
        )
        parser.add_argument(
            "--skip-existing",
            action="store_true",
            help="Skip downloading files that already exist locally",
        )

    def handle(self, *args, **options):
        styles = options["styles"]
        destination = options["destination"]
        skip_existing = options["skip_existing"]

        try:
            from iommi.style import get_global_style

            for style_name in styles:
                self.stdout.write(f"Processing style: {style_name}")

                style = get_global_style(style_name)
                if not style:
                    self.stdout.write(
                        self.style.WARNING(f'Style "{style_name}" not found, skipping')
                    )
                    continue

                resources = remote_resources_for_style(style)
                if not resources:
                    self.stdout.write(
                        self.style.WARNING(
                            f'No remote resources found for style "{style_name}"'
                        )
                    )
                    continue

                # Create destination directory if it doesn't exist
                style_dir = os.path.join(destination, style_name)
                os.makedirs(style_dir, exist_ok=True)

                # Download each resource
                for url, integrity in resources:
                    parsed_url = urlparse(url)
                    filename = unquote(os.path.basename(parsed_url.path))
                    filepath = os.path.join(style_dir, filename)

                    if skip_existing and os.path.exists(filepath):
                        if integrity:
                            # For files with integrity check, verify the existing file
                            try:
                                with open(filepath, "rb") as f:
                                    existing_content = f.read()
                                if verify_integrity(existing_content, integrity):
                                    self.stdout.write(
                                        f"Skipping existing file (integrity verified): {filename}"
                                    )
                                    continue
                                logger.debug(
                                    f"Integrity check failed for existing file {filename}, will download again"
                                )
                            except Exception as e:
                                logger.error(
                                    f"Error reading existing file {filename}: {e}"
                                )
                        else:
                            self.stdout.write(f"Skipping existing file: {filename}")
                            continue

                    try:
                        with urlopen(url, timeout=30) as response:
                            content = response.read()

                            # If integrity is provided, verify it
                            if integrity:
                                if not verify_integrity(content, integrity):
                                    raise ValueError("Integrity check failed")

                            with open(filepath, "wb") as f:
                                f.write(content)

                            self.stdout.write(
                                self.style.SUCCESS(f"Downloaded: {filename}")
                            )

                    except (URLError, HTTPError) as e:
                        logger.error(f"Failed to download {url}: {str(e)}")
                        self.stdout.write(
                            self.style.ERROR(f"Failed to download {filename}: {str(e)}")
                        )
                    except ValueError as e:
                        logger.error(f"Integrity check failed for {url}: {str(e)}")
                        self.stdout.write(
                            self.style.ERROR(
                                f"Integrity check failed for {filename}: {str(e)}"
                            )
                        )

                self.stdout.write(
                    self.style.SUCCESS(
                        f'Finished downloading assets for style "{style_name}"'
                    )
                )

            self.stdout.write(self.style.SUCCESS("All styles processed"))

        except ImportError as e:
            raise CommandError(
                "Failed to import required iommi modules. Is iommi installed?"
            ) from e
        except Exception as e:
            logger.error(f"Unexpected exception: {e}", e)
            raise CommandError(f"An error occurred: {str(e)}") from e

If you're interested in having something like this, let me know here and I'll put it into a PR so it can actually be reviewed.

[boxed]

I like it. It's certainly a good start.

I think get_style_assets needs to loop over the entire style definition recursively though. For example the select2 assets are defined inside the definitions for choice_queryset.