build

#!/usr/bin/env python
# (C) Copyright NuoDB, Inc. 2019  All Rights Reserved.

"""
Command-line tool for building the NuoDB client package.
"""

# Package: Released (publicly downloadable) artifact (like nuodb)
# Stage: Software within a package (like nuosql within nuodb)
# Bundle: Convenience grouping of stages from 1 or more packages
#
# This tool produces one or more bundles that group together
# stages as a convenience for users. Bundles group related stages
# (like client tools or drivers) and all their dependencies into
# a single tarball.
#
# Bundles are defined in the Bundles Enum in client.bundles.
#
# Package information describes how to download and unpack a
# downloadable release. Packages are defined in client.pkg.*.
# Every package defines at least one stage.
#
# Stages are defined in client.pkg.*. Each Stage is labeled
# with a bundle when it is instantiated. A Stage's bundle determines
# which tarball it ends up in.
#
# If you do not want to include a stage in a bundle explicitly,
# set its bundle to None. A stage can still be pulled into a bundle
# as a dependency, but it won't be put into a bundle otherwise.
# A stage that is only pulled into a bundle as a dependency will
# not be explicitly listed in the manifest.
#
# Every stage is labeled with a version, but currently we assume
# that every stage in a package has the same version (the version
# of the package).
#
# Bundles are versioned separately from any stage they contain.
# You *must* specify a version to use for *every* bundle generated
# in a single invocation of this build tool. A manifest will be
# generated for each bundle, which will include the version of
# each stage contained therein.
#
# By default all bundles are included in the same .tar.gz. The
# manifest and README generators will use bundle information for
# grouping stages together.
#
# To create one .tar.gz per bundle (.tar.gz name includes the
# bundle name), pass the --separate-bundles option.

import os
import sys
import argparse
import re

from string import Template
from datetime import datetime
from collections import defaultdict
from itertools import starmap

from client.exceptions import ClientError
from client.package import Package
from client.utils import Globals, info
from client.utils import run, runout, mkdir, rmdir, rmfile, rmrf
from client.utils import copyinto, copyfiles, loadfile, savefile

# Import all packages.  This forces them to register themselves.
from client.pkg import *

# Also disable any per-user site-packages for safety
os.environ['PYTHONNOUSERSITE'] = 'true'

os.environ['LC_ALL'] = 'C'
os.environ['LANG'] = 'C'

# Turn off file extension saving for tar (in MacOS)
os.environ['COPYFILE_DISABLE'] = 'true'

USER = 'nuodb'
GROUP = 'nuodb'

PYTHONVERSION = 3

DEFAULT_BUNDLE_NAME = 'client'


def bundle_to_pkgname(bundle, target):
    pkgname_template = 'nuodb-{}-{}.{}'
    b = str(bundle) if Globals.separate_bundles else DEFAULT_BUNDLE_NAME
    return pkgname_template.format(b, Globals.version, target)


def pkgname_to_pkgdir(pkgname):
    return os.path.join(Globals.finalroot, pkgname)


def build_readme(bundle_contents):
    def stage_to_s(stg):
        return '* {} version {} (bundled from: [{}]({}))'.format(
            stg.title,
            stg.version,
            stg.repo_title,
            stg.repo_url)
    def bundle_to_s(title, stages):
        return '### {}\n'.format(title) + '\n'.join(map(stage_to_s, stages))
    readme = '\n\n'.join(starmap(bundle_to_s, bundle_contents.items())) + '\n'
    savefile(os.path.join(Globals.finalroot, 'README.md'),
             readme)


def build_manifest(bundle_version, buildid, commit, pkgname, stages):
    pkgdir = pkgname_to_pkgdir(pkgname)
    stage_names = []
    stage_notes = []
    for stg in sorted(stages, key=lambda x: x.title):
        stage_names.append('  * {} version {}'.format(stg.title, stg.version))
        if stg.notes is None:
            notes = ''
        else:
            notes = '\nNotes:{}'.format(stg.notes)
        stage_notes.append("""{}
{}

Version: {}
Requirements: {}{}

Contents:
  {}""".format(stg.title,
               '-'*len(stg.title),
               stg.version,
               stg.requirements,
               notes,
               '\n  '.join(sorted(stg.getcontents()))))

    # Construct the various values for the README file
    replace = {'VERSION': bundle_version,
               'BUILD': buildid,
               'COMMIT': commit,
               'LICENSE': Package.getlicense('3BSD'),
               'MANIFEST': '\n'.join(stage_names),
               'PACKAGES': '\n\n'.join(stage_notes)}

    readme = loadfile('README.in')
    savefile(os.path.join(pkgdir, 'README.txt'),
             Template(readme).substitute(replace))


def build_clients(packages):
    buildid = Globals.buildid
    (ret, out, err) = runout(['git', 'rev-parse', '--short=10', 'HEAD'])
    commit = out if ret == 0 else 'UNK'

    target = Globals.target

    # This is handled inside Package so it can deal with prerequisites etc.
    Package.build_all(packages)

    # Track all package names to return later
    pkgnames = set()

    # We want to recreate each pkgdir when it is first used
    pkgdir_cleaned = set()
    def clean_pkgdir(d):
        if d in pkgdir_cleaned:
            return
        rmdir(d)
        mkdir(d)
        pkgdir_cleaned.add(d)

    # Now construct the final package from all the individual dist directories
    #
    # tarball_contents is used to build the per tarball README / manifest
    # bundle_contents is used to build the overall README / manifest for the release notes
    #
    # If --separate-packages is set, then tarball_contents and bundle_contents will contain the same contents
    tarball_contents = defaultdict(list)
    bundle_contents = defaultdict(list)
    for pkgnm in packages:
        pkg = Package.get_package(pkgnm)

        info("Installing package {} ...".format(pkg.name))
        for stg in pkg.staged:
            if stg.bundle is None:
                continue
            pkgname = bundle_to_pkgname(stg.bundle, target)
            pkgnames.add(pkgname)
            pkgdir = pkgname_to_pkgdir(pkgname)
            clean_pkgdir(pkgdir)
            copyinto(stg.stagedir, pkgdir)
            tarball_contents[pkgname].append(stg)
            bundle_contents[stg.bundle['title']].append(stg)

    for pkgname, stages in tarball_contents.items():
        build_manifest(Globals.version, buildid, commit, pkgname, stages)
    build_readme(bundle_contents)

    etc = os.path.join(pkgdir, 'etc')
    mkdir(etc)
    if Globals.target.startswith('lin'):
        copyfiles(['nuodb_setup.sh'], Globals.etcdir, etc)
    else:
        copyfiles(['nuodb_setup.bat'], Globals.etcdir, etc)

    return pkgnames


def create_package(pkgname):
    if Globals.target.startswith('lin'):
        out = '{}.tar.gz'.format(pkgname)
        rmfile(out)
        info("Creating {} ...".format(out))
        run(['tar', '-c', '-z', '-f', out, '--owner=%s' % USER, '--group=%s' % GROUP, pkgname], cwd=Globals.finalroot)
    else:
        out = '{}.zip'.format(pkgname)
        rmfile(out)
        info("Creating {} ...".format(out))
        run(['zip', '-r', out, pkgname], cwd=Globals.finalroot)


def main():
    Globals.init(clientroot=os.path.dirname(os.path.realpath(__file__)),
                 pythonversion=PYTHONVERSION)

    # Get a list of possible packages
    packages = Package.get_packages()

    parser = argparse.ArgumentParser(description='Build the NuoDB Client package')

    parser.add_argument(
        '--version',
        help='Version to use for each bundle')

    parser.add_argument(
        "-v",
        "--verbose",
        action="store_true",
        help="Show verbose output")

    parser.add_argument(
        "-p",
        "--platform",
        default='lin-x64',
        choices=['lin-x64', 'lin-arm64', 'win-x64'],
        help="Client platform")

    parser.add_argument(
        "--no-package",
        action="store_true",
        help="Don't create the final tarball/zip file.")

    parser.add_argument(
        "--clean",
        action="store_true",
        help="Clean previous builds.  Downloads are not cleaned.")

    parser.add_argument(
        "--build",
        default=0,
        help="NuoDB Client build number")

    parser.add_argument(
        "--real-clean",
        action="store_true",
        help="Clean previous builds and downloads.")

    parser.add_argument(
        '--separate-bundles',
        action='store_true',
        help='Create a separate .tar.gz per bundle.')

    parser.add_argument(
        'packages',
        metavar='PKGS',
        nargs='*',
        help='Packages to be built')

    options = parser.parse_args()

    kwargs = {'version': options.version,
              'isverbose': options.verbose,
              'target': options.platform,
              'buildid': options.build,
              'separate_bundles': options.separate_bundles}

    for arg in list(options.packages):
        m = re.match(r'([^=]+)=([^\d].*)', arg)
        if m is None:
            if arg != 'all' and arg not in packages:
                sys.exit("Invalid package '%s': must be one of:\n    all, %s"
                         % (arg, ', '.join(packages)))
        else:
            kwargs[m.group(1)] = m.group(2)
            options.packages.remove(arg)

    if not options.packages:
        options.packages = ['all']

    Globals.setup(**kwargs)

    try:
        if options.clean or options.real_clean:
            if 'all' in options.packages:
                info("Cleaning all packages ...")
                rmrf([Globals.tmproot, Globals.finalroot])
                if options.real_clean:
                    info("Cleaning downloads ...")
                    rmdir(Globals.downloadroot)
            else:
                for name in options.packages:
                    pkg = Package.get_package(name)
                    info('{}: Cleaning'.format(name))
                    pkg.clean(real=options.real_clean)
            return

        if options.version is None:
            parser.error('Must specify --version to build packages')

        if 'all' in options.packages:
            options.packages = packages

        pkgnames = build_clients(options.packages)

        if not options.no_package:
            for pkgname in pkgnames:
                create_package(pkgname)

    except ClientError as ex:
        sys.exit("Failed: {}".format(str(ex)))


main()