Add generating from archive option, fix exceptions, caused by multiline content in rpm spec files, remove shebangs, fix missing provides_extra field, fix username empty string, add maintainer option, add --local and --localfile in show subcommand, make name attribute optional if --local or --localfile field is provided

py2pack/__init__.py

@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
 # Copyright (c) 2013, Sascha Peilicke <[email protected]>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -40,6 +37,22 @@
                            get_metadata)
 
 from email import parser
+import tarfile
+import zipfile
+from packaging.requirements import Requirement
+from os.path import basename
+from io import StringIO
+
+
+try:
+    import distro
+    DEFAULT_TEMPLATE = {
+        'fedora': 'fedora.spec',
+        'debian': 'opensuse.dsc',
+        'mageia': 'mageia.spec'
+    }.get(distro.id(), 'opensuse.spec')
+except ModuleNotFoundError:
+    DEFAULT_TEMPLATE = 'opensuse.spec'
 
 
 def replace_string(output_string, replaces):
@@ -68,8 +81,14 @@ def pypi_json(project, release=None):
 
 
 def pypi_text_file(pkg_info_path):
-    with open(pkg_info_path, 'r') as pkg_info_file:
-        pkg_info_lines = parser.Parser().parse(pkg_info_file)
+    pkg_info_file = open(pkg_info_path, 'r')
+    text = pypi_text_stream(pkg_info_file)
+    pkg_info_file.close()
+    return text
+
+
+def pypi_text_stream(pkg_info_stream):
+    pkg_info_lines = parser.Parser().parse(pkg_info_stream)
     pkg_info_dict = {}
     for key, value in pkg_info_lines.items():
         key = key.lower().replace('-', '_')
@@ -85,15 +104,44 @@ def pypi_text_file(pkg_info_path):
 
 
 def pypi_json_file(file_path):
-    with open(file_path, 'r') as json_file:
-        js = json.load(json_file)
+    json_file = open(file_path, 'r')
+    js = pypi_json_stream(json_file)
+    json_file.close()
+    return js
+
+
+def pypi_json_stream(json_stream):
+    js = json.load(json_stream)
     if 'info' not in js:
         js = {'info': js}
     if 'urls' not in js:
         js['urls'] = []
     return js
 
 
+def pypi_archive_file_tar(file_path):
+    with tarfile.open(file_path, 'r') as archive:
+        for member in archive.getmembers():
+            if basename(member.name) == 'PKG-INFO':
+                return pypi_text_stream(StringIO(archive.extractfile(member).read().decode()))
+    raise KeyError('PKG-INFO not found on archive ' + file_path)
+
+
+def pypi_archive_file_zip(file_path):
+    with zipfile.ZipFile(file_path, 'r') as archive:
+        for member in archive.namelist():
+            if basename(member) == 'PKG-INFO':
+                return pypi_text_stream(StringIO(archive.open(member).read().decode()))
+    raise KeyError('PKG-INFO not found on archive ' + file_path)
+
+
+def pypi_archive_file(file_path):
+    try:
+        return pypi_archive_file_tar(file_path)
+    except tarfile.ReadError:
+        return pypi_archive_file_zip(file_path)
+
+
 def _get_template_dirs():
     """existing directories where to search for jinja2 templates. The order
     is important. The first found template from the first found dir wins!"""
@@ -111,6 +159,22 @@ def _get_template_dirs():
     ])
 
 
+def fix_data(data):
+    extra_from_req = re.compile(r'''\bextra\s+==\s+["']([^"']+)["']''')
+    extras = []
+    data_info = data["info"]
+    requires_dist = data_info.get("requires_dist", []) or []
+    provides_extra = data_info.get("provides_extra", []) or []
+    for required_dist in requires_dist:
+        req = Requirement(required_dist)
+        if found := extra_from_req.search(str(req.marker)):
+            extras.append(found.group(1))
+    provides_extra = list(sorted(set([*extras, *provides_extra])))
+    data_info["requires_dist"] = requires_dist
+    data_info["provides_extra"] = provides_extra
+    data_info["classifiers"] = (data_info.get("classifiers", []) or [])
+
+
 def list_packages(args=None):
     """query the "Simple API" of PYPI for all packages and print them."""
     print('listing all PyPI packages...')
@@ -142,7 +206,7 @@ def fetch(args):
     print('downloading package {0}-{1}...'.format(args.name, args.version))
     print('from {0}'.format(url['url']))
 
-    with requests.get(url['url']) as r:
+    with requests.get(url['download_url']) as r:
         with open(url['filename'], 'wb') as f:
             f.write(r.content)
 
@@ -357,23 +421,33 @@ def _get_source_url(pypi_name, filename):
         pypi_name[0], pypi_name, filename)
 
 
+def get_user_name(args=None):
+    try:
+        maintainer = args.maintainer
+        if maintainer is not None:
+            return maintainer
+    except Exception:
+        pass
+    return pwd.getpwuid(os.getuid()).pw_name
+
+
 def generate(args):
     # TODO (toabctl): remove this is a later release
     if args.run:
         warnings.warn("the '--run' switch is deprecated and a noop",
                       DeprecationWarning)
 
-    fetch_local_data(args)
+    fetch_data(args)
     if not args.template:
         args.template = file_template_list()[0]
     if not args.filename:
         args.filename = "python-" + args.name + '.' + args.template.rsplit('.', 1)[1]   # take template file ending
     print('generating spec file for {0}...'.format(args.name))
     data = args.fetched_data['info']
     durl = newest_download_url(args)
-    source_url = data['source_url'] = (args.source_url or (durl and durl['url']))
+    source_url = data['source_url'] = (args.source_url or (durl and durl['download_url']))
     data['year'] = datetime.datetime.now().year                             # set current year
-    data['user_name'] = pwd.getpwuid(os.getuid())[4]                        # set system user (packager)
+    data['user_name'] = get_user_name(args)                   # set system user (packager)
     data['summary_no_ending_dot'] = re.sub(r'(.*)\.', r'\g<1>', data.get('summary')) if data.get('summary') else ""
 
     # If package name supplied on command line differs in case from PyPI's one
@@ -407,6 +481,12 @@ def generate(args):
 
     _normalize_license(data)
 
+    for field in ['summary', 'license', 'home_page', 'source_url', 'summary_no_ending_dot']:
+        if field not in data:
+            continue
+        # remove line breaks to avoid multiline rpm spec file
+        data[field + '_singleline'] = str(data[field]).replace('\n', ' ')
+
     env = _prepare_template_env(_get_template_dirs())
     template = env.get_template(args.template)
     result = template.render(data).encode('utf-8')                          # render template and encode properly
@@ -417,31 +497,34 @@ def generate(args):
         outfile.close()
 
 
-def fetch_local_data(args):
-    localfile = args.localfile
+def fetch_data(args):
+    localfile = args.localfile or ''
     local = args.local
 
     if not localfile and local:
         localfile = os.path.join(f'{args.name}.egg-info', 'PKG-INFO')
     if os.path.isfile(localfile):
         try:
-            data = pypi_json_file(localfile)
-        except json.decoder.JSONDecodeError:
-            data = pypi_text_file(localfile)
+            data = pypi_archive_file(localfile)
+        except Exception:
+            try:
+                data = pypi_json_file(localfile)
+            except json.decoder.JSONDecodeError:
+                data = pypi_text_file(localfile)
         args.fetched_data = data
-        args.version = args.fetched_data['info']['version']
-        return
-    fetch_data(args)
-
-
-def fetch_data(args):
-    args.fetched_data = pypi_json(args.name, args.version)
-    urls = args.fetched_data.get('urls', [])
-    if len(urls) == 0:
-        print(f"unable to find a suitable release for {args.name}!")
-        sys.exit(1)
+        data_info = data['info']
+        args.version = data_info['version']
+        args.name = data_info['name']
+        fix_data(data)
     else:
-        args.version = args.fetched_data['info']['version']                 # return current release number
+        data = args.fetched_data = pypi_json(args.name, args.version)
+        urls = data.get('urls', [])
+        if len(urls) == 0:
+            print(f"unable to find a suitable release for {args.name}!")
+            sys.exit(1)
+        else:
+            args.version = data['info']['version']                 # return current release number
+        fix_data(data)
 
 
 def newest_download_url(args):
@@ -452,14 +535,14 @@ def newest_download_url(args):
     if not hasattr(args, "fetched_data"):
         return {}
     for release in args.fetched_data['urls']:     # Check download URLs in releases
-        if release['packagetype'] == 'sdist':                      # Found the source URL we care for
-            release['url'] = _get_source_url(args.name, release['filename'])
+        if release.get('packagetype') == 'sdist' and not release.get('download_url'):                      # Found the source URL we care for
+            release['download_url'] = _get_source_url(args.name, release['filename'])
             return release
     # No PyPI tarball release, let's see if an upstream download URL is provided:
     data = args.fetched_data['info']
-    if 'download_url' in data and data['download_url']:
-        url = data['download_url']
-        return {'url': url,
+    url = data.get('download_url')
+    if url:
+        return {'download_url': url,
                 'filename': os.path.basename(url)}
     return {}                                                               # We're all out of bubblegum
 
@@ -471,6 +554,17 @@ def file_template_list():
     return template_files
 
 
+def Munch(args):
+    import collections
+    d = collections.defaultdict(lambda: None, args.__dict__)
+    return type('Munch', tuple(), {
+        "__getattr__": d.__getitem__,
+        "__setattr__": d.__setitem__,
+        "__getitem__": d.__getitem__,
+        "__setitem__": d.__setitem__,
+        "__contains__": d.__contains__})()
+
+
 def main():
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument('--version', action='version', version='%(prog)s {0}'.format(py2pack_version.version))
@@ -485,8 +579,10 @@ def main():
     parser_search.set_defaults(func=search)
 
     parser_show = subparsers.add_parser('show', help='show metadata for package')
-    parser_show.add_argument('name', help='package name')
+    parser_show.add_argument('name', nargs='?', help='package name')
     parser_show.add_argument('version', nargs='?', help='package version (optional)')
+    parser_show.add_argument('--local', action='store_true', help='get metadata from local package')
+    parser_show.add_argument('--localfile', default='', help='path to the local PKG-INFO or json metadata')
     parser_show.set_defaults(func=show)
 
     parser_fetch = subparsers.add_parser('fetch', help='download package source tarball from PyPI')
@@ -496,13 +592,14 @@ def main():
     parser_fetch.set_defaults(func=fetch)
 
     parser_generate = subparsers.add_parser('generate', help='generate RPM spec or DEB dsc file for a package')
-    parser_generate.add_argument('name', help='package name')
+    parser_generate.add_argument('name', nargs='?', help='package name')
     parser_generate.add_argument('version', nargs='?', help='package version (optional)')
     parser_generate.add_argument('--source-url', default=None, help='source url')
+    parser_generate.add_argument('--maintainer', default=None, help='maintainer name')
     parser_generate.add_argument('--source-glob', help='source glob template')
-    parser_generate.add_argument('--local', action='store_true', help='build from local package')
+    parser_generate.add_argument('--local', action='store_true', help='get metadata from local package')
     parser_generate.add_argument('--localfile', default='', help='path to the local PKG-INFO or json metadata')
-    parser_generate.add_argument('-t', '--template', choices=file_template_list(), default='opensuse.spec', help='file template')
+    parser_generate.add_argument('-t', '--template', choices=file_template_list(), default=DEFAULT_TEMPLATE, help='file template')
     parser_generate.add_argument('-f', '--filename', help='spec filename (optional)')
     # TODO (toabctl): remove this is a later release
     parser_generate.add_argument(
@@ -513,7 +610,7 @@ def main():
     parser_help = subparsers.add_parser('help', help='show this help')
     parser_help.set_defaults(func=lambda args: parser.print_help())
 
-    args = parser.parse_args()
+    args = Munch(parser.parse_args())
 
     # set HTTP proxy if one is provided
     if args.proxy:
@@ -526,6 +623,13 @@ def main():
 
     if 'func' not in args:
         sys.exit(parser.print_help())
+
+    namestr = args.func.__name__
+    # Custom validation logic
+    if namestr in {'generate', 'show'}:
+        if args.localfile == '' and not args.local and not args.name:
+            subparsers.choices[namestr].error("The name argument is required if not --local or --localfile is provided.")
+
     args.func(args)