Fix plugin management bugs in nikola plugin and nikola import_wordpress (fix #3737)

CHANGES.txt

@@ -9,6 +9,8 @@ Features
 Bugfixes
 --------
 
+* Remove insecure HTTP fallback from ``nikola plugin``
+* Fix the ``nikola plugin`` command not working (Issue #3736, #3737)
 * Fix ``nikola new_post --available-formats`` crashing with TypeError
   (Issue #3750)
 * Fix the new plugin manager not loading plugins if the plugin folder is a symlink (Issue #3741)

nikola/plugin_categories.py

@@ -61,6 +61,7 @@ class BasePlugin:
     """Base plugin class."""
 
     logger = None
+    site: 'nikola.nikola.Nikola'
 
     def set_site(self, site):
         """Set site, which is a Nikola instance."""

nikola/plugin_manager.py

@@ -78,6 +78,7 @@ class PluginInfo:
     category: str
     compiler: Optional[str]
     source_dir: Path
+    py_file_location: Path
     module_name: str
     module_object: object
     plugin_object: BasePlugin
@@ -157,9 +158,10 @@ def locate_plugins(self) -> List[PluginCandidate]:
             )
         return self.candidates
 
-    def load_plugins(self, candidates: List[PluginCandidate]) -> None:
+    def load_plugins(self, candidates: List[PluginCandidate]) -> List[PluginInfo]:
         """Load selected candidate plugins."""
         plugins_root = Path(__file__).parent.parent
+        new_plugins = []
 
         for candidate in candidates:
             name = candidate.name
@@ -234,11 +236,13 @@ def load_plugins(self, candidates: List[PluginCandidate]) -> None:
                 category=candidate.category,
                 compiler=candidate.compiler,
                 source_dir=source_dir,
+                py_file_location=py_file_location,
                 module_name=module_name,
                 module_object=module_object,
                 plugin_object=plugin_object,
             )
             self.plugins.append(info)
+            new_plugins.append(info)
 
         self._plugins_by_category = {category: [] for category in CATEGORY_NAMES}
         for plugin_info in self.plugins:
@@ -251,6 +255,8 @@ def load_plugins(self, candidates: List[PluginCandidate]) -> None:
             self.logger.warning("Waiting 2 seconds before continuing.")
             time.sleep(2)
 
+        return new_plugins
+
     def get_plugins_of_category(self, category: str) -> List[PluginInfo]:
         """Get loaded plugins of a given category."""
         return self._plugins_by_category.get(category, [])

nikola/plugins/command/import_wordpress.py

@@ -38,7 +38,7 @@
 import requests
 from lxml import etree
 
-from nikola.plugin_categories import Command
+from nikola.plugin_categories import Command, CompilerExtension
 from nikola import utils, hierarchy_utils
 from nikola.nikola import DEFAULT_TRANSLATIONS_PATTERN
 from nikola.utils import req_missing
@@ -68,12 +68,8 @@ def install_plugin(site, plugin_name, output_dir=None, show_install_notes=False)
     # Get hold of the 'plugin' plugin
     plugin_installer_info = site.plugin_manager.get_plugin_by_name('plugin', 'Command')
     if plugin_installer_info is None:
-        LOGGER.error('Internal error: cannot find the "plugin" plugin which is supposed to come with Nikola!')
+        LOGGER.error('Internal error: cannot find the "plugin" plugin which is supposed to come with Nikola - it might be disabled in conf.py')
         return False
-    if not plugin_installer_info.is_activated:
-        # Someone might have disabled the plugin in the `conf.py` used
-        site.plugin_manager.activatePluginByName(plugin_installer_info.name)
-        plugin_installer_info.plugin_object.set_site(site)
     plugin_installer = plugin_installer_info.plugin_object
     # Try to install the requested plugin
     options = {}
@@ -85,9 +81,16 @@ def install_plugin(site, plugin_name, output_dir=None, show_install_notes=False)
     if plugin_installer.execute(options=options) > 0:
         return False
     # Let the plugin manager find newly installed plugins
-    site.plugin_manager.collectPlugins()
-    # Re-scan for compiler extensions
-    site.compiler_extensions = site._activate_plugins_of_category("CompilerExtension")
+    old_candidates = set(site.plugin_manager.candidates)
+    new_candidates = set(site.plugin_manager.locate_plugins())
+    missing_candidates = list(new_candidates - old_candidates)
+    new_plugins = site.plugin_manager.load_plugins(missing_candidates)
+
+    # Activate new plugins
+    for p in new_plugins:
+        site._activate_plugin(p)
+        if isinstance(p.plugin_object, CompilerExtension):
+            site.compiler_extensions.append(p)
     return True
 
 
@@ -248,12 +251,16 @@ def _find_wordpress_compiler(self):
         """Find WordPress compiler plugin."""
         if self.wordpress_page_compiler is not None:
             return
+
         plugin_info = self.site.plugin_manager.get_plugin_by_name('wordpress', 'PageCompiler')
-        if plugin_info is not None:
-            if not plugin_info.is_activated:
-                self.site.plugin_manager.activatePluginByName(plugin_info.name)
-                plugin_info.plugin_object.set_site(self.site)
-            self.wordpress_page_compiler = plugin_info.plugin_object
+        if plugin_info is None:
+            candidates = self.site.plugin_manager.locate_plugins()
+            wordpress_candidates = [c for c in candidates if c.name == "wordpress" and c.category == "PageCompiler"]
+            if wordpress_candidates:
+                new_plugins = self.site.plugin_manager.load_plugins(wordpress_candidates)
+                for p in new_plugins:
+                    self.site._activate_plugin(p)
+                    self.wordpress_page_compiler = p
 
     def _read_options(self, options, args):
         """Read command-line options."""

nikola/plugins/command/plugin.py

@@ -28,11 +28,12 @@
 
 import io
 import json.decoder
-import os
+import pathlib
 import sys
 import shutil
 import subprocess
-import time
+import typing
+
 import requests
 
 import pygments
@@ -52,8 +53,9 @@ class CommandPlugin(Command):
     name = "plugin"
     doc_usage = "[-u url] [--user] [-i name] [-r name] [--upgrade] [-l] [--list-installed]"
     doc_purpose = "manage plugins"
-    output_dir = None
+    output_dir: pathlib.Path = None
     needs_config = False
+    never_upgrade = {'emoji'}  # plugin with the same name is shipped with Nikola
     cmd_options = [
         {
             'name': 'install',
@@ -133,16 +135,16 @@ def _execute(self, options, args):
             return 2
 
         if options.get('output_dir') is not None:
-            self.output_dir = options.get('output_dir')
+            self.output_dir = pathlib.Path(options.get('output_dir'))
         else:
             if not self.site.configured and not user_mode and install:
                 LOGGER.warning('No site found, assuming --user')
                 user_mode = True
 
             if user_mode:
-                self.output_dir = os.path.expanduser(os.path.join('~', '.nikola', 'plugins'))
+                self.output_dir = pathlib.Path.home() / ".nikola" / "plugins"
             else:
-                self.output_dir = 'plugins'
+                self.output_dir = pathlib.Path("plugins")
 
         if list_available:
             return self.list_available(url)
@@ -166,14 +168,7 @@ def list_available(self, url):
 
     def list_installed(self):
         """List installed plugins."""
-        plugins = []
-        for plugin in self.site.plugin_manager.getAllPlugins():
-            p = plugin.path
-            if os.path.isdir(p):
-                p = p + os.sep
-            else:
-                p = p + '.py'
-            plugins.append([plugin.name, p])
+        plugins = self.get_plugins()
 
         plugins.sort()
         print('Installed Plugins:')
@@ -196,25 +191,18 @@ def do_upgrade(self, url):
         """Upgrade all installed plugins."""
         LOGGER.warning('This is not very smart, it just reinstalls some plugins and hopes for the best')
         data = self.get_json(url)
-        plugins = []
-        for plugin in self.site.plugin_manager.getAllPlugins():
-            p = plugin.path
-            if os.path.isdir(p):
-                p = p + os.sep
-            else:
-                p = p + '.py'
-            if plugin.name in data:
-                plugins.append([plugin.name, p])
-        print('Will upgrade {0} plugins: {1}'.format(len(plugins), ', '.join(n for n, _ in plugins)))
+        plugins = [(n, p) for n, p in self.get_plugins() if n in data and n not in self.never_upgrade]
+        LOGGER.info('Will upgrade {0} plugins: {1}'.format(len(plugins), ', '.join(n for n, _ in plugins)))
         for name, path in plugins:
-            print('Upgrading {0}'.format(name))
+            path: pathlib.Path
+            LOGGER.info('Upgrading {0}'.format(name))
             p = path
             while True:
-                tail, head = os.path.split(path)
+                tail, head = path.parent, path.name
                 if head == 'plugins':
                     self.output_dir = path
                     break
-                elif tail == '':
+                elif path == tail:
                     LOGGER.error("Can't find the plugins folder for path: {0}".format(p))
                     return 1
                 else:
@@ -229,104 +217,83 @@ def do_install(self, url, name, show_install_notes=True):
             utils.makedirs(self.output_dir)
             url = data[name]
             LOGGER.info("Downloading '{0}'".format(url))
-            try:
-                zip_data = requests.get(url).content
-            except requests.exceptions.SSLError:
-                LOGGER.warning("SSL error, using http instead of https (press ^C to abort)")
-                time.sleep(1)
-                url = url.replace('https', 'http', 1)
-                zip_data = requests.get(url).content
+            zip_data = requests.get(url).content
 
             zip_file = io.BytesIO()
             zip_file.write(zip_data)
             LOGGER.info('Extracting: {0} into {1}/'.format(name, self.output_dir))
             utils.extract_all(zip_file, self.output_dir)
-            dest_path = os.path.join(self.output_dir, name)
+            dest_path = self.output_dir / name
         else:
             LOGGER.error("Can't find plugin " + name)
             return 1
 
-        reqpath = os.path.join(dest_path, 'requirements.txt')
-        if os.path.exists(reqpath):
+        requirements_path = dest_path / 'requirements.txt'
+        if requirements_path.exists():
             LOGGER.warning('This plugin has Python dependencies.')
             LOGGER.info('Installing dependencies with pip...')
             try:
-                subprocess.check_call((sys.executable, '-m', 'pip', 'install', '-r', reqpath))
+                subprocess.check_call((sys.executable, '-m', 'pip', 'install', '-r', str(requirements_path)))
             except subprocess.CalledProcessError:
                 LOGGER.error('Could not install the dependencies.')
                 print('Contents of the requirements.txt file:\n')
-                with io.open(reqpath, 'r', encoding='utf-8-sig') as fh:
-                    print(utils.indent(fh.read(), 4 * ' '))
-                print('You have to install those yourself or through a '
-                      'package manager.')
+                print(utils.indent(requirements_path.read_text(), 4 * ' '))
+                print('You have to install those yourself or through a package manager.')
             else:
                 LOGGER.info('Dependency installation succeeded.')
 
-        reqnpypath = os.path.join(dest_path, 'requirements-nonpy.txt')
-        if os.path.exists(reqnpypath):
-            LOGGER.warning('This plugin has third-party '
-                           'dependencies you need to install '
-                           'manually.')
+        requirements_nonpy_path = dest_path / 'requirements-nonpy.txt'
+        if requirements_nonpy_path.exists():
+            LOGGER.warning('This plugin has third-party dependencies you need to install manually.')
             print('Contents of the requirements-nonpy.txt file:\n')
-            with io.open(reqnpypath, 'r', encoding='utf-8-sig') as fh:
-                for l in fh.readlines():
-                    i, j = l.split('::')
-                    print(utils.indent(i.strip(), 4 * ' '))
-                    print(utils.indent(j.strip(), 8 * ' '))
-                    print()
+            for l in requirements_nonpy_path.read_text().strip().splitlines():
+                i, j = l.split('::')
+                print(utils.indent(i.strip(), 4 * ' '))
+                print(utils.indent(j.strip(), 8 * ' '))
+                print()
 
             print('You have to install those yourself or through a package '
                   'manager.')
 
-        req_plug_path = os.path.join(dest_path, 'requirements-plugins.txt')
-        if os.path.exists(req_plug_path):
+        requirements_plugins_path = dest_path / 'requirements-plugins.txt'
+        if requirements_plugins_path.exists():
             LOGGER.info('This plugin requires other Nikola plugins.')
             LOGGER.info('Installing plugins...')
             plugin_failure = False
             try:
-                with io.open(req_plug_path, 'r', encoding='utf-8-sig') as inf:
-                    for plugname in inf.readlines():
-                        plugin_failure = self.do_install(url, plugname.strip(), show_install_notes) != 0
+                for plugin_name in requirements_plugins_path.read_text().strip().splitlines():
+                    plugin_failure = self.do_install(url, plugin_name.strip(), show_install_notes) != 0
             except Exception:
                 plugin_failure = True
             if plugin_failure:
                 LOGGER.error('Could not install a plugin.')
                 print('Contents of the requirements-plugins.txt file:\n')
-                with io.open(req_plug_path, 'r', encoding='utf-8-sig') as fh:
-                    print(utils.indent(fh.read(), 4 * ' '))
+                print(utils.indent(requirements_plugins_path.read_text(), 4 * ' '))
                 print('You have to install those yourself manually.')
             else:
                 LOGGER.info('Dependency installation succeeded.')
 
-        confpypath = os.path.join(dest_path, 'conf.py.sample')
-        if os.path.exists(confpypath) and show_install_notes:
+        confpy_path = dest_path / 'conf.py.sample'
+        if confpy_path.exists() and show_install_notes:
             LOGGER.warning('This plugin has a sample config file.  Integrate it with yours in order to make this plugin work!')
             print('Contents of the conf.py.sample file:\n')
-            with io.open(confpypath, 'r', encoding='utf-8-sig') as fh:
-                if self.site.colorful:
-                    print(pygments.highlight(fh.read(), PythonLexer(), TerminalFormatter()))
-                else:
-                    print(fh.read())
+            if self.site.colorful:
+                print(pygments.highlight(confpy_path.read_text(), PythonLexer(), TerminalFormatter()))
+            else:
+                print(confpy_path.read_text())
         return 0
 
     def do_uninstall(self, name):
         """Uninstall a plugin."""
-        for plugin in self.site.plugin_manager.getAllPlugins():  # FIXME: this is repeated thrice
-            if name == plugin.name:  # Uninstall this one
-                p = plugin.path
-                if os.path.isdir(p):
-                    # Plugins that have a package in them need to delete parent
-                    # Issue #2356
-                    p = p + os.sep
-                    p = os.path.abspath(os.path.join(p, os.pardir))
-                else:
-                    p = os.path.dirname(p)
+        for found_name, path in self.get_plugins():
+            if name == found_name:  # Uninstall this one
+                to_delete = path.parent  # Delete parent of .py file or parent of package
                 LOGGER.warning('About to uninstall plugin: {0}'.format(name))
-                LOGGER.warning('This will delete {0}'.format(p))
+                LOGGER.warning('This will delete {0}'.format(to_delete))
                 sure = utils.ask_yesno('Are you sure?')
                 if sure:
-                    LOGGER.warning('Removing {0}'.format(p))
-                    shutil.rmtree(p)
+                    LOGGER.warning('Removing {0}'.format(to_delete))
+                    shutil.rmtree(to_delete)
                     return 0
                 return 1
         LOGGER.error('Unknown plugin: {0}'.format(name))
@@ -336,19 +303,24 @@ def get_json(self, url):
         """Download the JSON file with all plugins."""
         if self.json is None:
             try:
-                try:
-                    self.json = requests.get(url).json()
-                except requests.exceptions.SSLError:
-                    LOGGER.warning("SSL error, using http instead of https (press ^C to abort)")
-                    time.sleep(1)
-                    url = url.replace('https', 'http', 1)
-                    self.json = requests.get(url).json()
+                self.json = requests.get(url).json()
             except json.decoder.JSONDecodeError as e:
                 LOGGER.error("Failed to decode JSON data in response from server.")
                 LOGGER.error("JSON error encountered: " + str(e))
-                LOGGER.error("This issue might be caused by server-side issues, or by to unusual activity in your "
+                LOGGER.error("This issue might be caused by server-side issues, or by unusual activity in your "
                              "network (as determined by CloudFlare). Please visit https://plugins.getnikola.com/ in "
                              "a browser.")
                 sys.exit(2)
 
         return self.json
+
+    def get_plugins(self) -> typing.List[typing.Tuple[str, pathlib.Path]]:
+        """Get currently installed plugins in site."""
+        plugins = []
+        for plugin in self.site.plugin_manager.plugins:
+            if plugin.py_file_location.name == "__init__.py":
+                path = plugin.py_file_location.parent
+            else:
+                path = plugin.py_file_location
+            plugins.append((plugin.name, path))
+        return plugins