openpathsampling · dwhswenson · Sep 26, 2020 · Mar 13, 2020 · Mar 18, 2020 · May 14, 2020
diff --git a/.travis.yml b/.travis.yml
@@ -45,7 +45,7 @@ script:
   - py.test -vv --cov --cov-report xml:cov.xml
 
 after_success:
-  - COVERALLS_PARALLEL=true coveralls
+  - bash <(curl -s https://codecov.io/bash)
 
 import:
-  - dwhswenson/autorelease:autorelease-travis.yml@v0.2.0
+  - dwhswenson/autorelease:autorelease-travis.yml@v0.2.1
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 [![Build Status](https://travis-ci.com/openpathsampling/openpathsampling-cli.svg?branch=master)](https://travis-ci.com/openpathsampling/openpathsampling-cli)
 [![Documentation Status](https://readthedocs.org/projects/openpathsampling-cli/badge/?version=latest)](https://openpathsampling-cli.readthedocs.io/en/latest/?badge=latest)
-[![Coverage Status](https://coveralls.io/repos/github/openpathsampling/openpathsampling-cli/badge.svg?branch=master)](https://coveralls.io/github/openpathsampling/openpathsampling-cli?branch=master)
+[![Coverage Status](https://codecov.io/gh/openpathsampling/openpathsampling-cli/branch/master/graph/badge.svg)](https://codecov.io/gh/openpathsampling/openpathsampling-cli)
 [![Maintainability](https://api.codeclimate.com/v1/badges/0d1ee29e1a05cfcdc01a/maintainability)](https://codeclimate.com/github/openpathsampling/openpathsampling-cli/maintainability)
 
 # OpenPathSampling CLI

diff --git a/paths_cli/cli.py b/paths_cli/cli.py
@@ -7,23 +7,15 @@
 import logging
 import logging.config
 import os
+import pathlib
 
 import click
 # import click_completion
 # click_completion.init()
 
-CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
-
-_POSSIBLE_PLUGIN_FOLDERS = [
-    os.path.join(os.path.dirname(__file__), 'commands'),
-    os.path.join(click.get_app_dir("OpenPathSampling"), 'cli-plugins'),
-    os.path.join(click.get_app_dir("OpenPathSampling", force_posix=True),
-                 'cli-plugins'),
-]
-
-OPSPlugin = collections.namedtuple("OPSPlugin",
-                                   ['name', 'filename', 'func', 'section'])
+from .plugin_management import FilePluginLoader, NamespacePluginLoader
 
+CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
 
 class OpenPathSamplingCLI(click.MultiCommand):
     """Main class for the command line interface
@@ -32,13 +24,20 @@ class OpenPathSamplingCLI(click.MultiCommand):
     """
     def __init__(self, *args, **kwargs):
         # the logic here is all about loading the plugins
-        self.plugin_folders = []
-        for folder in _POSSIBLE_PLUGIN_FOLDERS:
-            if folder not in self.plugin_folders and os.path.exists(folder):
-                self.plugin_folders.append(folder)
+        commands = str(pathlib.Path(__file__).parent.resolve() / 'commands')
+        def app_dir_plugins(posix):
+            return str(pathlib.Path(
+                click.get_app_dir("OpenPathSampling", force_posix=posix)
+            ).resolve() / 'cli-plugins')
+
+        self.plugin_loaders = [
+            FilePluginLoader(commands),
+            FilePluginLoader(app_dir_plugins(posix=False)),
+            FilePluginLoader(app_dir_plugins(posix=True)),
+            NamespacePluginLoader('paths_cli.plugins')
+        ]
 
-        plugin_files = self._list_plugin_files(self.plugin_folders)
-        plugins = self._load_plugin_files(plugin_files)
+        plugins = sum([loader() for loader in self.plugin_loaders], [])
 
         self._get_command = {}
         self._sections = collections.defaultdict(list)
@@ -62,45 +61,6 @@ def _deregister_plugin(self, plugin):
     def plugin_for_command(self, command_name):
         return {p.name: p for p in self.plugins}[command_name]
 
-    @staticmethod
-    def _list_plugin_files(plugin_folders):
-        def is_plugin(filename):
-            return (
-                filename.endswith(".py") and not filename.startswith("_")
-                and not filename.startswith(".")
-            )
-
-        plugin_files = []
-        for folder in plugin_folders:
-            files = [os.path.join(folder, f) for f in os.listdir(folder)
-                     if is_plugin(f)]
-            plugin_files += files
-        return plugin_files
-
-    @staticmethod
-    def _filename_to_command_name(filename):
-        command_name = filename[:-3]  # get rid of .py
-        command_name = command_name.replace('_', '-')  # commands use -
-        return command_name
-
-    @staticmethod
-    def _load_plugin(name):
-        ns = {}
-        with open(name) as f:
-            code = compile(f.read(), name, 'exec')
-            eval(code, ns, ns)
-        return ns['CLI'], ns['SECTION']
-
-    def _load_plugin_files(self, plugin_files):
-        plugins = []
-        for full_name in plugin_files:
-            _, filename = os.path.split(full_name)
-            command_name = self._filename_to_command_name(filename)
-            func, section = self._load_plugin(full_name)
-            plugins.append(OPSPlugin(name=command_name, filename=full_name,
-                                     func=func, section=section))
-        return plugins
-
     def list_commands(self, ctx):
         return list(self._get_command.keys())
 

diff --git a/paths_cli/commands/pathsampling.py b/paths_cli/commands/pathsampling.py
@@ -15,15 +15,15 @@
 @SCHEME.clicked(required=False)
 @INIT_CONDS.clicked(required=False)
 @N_STEPS_MC
-def path_sampling(input_file, output_file, scheme, init_conds, nsteps):
+def pathsampling(input_file, output_file, scheme, init_conds, nsteps):
     """General path sampling, using setup in INPUT_FILE"""
     storage = INPUT_FILE.get(input_file)
-    path_sampling_main(output_storage=OUTPUT_FILE.get(output_file),
-                       scheme=SCHEME.get(storage, scheme),
-                       init_conds=INIT_CONDS.get(storage, init_conds),
-                       n_steps=nsteps)
+    pathsampling_main(output_storage=OUTPUT_FILE.get(output_file),
+                      scheme=SCHEME.get(storage, scheme),
+                      init_conds=INIT_CONDS.get(storage, init_conds),
+                      n_steps=nsteps)
 
-def path_sampling_main(output_storage, scheme, init_conds, n_steps):
+def pathsampling_main(output_storage, scheme, init_conds, n_steps):
     import openpathsampling as paths
     init_conds = scheme.initial_conditions_from_trajectories(init_conds)
     simulation = paths.PathSampling(
@@ -37,6 +37,6 @@ def path_sampling_main(output_storage, scheme, init_conds, n_steps):
     return simulation.sample_set, simulation
 
 
-CLI = path_sampling
+CLI = pathsampling
 SECTION = "Simulation"
 REQUIRES_OPS = (1, 0)
diff --git a/paths_cli/plugin_management.py b/paths_cli/plugin_management.py
@@ -0,0 +1,140 @@
+import collections
+import pkgutil
+import importlib
+import os
+
+OPSPlugin = collections.namedtuple(
+    "OPSPlugin", ['name', 'location', 'func', 'section', 'plugin_type']
+)
+
+class CLIPluginLoader(object):
+    """Abstract object for CLI plugins
+
+    The overall approach involves 5 steps, each of which can be overridden:
+
+    1. Find candidate plugins (which must be Python modules)
+    2. Load the namespaces associated into a dict (nsdict)
+    3. Based on those namespaces, validate that the module *is* a plugin
+    4. Get the associated command name
+    5. Return an OPSPlugin object for each plugin
+
+    Details on steps 1, 2, and 4 differ based on whether this is a
+    filesystem-based plugin or a namespace-based plugin.
+    """
+    def __init__(self, plugin_type, search_path):
+        self.plugin_type = plugin_type
+        self.search_path = search_path
+
+    def _find_candidates(self):
+        raise NotImplementedError()
+
+    @staticmethod
+    def _make_nsdict(candidate):
+        raise NotImplementedError()
+
+    @staticmethod
+    def _validate(nsdict):
+        for attr in ['CLI', 'SECTION']:
+            if attr not in nsdict:
+                return False
+        return True
+
+    def _get_command_name(self, candidate):
+        raise NotImplementedError()
+
+    def _find_valid(self):
+        candidates = self._find_candidates()
+        namespaces = {cand: self._make_nsdict(cand) for cand in candidates}
+        valid = {cand: ns for cand, ns in namespaces.items()
+                 if self._validate(ns)}
+        return valid
+
+    def __call__(self):
+        valid = self._find_valid()
+        plugins = [
+            OPSPlugin(name=self._get_command_name(cand),
+                      location=cand,
+                      func=ns['CLI'],
+                      section=ns['SECTION'],
+                      plugin_type=self.plugin_type)
+            for cand, ns in valid.items()
+        ]
+        return plugins
+
+
+class FilePluginLoader(CLIPluginLoader):
+    """File-based plugins (quick and dirty)
+
+    Parameters
+    ----------
+    search_path : str
+        path to the directory that contains plugins (OS-dependent format)
+    """
+    def __init__(self, search_path):
+        super().__init__(plugin_type="file", search_path=search_path)
+
+    def _find_candidates(self):
+        def is_plugin(filename):
+            return (
+                filename.endswith(".py") and not filename.startswith("_")
+                and not filename.startswith(".")
+            )
+
+        if not os.path.exists(os.path.join(self.search_path)):
+            return []
+
+        candidates = [os.path.join(self.search_path, f)
+                      for f in os.listdir(self.search_path)
+                      if is_plugin(f)]
+        return candidates
+
+    @staticmethod
+    def _make_nsdict(candidate):
+        ns = {}
+        with open(candidate) as f:
+            code = compile(f.read(), candidate, 'exec')
+            eval(code, ns, ns)
+        return ns
+
+    def _get_command_name(self, candidate):
+        _, command_name = os.path.split(candidate)
+        command_name = command_name[:-3]  # get rid of .py
+        command_name = command_name.replace('_', '-')  # commands use -
+        return command_name
+
+
+class NamespacePluginLoader(CLIPluginLoader):
+    """Load namespace plugins (plugins for wide distribution)
+
+    Parameters
+    ----------
+    search_path : str
+        namespace (dot-separated) where plugins can be found
+    """
+    def __init__(self, search_path):
+        super().__init__(plugin_type="namespace", search_path=search_path)
+
+    def _find_candidates(self):
+        # based on https://packaging.python.org/guides/creating-and-discovering-plugins/#using-namespace-packages
+        def iter_namespace(ns_pkg):
+            return pkgutil.iter_modules(ns_pkg.__path__,
+                                        ns_pkg.__name__ + ".")
+
+        ns = importlib.import_module(self.search_path)
+        candidates = [
+            importlib.import_module(name)
+            for _, name, _ in iter_namespace(ns)
+        ]
+        return candidates
+
+    @staticmethod
+    def _make_nsdict(candidate):
+        return vars(candidate)
+
+    def _get_command_name(self, candidate):
+        # +1 for the dot
+        command_name = candidate.__name__
+        command_name = command_name[len(self.search_path) + 1:]
+        command_name = command_name.replace('_', '-')  # commands use -
+        return command_name
+
diff --git a/paths_cli/plugins/__init__.py b/paths_cli/plugins/__init__.py
diff --git a/paths_cli/tests/commands/test_contents.py b/paths_cli/tests/commands/test_contents.py
@@ -0,0 +1,42 @@
+import os
+import tempfile
+import pytest
+from unittest.mock import patch
+from click.testing import CliRunner
+
+import openpathsampling as paths
+
+from paths_cli.commands.contents import *
+
+def test_contents(tps_fixture):
+    # we just do a full integration test of this one
+    scheme, network, engine, init_conds = tps_fixture
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        storage = paths.Storage("setup.nc", 'w')
+        for obj in tps_fixture:
+            storage.save(obj)
+        storage.tags['initial_conditions'] = init_conds
+
+        results = runner.invoke(contents, ['setup.nc'])
+        cwd = os.getcwd()
+        expected = [
+            f"Storage @ '{cwd}/setup.nc'",
+            "CVs: 1 item", "* x",
+            "Volumes: 8 items", "* A", "* B", "* plus 6 unnamed items",
+            "Engines: 2 items", "* flat", "* plus 1 unnamed item",
+            "Networks: 1 item", "* 1 unnamed item",
+            "Move Schemes: 1 item", "* 1 unnamed item",
+            "Simulations: 0 items",
+            "Tags: 1 item", "* initial_conditions",
+            "", "Data Objects:",
+            "Steps: 0 unnamed items",
+            "Move Changes: 0 unnamed items",
+            "SampleSets: 1 unnamed item",
+            "Trajectories: 1 unnamed item",
+            f"Snapshots: {2*len(init_conds[0])} unnamed items", ""
+        ]
+        assert results.exit_code == 0
+        assert results.output.split('\n') == expected
+        for truth, beauty in zip(expected, results.output.split('\n')):
+            assert truth == beauty
diff --git a/paths_cli/tests/commands/test_equilibrate.py b/paths_cli/tests/commands/test_equilibrate.py
@@ -0,0 +1,53 @@
+import os
+import pytest
+from unittest.mock import patch
+import tempfile
+from click.testing import CliRunner
+
+from paths_cli.commands.equilibrate import *
+
+import openpathsampling as paths
+
+def print_test(output_storage, scheme, init_conds, multiplier, extra_steps):
+    print(isinstance(output_storage, paths.Storage))
+    print(scheme.__uuid__)
+    print(init_conds.__uuid__)
+    print(multiplier, extra_steps)
+
+
+@patch('paths_cli.commands.equilibrate.equilibrate_main', print_test)
+def test_equilibrate(tps_fixture):
+    # integration test (click and parameters)
+    scheme, network, engine, init_conds = tps_fixture
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        storage = paths.Storage("setup.nc", 'w')
+        for obj in tps_fixture:
+            storage.save(obj)
+        storage.tags['initial_conditions'] = init_conds
+
+        results = runner.invoke(
+            equilibrate,
+            ["setup.nc", "-o", "foo.nc"]
+        )
+        out_str = "True\n{schemeid}\n{condsid}\n1 0\n"
+        expected_output = out_str.format(schemeid=scheme.__uuid__,
+                                         condsid=init_conds.__uuid__)
+        assert results.exit_code == 0
+        assert results.output == expected_output
+
+def test_equilibrate_main(tps_fixture):
+    # smoke test
+    tempdir = tempfile.mkdtemp()
+    store_name = os.path.join(tempdir, "equil.nc")
+    try:
+        storage = paths.Storage(store_name, mode='w')
+        scheme, network, engine, init_conds = tps_fixture
+        equilibrated, sim = equilibrate_main(storage, scheme, init_conds,
+                                             multiplier=1, extra_steps=1)
+        assert isinstance(equilibrated, paths.SampleSet)
+        assert isinstance(sim, paths.PathSampling)
+    finally:
+        if os.path.exists(store_name):
+            os.remove(store_name)
+        os.rmdir(tempdir)