DeepLabCut · AlexEMG · Nov 4, 2022 · Oct 21, 2022 · Oct 22, 2022 · Oct 22, 2022
diff --git a/README.md b/README.md
@@ -36,7 +36,7 @@ DeepLabCut is a toolbox for markerless pose estimation of animals performing var
 
 :purple_heart: **DeepLabCut supports multi-animal pose estimation!** maDLC is out of beta/rc mode and beta is deprecated, thanks to the testers out there for feedback! Your labeled data will be backwards compatible, but not all other steps. Please see the [new `2.2+` releases](https://github.com/DeepLabCut/DeepLabCut/releases) for what's new & how to install it, please see our new [paper, Lauer et al 2022](https://www.nature.com/articles/s41592-022-01443-0), and the [new docs]( https://deeplabcut.github.io/DeepLabCut) on how to use it!
 
-:purple_heart: We support mulit-animal re-identification, see [Lauer et al 2022](https://www.nature.com/articles/s41592-022-01443-0).
+:purple_heart: We support multi-animal re-identification, see [Lauer et al 2022](https://www.nature.com/articles/s41592-022-01443-0).
 
 :purple_heart: We have a **real-time** package available! http://DLClive.deeplabcut.org
 

diff --git a/_toc.yml b/_toc.yml
@@ -27,6 +27,9 @@ parts:
   chapters:
   - file: docs/recipes/UsingModelZooPupil
   - file: docs/recipes/MegaDetectorDLCLive
+- caption: DeepLabCut Benchmark
+  chapters:
+  - file: docs/benchmark
 - caption: Hardware
   chapters:
   - file: docs/recipes/TechHardware

diff --git a/conda-environments/DEEPLABCUT.yaml b/conda-environments/DEEPLABCUT.yaml
@@ -25,4 +25,4 @@ dependencies:
   - nb_conda
   - ffmpeg
   - pip:
-    - "deeplabcut[gui]"
+    - "deeplabcut[gui,tf]"
diff --git a/conda-environments/DEEPLABCUT_M1.yaml b/conda-environments/DEEPLABCUT_M1.yaml
@@ -8,30 +8,37 @@
 #https://github.com/DeepLabCut/DeepLabCut/blob/master/AUTHORS
 #Licensed under GNU Lesser General Public License v3.0
 #
-# DeepLabCut M1 environment instructions:
-# For the M1 chip, please first download the Rosetta wheel for tensorflow 2.4.1:
-# https://drive.google.com/file/d/17pSwfoNuyf3YR8vCaVggHeI-pMQ3xL7l/view?usp=sharing
+# DeepLabCut M1/M2 (Apple Silicon) environment instructions
 #
-# Then, run:
+# We'll get the miniconda M1 bash installer, as explained in
+# https://docs.conda.io/projects/conda/en/latest/user-guide/install/macos.html
 #
-# conda env create -f DEEPLABCUT_M1.yaml
+# In the Terminal, run the following commands:
+# wget https://repo.anaconda.com/miniconda/Miniconda3-py39_4.12.0-MacOSX-arm64.sh -O ~/miniconda.sh
+# bash ~/miniconda.sh -b -p $HOME/miniconda
+# source ~/miniconda/bin/activate
+# conda init zsh
 #
-# Next, activate the environment, and in the terminal cd to  Downloads, and then run:
-# pip install tensorflow-2.4.1-py3-none-any.whl --no-dependencies --force-reinstall
+# Then, `git clone DeepLabCut`, and run:
 #
-# Next, launch DLC with pythonw -m deeplabcut
+# conda env create -f conda-environments/DEEPLABCUT_M1.yaml
+#
+# Next, activate the environment, and launch DLC with pythonw -m deeplabcut
 
 name: DEEPLABCUT_M1
 channels:
   - conda-forge
   - defaults
 dependencies:
-  - python=3.7
+  - python=3.9
   - pip
   - ipython
   - jupyter
   - nb_conda
   - python.app
   - ffmpeg
+  - apple::tensorflow-deps
   - pip:
-    - "deeplabcut[gui]"
+    - tensorflow-macos
+    - tensorflow-metal
+    - -e ../[gui]
diff --git a/conda-environments/DLC2_3_M1.yaml b/conda-environments/DLC2_3_M1.yaml
@@ -0,0 +1,44 @@
+# DLC2_3_M1.yaml
+
+#DeepLabCut2.0 Toolbox (deeplabcut.org)
+#© A. & M. Mathis Labs
+#https://github.com/DeepLabCut/DeepLabCut
+#Please see AUTHORS for contributors.
+
+#https://github.com/DeepLabCut/DeepLabCut/blob/master/AUTHORS
+#Licensed under GNU Lesser General Public License v3.0
+#
+# DeepLabCut M1/M2 (Apple Silicon) environment instructions
+#
+# We'll get the miniconda M1 bash installer, as explained in
+# https://docs.conda.io/projects/conda/en/latest/user-guide/install/macos.html
+#
+# In the Terminal, run the following commands:
+# wget https://repo.anaconda.com/miniconda/Miniconda3-py39_4.12.0-MacOSX-arm64.sh -O ~/miniconda.sh
+# bash ~/miniconda.sh -b -p $HOME/miniconda
+# source ~/miniconda/bin/activate
+# conda init zsh
+#
+# Then, `git clone DeepLabCut`, and run:
+#
+# conda env create -f conda-environments/DEEPLABCUT_M1.yaml
+#
+# Next, activate the environment, and launch DLC with pythonw -m deeplabcut
+
+name: DLC2_3_M1
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.9
+  - pip
+  - ipython
+  - jupyter
+  - nb_conda
+  - python.app
+  - ffmpeg
+  - apple::tensorflow-deps
+  - pip:
+    - tensorflow-macos
+    - tensorflow-metal
+    - -e ../[gui]
diff --git a/deeplabcut/__main__.py b/deeplabcut/__main__.py
@@ -8,7 +8,7 @@
 Licensed under GNU Lesser General Public License v3.0
 """
 try:
-    import PySide2
+    import PySide6
 
     lite = False
 except ModuleNotFoundError:

diff --git a/deeplabcut/benchmark/__init__.py b/deeplabcut/benchmark/__init__.py
@@ -0,0 +1,116 @@
+# DeepLabCut2.0 Toolbox (deeplabcut.org)
+# © A. & M. Mathis Labs
+# https://github.com/AlexEMG/DeepLabCut
+# Please see AUTHORS for contributors.
+#
+# https://github.com/AlexEMG/DeepLabCut/blob/master/AUTHORS
+# Licensed under GNU Lesser General Public License v3.0
+
+
+import json
+import os
+from typing import Container
+from typing import Literal
+
+from deeplabcut.benchmark.base import Benchmark, Result, ResultCollection
+
+DATA_ROOT = os.path.join(os.getcwd(), "data")
+CACHE = os.path.join(os.getcwd(), ".results")
+
+__registry = []
+
+
+def register(cls):
+    """Add a benchmark to the list of evaluations to run.
+
+    Apply this function as a decorator to a class. Note that the
+    class needs to be a subclass of the ``benchmark.base.Benchmark``
+    base class.
+
+    In most situations, it will be a subclass of one of the pre-defined
+    benchmarks in ``benchmark.benchmarks``.
+
+    Throws:
+        ``ValueError`` if the decorator is applied to a class that is
+        not a subclass of ``benchmark.base.Benchmark``.
+    """
+    if not issubclass(cls, Benchmark):
+        raise ValueError(
+            f"Can only register subclasses of {type(Benchmark)}, " f"but got {cls}."
+        )
+    __registry.append(cls)
+
+
+def evaluate(
+    include_benchmarks: Container[str] = None,
+    results: ResultCollection = None,
+    on_error="return",
+) -> ResultCollection:
+    """Run evaluation for all benchmarks and methods.
+
+    Note that in order for your custom benchmark to be included during
+    evaluation, the following conditions need to be met:
+
+        - The benchmark subclassed one of the benchmark definitions in
+          in ``benchmark.benchmarks``
+        - The benchmark is registered by applying the ``@benchmark.register``
+          decorator to the class
+        - The benchmark was imported. This is done automatically for all
+          benchmarks that are defined in submodules or subpackages of the
+          ``benchmark.submissions`` module. For all other locations, make
+          sure to manually import the packages **before** calling the
+          ``evaluate()`` function.
+
+    Args:
+        include_benchmarks:
+            If ``None``, run all benchmarks that were discovered. If a container
+            is passed, only include methods that were defined on benchmarks with
+            the specified names. E.g., ``include_benchmarks = ["trimouse"]`` would
+            only evaluate methods of the trimouse benchmark dataset.
+        on_error:
+            see documentation in ``benchmark.base.Benchmark.evaluate()``
+
+    Returns:
+        A collection of all results, which can be printed or exported to
+        ``pd.DataFrame`` or ``json`` file formats.
+    """
+    if results is None:
+        results = ResultCollection()
+    for benchmark_cls in __registry:
+        if include_benchmarks is not None:
+            if benchmark_cls.name not in include_benchmarks:
+                continue
+        benchmark = benchmark_cls()
+        for name in benchmark.names():
+            if Result(method_name=name, benchmark_name=benchmark_cls.name) in results:
+                continue
+            else:
+                result = benchmark.evaluate(name, on_error=on_error)
+                results.add(result)
+    return results
+
+
+def get_filepath(basename: str):
+    return os.path.join(DATA_ROOT, basename)
+
+
+def savecache(results: ResultCollection):
+    with open(CACHE, "w") as fh:
+        json.dump(results.todicts(), fh, indent=2)
+
+
+def loadcache(
+    cache=CACHE, on_missing: Literal["raise", "ignore"] = "ignore"
+) -> ResultCollection:
+    if not os.path.exists(cache):
+        if on_missing == "raise":
+            raise FileNotFoundError(cache)
+        return ResultCollection()
+    with open(cache, "r") as fh:
+        try:
+            data = json.load(fh)
+        except json.decoder.JSONDecodeError as e:
+            if on_missing == "raise":
+                raise e
+            return ResultCollection()
+    return ResultCollection.fromdicts(data)
diff --git a/deeplabcut/benchmark/__main__.py b/deeplabcut/benchmark/__main__.py
@@ -0,0 +1,12 @@
+# DeepLabCut2.0 Toolbox (deeplabcut.org)
+# © A. & M. Mathis Labs
+# https://github.com/AlexEMG/DeepLabCut
+# Please see AUTHORS for contributors.
+#
+# https://github.com/AlexEMG/DeepLabCut/blob/master/AUTHORS
+# Licensed under GNU Lesser General Public License v3.0
+
+from deeplabcut.benchmark.cli import main
+
+if __name__ == "__main__":
+    main()