Env variable for faster Python imports (#1175)

.github/workflows/build-docs.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Install dependencies
         run: | 
           sudo apt-get update
-          sudo apt-get install -y build-essential libeigen3-dev libyaml-dev libfftw3-dev libavcodec-dev libavformat-dev libavutil-dev libswresample-dev libsamplerate0-dev libtag1-dev libchromaprint-dev python3-dev python3-numpy-dev python3-numpy python3-yaml python3-six
+          sudo apt-get install -y build-essential libeigen3-dev libyaml-dev libfftw3-dev libavcodec-dev libavformat-dev libavutil-dev libswresample-dev libsamplerate0-dev libtag1-dev libchromaprint-dev python3-dev python3-numpy-dev python3-numpy python3-yaml
           sudo apt-get install -y doxygen python3-pip pandoc
           pip3 install sphinx pyparsing sphinxcontrib-doxylink docutils jupyter sphinxprettysearchresults sphinx-toolbox
           # Install TensorFlow

doc/sphinxdoc/installing.rst

@@ -55,7 +55,7 @@ You can install those dependencies on a Debian/Ubuntu system from official repos
 
 In order to use Python 3 bindings for the library, you might also need to install python3-dev, python3-numpy-dev (or python3-numpy on Ubuntu) and python3-yaml for YAML support in python::
 
-  sudo apt-get install python3-dev python3-numpy-dev python3-numpy python3-yaml python3-six
+  sudo apt-get install python3-dev python3-numpy-dev python3-numpy python3-yaml
 
 Note that, depending on the version of Essentia, different versions of ``libav*`` and ``libtag1-dev`` packages are required. See `release notes for official releases <https://github.com/MTG/essentia/releases>`_.
 

doc/sphinxdoc/machine_learning.rst

@@ -59,7 +59,7 @@ Install the `dependencies <https://essentia.upf.edu/installing.html#installing-d
 
 .. code-block::
 
-    apt-get install build-essential libyaml-dev libfftw3-dev libavcodec-dev libavformat-dev libavutil-dev libavresample-dev python-dev libsamplerate0-dev libtag1-dev libchromaprint-dev python-six python3-dev python3-numpy-dev python3-numpy python3-yaml libeigen3-dev
+    apt-get install build-essential libyaml-dev libfftw3-dev libavcodec-dev libavformat-dev libavutil-dev libavresample-dev python-dev libsamplerate0-dev libtag1-dev libchromaprint-dev python3-dev python3-numpy-dev python3-numpy python3-yaml libeigen3-dev
 
 Configure Essentia with TensorFlow and Python 3:
 

pyproject-tensorflow.toml

@@ -1,3 +1,7 @@
+[tool.cibuildwheel]
+build-verbosity = 3
+
+
 [tool.cibuildwheel.linux]
 
 manylinux-x86_64-image = "mtgupf/essentia-builds:manylinux2014_x86_64"

pyproject.toml

@@ -1,4 +1,6 @@
 [tool.cibuildwheel]
+build-verbosity = 3
+
 
 [tool.cibuildwheel.linux]
 manylinux-x86_64-image = "mtgupf/essentia-builds:manylinux2014_x86_64"

setup.py

@@ -103,7 +103,7 @@ def get_version():
 Website: https://essentia.upf.edu
 '''
 
-setup_requires = ['numpy>=1.8.2', 'six']
+setup_requires = ['numpy>=1.8.2']
 install_requires = setup_requires + ['pyyaml']
 
 # Require tensorflow for the package essentia-tensorflow

src/python/essentia.cpp

@@ -77,26 +77,17 @@ init_essentia() {
       PyType_Ready(&VectorVectorStereoSampleType) < 0) {
 
     cerr << "Unable to instantiate Essentia's wrapper types." << endl;
-#if PY_MAJOR_VERSION >= 3
     return NULL;
-#else
-    return;
-#endif
   }
 
   // import the NumPy C api
   int numpy_error = _import_array();
   if (numpy_error) {
     cerr << "Unable to import NumPy C API from Essentia module. Error code = " << numpy_error << endl;
-#if PY_MAJOR_VERSION >= 3
     return NULL;
-#else
-    return;
-#endif
   }
 
 
-#if PY_MAJOR_VERSION >= 3
   static struct PyModuleDef moduledef = {
     PyModuleDef_HEAD_INIT,
     "_essentia",     /* m_name */
@@ -108,22 +99,12 @@ init_essentia() {
     NULL,                /* m_clear */
     NULL,                /* m_free */
   };
-#endif
 
-#if PY_MAJOR_VERSION >= 3
   Essentia__Module = PyModule_Create(&moduledef);
-#else
-  Essentia__Module = Py_InitModule3("_essentia", Essentia__Methods,
-                                    "Module that allows access to essentia plugins and algorithms.");
-#endif
 
   if (Essentia__Module == NULL) {
     cerr << "Error loading _essentia python/C module" << endl;
-#if PY_MAJOR_VERSION >= 3
     return NULL;
-#else
-    return;
-#endif
   }
 
   // insert the Algorithm class
@@ -143,9 +124,5 @@ init_essentia() {
   essentia::init();
 
   E_DEBUG(EPyBindings, "Successfully initialized _essentia python/C module");
-#if PY_MAJOR_VERSION >= 3
   return Essentia__Module;
-#else
-  return;
-#endif
 }

src/python/essentia/common.py

@@ -16,7 +16,6 @@
 # version 3 along with this program. If not, see http://www.gnu.org/licenses/
 
 import numpy
-from six import iteritems
 from . import _essentia
 
 
@@ -231,7 +230,7 @@ def determineEdt(obj):
             firstType = None
             allKeysAreStrings = True
             allTypesEqual = True
-            for key, val in iteritems(obj):
+            for key, val in iter(obj.items()):
                 if not isinstance(key, str):
                     allKeysAreStrings = False
                     break
@@ -360,7 +359,7 @@ def __init__(self, poolRep=None):
 
         elif isinstance(poolRep, dict):
             self.cppPool = _essentia.Pool()
-            for key, val in iteritems(poolRep):
+            for key, val in iter(poolRep.items()):
                 for v in val:
                     self.add(key, v)
 

src/python/essentia/meta.py

@@ -0,0 +1,35 @@
+from json import dump
+from os.path import join
+import essentia
+from ._essentia import Algorithm, StreamingAlgorithm, keys, skeys
+
+
+def _metadata_standard():
+    meta = {}
+    for name in keys():
+        essentia.log.debug(essentia.EPython, 'Loading __doc__ and __struct__ metadata for essentia.standard class: %s' % name)
+        _algoInstance = Algorithm(name)
+        meta[name] = {}
+        meta[name]['__doc__'] = _algoInstance.getDoc()
+        meta[name]['__struct__'] = _algoInstance.getStruct()
+        del _algoInstance
+    return meta
+
+
+def _metadata_streaming():
+    meta = {}
+    for name in skeys():
+        essentia.log.debug(essentia.EPython, 'Loading __doc__ and __struct__ metadata for essentia.streaming class: %s' % name)
+        _algoInstance = StreamingAlgorithm(name)
+        meta[name] = {}
+        meta[name]['__doc__'] = _algoInstance.getDoc()
+        meta[name]['__struct__'] = _algoInstance.getStruct()
+        del _algoInstance
+    return meta
+
+
+def _extract_metadata(filepath_standard, filepath_streaming):
+    """ Loads algorithms' metadata (__doc__ and __struct__) from the C extension
+    and stores it to files in a filedir"""
+    dump(_metadata_standard(), open(filepath_standard, 'w'))
+    dump(_metadata_streaming(), open(filepath_streaming, 'w'))

src/python/essentia/standard.py

@@ -15,22 +15,22 @@
 # You should have received a copy of the Affero GNU General Public License
 # version 3 along with this program. If not, see http://www.gnu.org/licenses/
 
-from six import iteritems
 from . import _essentia
 import essentia
 from . import common as _c
 import sys as _sys
 from ._essentia import keys as algorithmNames, info as algorithmInfo
 from copy import copy
+import os.path
+import json
+
 
 # given an essentia algorithm name, create the corresponding class
-def _create_essentia_class(name, moduleName = __name__):
+def _create_essentia_class(name, meta, moduleName = __name__):
     essentia.log.debug(essentia.EPython, 'Creating essentia.standard class: %s' % name)
 
-    _algoInstance = _essentia.Algorithm(name)
-    _algoDoc = _algoInstance.getDoc()
-    _algoStruct = _algoInstance.getStruct()
-    del _algoInstance
+    _algoDoc = meta[name]['__doc__']
+    _algoStruct = meta[name]['__struct__']
 
     class Algo(_essentia.Algorithm):
         __doc__ = _algoDoc
@@ -45,7 +45,7 @@ def __init__(self, **kwargs):
 
         def configure(self, **kwargs):
             # verify that all types match and do any necessary conversions
-            for name, val in iteritems(kwargs):
+            for name, val in iter(kwargs.items()):
                 goalType = self.paramType(name)
 
                 if type(val).__module__ == 'numpy':
@@ -135,8 +135,17 @@ def __str__(self):
 
 # load all classes into python
 def _reloadAlgorithms(moduleName = __name__):
-    for name in _essentia.keys():
-        _create_essentia_class(name, moduleName)
+    meta_file = 'standard.meta.json'
+    essentia.log.debug(essentia.EPython, f'Loading __doc__ and __struct__ metadata for essentia.standard from {meta_file}')
+    # Looking for a metadata file in the same directory as `standard.py`
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+    file_path = os.path.join(dir_path, meta_file)
+    with open(file_path, 'r') as f:
+        meta = json.load(f)
+
+    for name in algorithmNames():
+        _create_essentia_class(name, meta, moduleName)
+
 
 _reloadAlgorithms()
 

src/python/essentia/streaming.py

@@ -15,12 +15,14 @@
 # You should have received a copy of the Affero GNU General Public License
 # version 3 along with this program. If not, see http://www.gnu.org/licenses/
 
-from six import iteritems
 from . import _essentia
 import essentia
 import sys as _sys
 from . import common as _c
 from ._essentia import skeys as algorithmNames, sinfo as algorithmInfo
+import os.path
+import json
+
 
 # Used as a place-holder for sources and sinks, implements the right shift
 # operator
@@ -136,13 +138,11 @@ def totalProduced(self):
 
 
 
-def _create_streaming_algo(givenname):
+def _create_streaming_algo(givenname, meta):
     essentia.log.debug(essentia.EPython, 'Creating essentia.streaming class: %s' % givenname)
 
-    _algoInstance = _essentia.StreamingAlgorithm(givenname)
-    _algoDoc = _algoInstance.getDoc()
-    _algoStruct = _algoInstance.getStruct()
-    del _algoInstance
+    _algoDoc = meta[givenname]['__doc__']
+    _algoStruct = meta[givenname]['__struct__']
 
     class StreamingAlgo(_essentia.StreamingAlgorithm):
         __doc__ = _algoDoc
@@ -174,7 +174,7 @@ def __init__(self, **kwargs):
 
         def configure(self, **kwargs):
             # verify that all types match and do any necessary conversions
-            for name, val in iteritems(kwargs):
+            for name, val in iter(kwargs.items()):
                 goalType = self.paramType(name)
                 try:
                     convertedVal = _c.convertData(val, goalType)
@@ -195,11 +195,21 @@ def configure(self, **kwargs):
 
 # load all streaming algorithms into module
 def _reloadStreamingAlgorithms():
+    meta_file = 'streaming.meta.json'
+    essentia.log.debug(essentia.EPython, f'Loading __doc__ and __struct__ metadata for essentia.streaming from {meta_file}')
+    # Looking for a metadata file in the same directory as `streaming.py`
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+    file_path = os.path.join(dir_path, meta_file)
+    with open(file_path, 'r') as f:
+        meta = json.load(f)
+
     for name in algorithmNames():
-        _create_streaming_algo(name)
+        _create_streaming_algo(name, meta)
+
 
 _reloadStreamingAlgorithms()
 
+
 # This subclass provides some more functionality for VectorInput
 class VectorInput(_essentia.VectorInput):
     __doc__ = 'VectorInput v1.0\n\n\n'+\