Merge branch 'develop'

requirements.txt

@@ -1,6 +1,5 @@
 numpy>=1.13.3
 scipy>=0.19.1
-scikit-learn>=0.19.0
 scikit-image>=0.13.0
 lightgbm>=2.0.11
 sentinelhub>=2.4.6

s2cloudless/PixelClassifier.py

@@ -1,6 +1,8 @@
 """
 Module implementing pixel-based classifier
 """
+import numpy as np
+from lightgbm import Booster
 
 
 class PixelClassifier:
@@ -12,40 +14,43 @@ class PixelClassifier:
     Pixel classifier divides the image into individual pixels, runs classifier over
     them, and finally produces a classification mask of the same size as image.
 
-    The classifier can be of any type as long as it has the following two methods
-    implemented:
+    The classifier can be of a type that is explicitly supported (e.g. lightgbm.Booster) or of any type as long as
+    it has the following two methods implemented:
         - predict(X)
         - predict_proba(X)
 
     This is true for all classifiers that follow scikit-learn's API.
     The APIs of scikit-learn's objects is described
     at: http://scikit-learn.org/stable/developers/contributing.html#apis-of-scikit-learn-objects.
-
-    :param classifier: trained classifier that will be executed over an entire image
-    :type classifier: any classifier with predict(X) and predict_proba(X) methods
     """
     # pylint: disable=invalid-name
     def __init__(self, classifier):
-        self.receptive_field = (1, 1)
+        """
+        :param classifier: An instance of trained classifier that will be executed over an entire image
+        :type classifier: Booster or object that implements methods predict and predict_proba
+        """
         self._check_classifier(classifier)
         self.classifier = classifier
 
     @staticmethod
     def _check_classifier(classifier):
         """
-        Check if the classifier implements predict and predict_proba methods.
+        Checks if the classifier is of correct type or if it implements predict and predict_proba methods
         """
-        predict = getattr(classifier, "predict", None)
+        if isinstance(classifier, Booster):
+            return
+
+        predict = getattr(classifier, 'predict', None)
         if not callable(predict):
             raise ValueError('Classifier does not have a predict method!')
 
-        predict_proba = getattr(classifier, "predict_proba", None)
+        predict_proba = getattr(classifier, 'predict_proba', None)
         if not callable(predict_proba):
             raise ValueError('Classifier does not have a predict_proba method!')
 
     @staticmethod
     def extract_pixels(X):
-        """ Extract pixels from array X
+        """ Extracts pixels from array X
 
         :param X: Array of images to be classified.
         :type X: numpy array, shape = [n_images, n_pixels_y, n_pixels_x, n_bands]
@@ -57,40 +62,46 @@ def extract_pixels(X):
             raise ValueError('Array of input images has to be a 4-dimensional array of shape'
                              '[n_images, n_pixels_y, n_pixels_x, n_bands]')
 
-        new_shape = (X.shape[0] * X.shape[1] * X.shape[2], X.shape[3],)
+        new_shape = X.shape[0] * X.shape[1] * X.shape[2], X.shape[3]
         pixels = X.reshape(new_shape)
         return pixels
 
-    def image_predict(self, X):
+    def image_predict(self, X, **kwargs):
         """
-        Predicts class label for the entire image.
+        Predicts class labels for the entire image.
 
         :param X: Array of images to be classified.
         :type X: numpy array, shape = [n_images, n_pixels_y, n_pixels_x, n_bands]
-
+        :param kwargs: Any keyword arguments that will be passed to the classifier's prediction method
         :return: raster classification map
         :rtype: numpy array, [n_samples, n_pixels_y, n_pixels_x]
         """
-
         pixels = self.extract_pixels(X)
 
-        predictions = self.classifier.predict(pixels)
+        if isinstance(self.classifier, Booster):
+            raise NotImplementedError('An instance of lightgbm.Booster can only return prediction probabilities, '
+                                      'use PixelClassifier.image_predict_proba instead')
+
+        predictions = self.classifier.predict(pixels, **kwargs)
 
         return predictions.reshape(X.shape[0], X.shape[1], X.shape[2])
 
-    def image_predict_proba(self, X):
+    def image_predict_proba(self, X, **kwargs):
         """
         Predicts class probabilities for the entire image.
 
         :param X: Array of images to be classified.
         :type X: numpy array, shape = [n_images, n_pixels_y, n_pixels_x, n_bands]
-
+        :param kwargs: Any keyword arguments that will be passed to the classifier's prediction method
         :return: classification probability map
         :rtype: numpy array, [n_samples, n_pixels_y, n_pixels_x]
         """
-
         pixels = self.extract_pixels(X)
 
-        probabilities = self.classifier.predict_proba(pixels)
+        if isinstance(self.classifier, Booster):
+            probabilities = self.classifier.predict(pixels, **kwargs)
+            probabilities = np.vstack((1. - probabilities, probabilities)).transpose()
+        else:
+            probabilities = self.classifier.predict_proba(pixels, **kwargs)
 
         return probabilities.reshape(X.shape[0], X.shape[1], X.shape[2], probabilities.shape[1])

s2cloudless/S2PixelCloudDetector.py

@@ -4,21 +4,18 @@
 
 import copy
 import os
-import warnings
 import numpy as np
 
 from scipy.ndimage.filters import convolve
 from skimage.morphology import disk, dilation
-from sklearn.externals import joblib
+from lightgbm import Booster
 
 from sentinelhub import CustomUrlParam, MimeType
 
 from .PixelClassifier import PixelClassifier
 
 
-warnings.filterwarnings("ignore", category=UserWarning)
-
-MODEL_FILENAME = 'pixel_s2_cloud_detector_lightGBM_v0.1.joblib.dat'
+MODEL_FILENAME = 'pixel_s2_cloud_detector_lightGBM_v0.1.txt'
 
 MODEL_EVALSCRIPT = 'return [B01,B02,B04,B05,B08,B8A,B09,B10,B11,B12]'
 S2_BANDS_EVALSCRIPT = 'return [B01,B02,B03,B04,B05,B06,B07,B08,B8A,B09,B10,B11,B12]'
@@ -68,22 +65,28 @@ def __init__(self, threshold=0.4, all_bands=False, average_over=1, dilation_size
         if model_filename is None:
             package_dir = os.path.dirname(__file__)
             model_filename = os.path.join(package_dir, 'models', MODEL_FILENAME)
+        self.model_filename = model_filename
 
-        self._load_classifier(model_filename)
+        self._classifier = None
 
         if average_over > 0:
             self.conv_filter = disk(average_over) / np.sum(disk(average_over))
 
         if dilation_size > 0:
             self.dilation_filter = disk(dilation_size)
 
-    def _load_classifier(self, filename):
+    @property
+    def classifier(self):
         """
-        Loads the classifier.
+        Provides a classifier object. It also loads it if it hasn't been loaded yet. This way the classifier is loaded
+        only when it is actually required.
         """
-        self.classifier = PixelClassifier(joblib.load(filename))
+        if self._classifier is None:
+            self._classifier = PixelClassifier(Booster(model_file=self.model_filename))
+
+        return self._classifier
 
-    def get_cloud_probability_maps(self, X):
+    def get_cloud_probability_maps(self, X, **kwargs):
         """
         Runs the cloud detection on the input images (dimension n_images x n x m x 10
         or n_images x n x m x 13) and returns an array of cloud probability maps (dimension
@@ -93,6 +96,7 @@ def get_cloud_probability_maps(self, X):
         :param X: input Sentinel-2 image obtained with Sentinel-Hub's WMS/WCS request
                   (see https://github.com/sentinel-hub/sentinelhub-py)
         :type X: numpy array (shape n_images x n x m x 10 or n x m x 13)
+        :param kwargs: Any keyword arguments that will be passed to the classifier's prediction method
         :return: cloud probability map
         :rtype: numpy array (shape n_images x n x m)
         """
@@ -105,9 +109,9 @@ def get_cloud_probability_maps(self, X):
         if self.all_bands:
             X = X[..., self.BAND_IDXS]
 
-        return self.classifier.image_predict_proba(X)[..., 1]
+        return self.classifier.image_predict_proba(X, **kwargs)[..., 1]
 
-    def get_cloud_masks(self, X):
+    def get_cloud_masks(self, X, **kwargs):
         """
         Runs the cloud detection on the input images (dimension n_images x n x m x 10
         or n_images x n x m x 13) and returns the raster cloud mask (dimension n_images x n x m).
@@ -117,11 +121,12 @@ def get_cloud_masks(self, X):
         :param X: input Sentinel-2 image obtained with Sentinel-Hub's WMS/WCS request
                   (see https://github.com/sentinel-hub/sentinelhub-py)
         :type X: numpy array (shape n_images x n x m x 10 or n x m x 13)
+        :param kwargs: Any keyword arguments that will be passed to the classifier's prediction method
         :return: raster cloud mask
         :rtype: numpy array (shape n_images x n x m)
         """
 
-        cloud_probs = self.get_cloud_probability_maps(X)
+        cloud_probs = self.get_cloud_probability_maps(X, **kwargs)
 
         return self.get_mask_from_prob(cloud_probs)
 
@@ -148,7 +153,6 @@ def get_mask_from_prob(self, cloud_probs, threshold=None):
         if self.dilation_size:
             cloud_masks = np.asarray([dilation(cloud_mask, self.dilation_filter) for cloud_mask in cloud_masks],
                                      dtype=np.int8)
-
         return cloud_masks
 
 

s2cloudless/__init__.py

@@ -6,4 +6,4 @@
 from .PixelClassifier import PixelClassifier
 
 
-__version__ = '1.3.0'
+__version__ = '1.4.0'

setup.py

@@ -34,7 +34,7 @@ def parse_requirements(file):
     packages=find_packages(),
     package_dir={'': '.'},
     include_package_data=True,
-    package_data={'s2cloudless': ['models/pixel_s2_cloud_detector_lightGBM_v0.1.joblib.dat']},
+    package_data={'s2cloudless': ['models/pixel_s2_cloud_detector_lightGBM_v0.1.txt']},
     install_requires=parse_requirements('requirements.txt'),
     extras_require={'DEV': parse_requirements('requirements-dev.txt')},
     zip_safe=False,

tests/test_cloud_detector.py

@@ -20,7 +20,7 @@ def test_cloud_detector(self):
         cloud_detector = S2PixelCloudDetector(all_bands=True)
         cloud_probs = cloud_detector.get_cloud_probability_maps(self.templates['s2_im'])
         cloud_mask = cloud_detector.get_cloud_masks(self.templates['s2_im'])
-        self.assertTrue(np.isclose(cloud_probs, self.templates['cl_probs']).all())
+        self.assertTrue(np.array_equal(cloud_probs, self.templates['cl_probs']))
         self.assertTrue(np.array_equal(cloud_mask, self.templates['cl_mask']))
 
         cloud_detector = S2PixelCloudDetector(all_bands=False)