Merge remote-tracking branch 'origin/2.2' into 2.3

doc/administrator/editing.rst

@@ -133,3 +133,24 @@ Example:
 
    * Source and destination layers must have the same geometry type.
    * Only the geometry will be copied, the attributes will not be.
+
+Edit views
+----------
+
+To be able to edit PostgreSQL views a primary key must be manually configured.
+Add a layer metadata ``geotablePrimaryKey`` with value the name of the column to use as primary key.
+That column must be of type ``Integer``.
+
+Example:
+
+    * geotablePrimaryKey: ``id``
+
+Enable snapping
+---------------
+
+To be able to snap while editing, the ``snappingConfig`` must be set on the layer metadata.
+The value is a ``json`` object containing the following optional properties:
+
+    * edge (boolean): whether to allow snapping on edges or not;
+    * vertex (boolean): whether to allow snapping on vertices or not;
+    * tolerance (number): the pixel tolerance.

doc/developer/debugging.rst

@@ -205,7 +205,7 @@ Performance or network error
 ----------------------------
 
 For performance and proxy issues make sure that all internal URLs in the config file
-use localhost (use ``curl "http://localhost/<path>" -H Host:<server_name>``
+use localhost (use ``curl "http://localhost/<path>" --header Host:<server_name>``
 to test it).
 
 Tilecloud chain

doc/integrator/checker.rst

@@ -68,6 +68,29 @@ The checker use the following configuration structure in ``vars_<project>.yaml``
         checker:
             forward_headers: ['Cookie', 'Authorisation']
 
+.. note::
+
+    The checker assumes that it can access the c2cgeoportal services via ``http://localhost``.
+    If this is not allowed on your server, you can override this behaviour as follows.
+    In your ``vars`` file, add the following:
+
+    .. code:: yaml
+
+        vars:
+            checker:
+                rewrite_as_http_localhost: False
+
+    Now, in your configuration file ``project.yaml.mako``, instead of defining the ``checker_path``,
+    define a ``checker_url`` with the full URL to be used, for example:
+
+    .. code:: yaml
+
+        ...
+        host: ${host}
+        checker_url: https://${host}/${instanceid}/wsgi/check_collector?
+        ...
+
+
 ``print``
 ~~~~~~~~~
 

doc/integrator/https.rst

@@ -78,3 +78,17 @@ Then you can access resources by building urls using the following schema:
 For example:
 
 ``http://geoportail.camptocamp.com/main/wsgi/resourceproxy?target=rfinfo&values=(175,2633)``
+
+Local certificate checks
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Certain c2cgeoportal features open a http session to your c2cgeoportal services,
+for example the ``checker`` or the ``lingua_extractor``.
+If you are running your server in https and wish to disable certificate checks in these
+connections, you can achieve this by adding the following configuration element to your ``vars`` file:
+
+.. code:: yaml
+
+    vars:
+        http_options:
+            disable_ssl_certificate_validation: True

geoportal/c2cgeoportal_geoportal/lib/__init__.py

@@ -30,6 +30,7 @@
 
 import datetime
 import dateutil
+import httplib2
 import json
 import re
 import urllib.request
@@ -55,6 +56,10 @@ def get_types_map(types_array):
     return types_map
 
 
+def get_http(request):
+    return httplib2.Http(**request.registry.settings.get("http_options", {}))
+
+
 def get_url(url, request, default=None, errors=None):
     if url is None:
         return default

geoportal/c2cgeoportal_geoportal/lib/dbreflection.py

@@ -31,7 +31,7 @@
 import warnings
 from typing import Dict, Tuple  # noqa, pylint: disable=unused-import
 
-from sqlalchemy import Table, MetaData
+from sqlalchemy import Table, MetaData, Column, Integer
 from sqlalchemy.orm import relationship
 from sqlalchemy.orm.util import class_mapper
 from sqlalchemy.exc import SAWarning
@@ -125,7 +125,7 @@ def _get_schema(tablename):
     return tablename, schema
 
 
-def get_table(tablename, schema=None, session=None):
+def get_table(tablename, schema=None, session=None, primary_key=None):
     if schema is None:
         tablename, schema = _get_schema(tablename)
 
@@ -144,16 +144,20 @@ def get_table(tablename, schema=None, session=None):
             "ignore",
             "Did not recognize type 'geometry' of column",
             SAWarning)
+        args = [tablename, metadata]
+        if primary_key is not None:
+            # Ensure we have a primary key to be able to edit views
+            args.append(Column(primary_key, Integer, primary_key=True))
         table = Table(
-            tablename, metadata,
+            *args,
             schema=schema,
             autoload=True,
-            autoload_with=engine,
+            autoload_with=engine
         )
     return table
 
 
-def get_class(tablename, session=None, exclude_properties=None):
+def get_class(tablename, session=None, exclude_properties=None, primary_key=None):
     """
     Get the SQLAlchemy mapped class for "tablename". If no class exists
     for "tablename" one is created, and added to the cache. "tablename"
@@ -164,12 +168,12 @@ def get_class(tablename, session=None, exclude_properties=None):
     if exclude_properties is None:
         exclude_properties = []
     tablename, schema = _get_schema(tablename)
-    cache_key = (schema, tablename, ",".join(exclude_properties))
+    cache_key = (schema, tablename, ",".join(exclude_properties), primary_key)
 
     if cache_key in _class_cache:
         return _class_cache[cache_key]
 
-    table = get_table(tablename, schema, session)
+    table = get_table(tablename, schema, session, primary_key=primary_key)
 
     # create the mapped class
     cls = _create_class(table, exclude_properties)

geoportal/c2cgeoportal_geoportal/lib/filter_capabilities.py

@@ -29,7 +29,6 @@
 
 
 import logging
-import httplib2
 import copy
 from io import StringIO
 from urllib.parse import urlsplit, urljoin
@@ -47,7 +46,7 @@
 
 from c2cgeoportal_geoportal.lib import caching, get_protected_layers_query, \
     get_writable_layers_query, add_url_params, get_ogc_server_wms_url_ids,\
-    get_ogc_server_wfs_url_ids
+    get_ogc_server_wfs_url_ids, get_http
 from c2cgeoportal_commons.models import DBSession
 from c2cgeoportal_commons.models.main import LayerWMS, OGCServer
 
@@ -83,7 +82,7 @@ def get_writable_layers(role_id, ogc_server_ids):
 
 
 @cache_region.cache_on_arguments()
-def wms_structure(wms_url, host):
+def wms_structure(wms_url, host, request):
     url = urlsplit(wms_url)
     wms_url = add_url_params(wms_url, {
         "SERVICE": "WMS",
@@ -92,7 +91,7 @@ def wms_structure(wms_url, host):
     })
 
     # Forward request to target (without Host Header)
-    http = httplib2.Http()
+    http = get_http(request)
     headers = dict()
     if url.hostname == "localhost" and host is not None:  # pragma: no cover
         headers["Host"] = host
@@ -174,7 +173,7 @@ def filter_capabilities(content, role_id, wms, url, headers, proxies, request):
     if proxies:  # pragma: no cover
         enable_proxies(proxies)
 
-    wms_structure_ = wms_structure(url, headers.get("Host"))
+    wms_structure_ = wms_structure(url, headers.get("Host"), request)
 
     ogc_server_ids = (
         get_ogc_server_wms_url_ids(request) if wms else

geoportal/c2cgeoportal_geoportal/lib/lingua_extractor.py

@@ -58,7 +58,6 @@
 from c2cgeoportal_geoportal import init_dbsessions
 from c2cgeoportal_geoportal.lib import add_url_params, get_url2
 from c2cgeoportal_geoportal.lib.bashcolor import colorize, RED
-from c2cgeoportal_geoportal.lib.dbreflection import get_class
 from c2cgeoportal_geoportal.lib.caching import init_region
 
 from c2cgeoportal_geoportal.lib.print_ import *  # noqa
@@ -421,15 +420,9 @@ def _import_layer_wms(self, layer, messages):
         for wms_layer in layer.layer.split(","):
             self._import_layer_attributes(url, wms_layer, layer.item_type, layer.name, messages)
         if layer.geo_table is not None and layer.geo_table != "":
-            exclude = [] if layer.exclude_properties is None else layer.exclude_properties.split(",")
-            last_update_date = layer.get_metadatas("lastUpdateDateColumn")
-            if len(last_update_date) == 1:
-                exclude.append(last_update_date[0].value)
-            last_update_user = layer.get_metadatas("lastUpdateUserColumn")
-            if len(last_update_user) == 1:
-                exclude.append(last_update_user[0].value)
             try:
-                cls = get_class(layer.geo_table, exclude_properties=exclude)
+                from c2cgeoportal_geoportal.views.layers import get_layer_class
+                cls = get_layer_class(layer)
                 for column_property in class_mapper(cls).iterate_properties:
                     if isinstance(column_property, ColumnProperty) and len(column_property.columns) == 1:
                         column = column_property.columns[0]

geoportal/c2cgeoportal_geoportal/scripts/c2cupgrade.py

@@ -276,7 +276,7 @@ def test_checkers(self):
                 "Run `curl {} '{}'` for more information."
             ]).format(
                 ' '.join([
-                    '--header={}={}'.format(*i) for i in self.project.get("checker_headers", {}).items()
+                    '--header {}={}'.format(*i) for i in self.project.get("checker_headers", {}).items()
                 ]),
                 self.project["checker_url"],
             )

geoportal/c2cgeoportal_geoportal/views/entry.py

@@ -28,7 +28,6 @@
 # either expressed or implied, of the FreeBSD Project.
 
 
-import httplib2
 import logging
 import json
 import sys
@@ -55,7 +54,7 @@
 from c2cgeoportal_commons import models
 from c2cgeoportal_commons.models import main, static
 from c2cgeoportal_geoportal.lib import get_setting, get_protected_layers_query, \
-    get_url2, get_url, get_typed, get_types_map, add_url_params
+    get_url2, get_url, get_typed, get_types_map, add_url_params, get_http
 from c2cgeoportal_geoportal.lib.cacheversion import get_cache_version
 from c2cgeoportal_geoportal.lib.caching import get_region, \
     set_common_headers, NO_CACHE, PUBLIC_CACHE, PRIVATE_CACHE
@@ -235,7 +234,7 @@ def _wms_getcap(self, ogc_server=None):
 
     @cache_region.cache_on_arguments()
     def get_http_cached(self, url, headers):
-        http = httplib2.Http()
+        http = get_http(self.request)
         return http.request(url, method="GET", headers=headers)
 
     @cache_region.cache_on_arguments()

geoportal/c2cgeoportal_geoportal/views/layers.py

@@ -80,7 +80,7 @@ def _get_geom_col_info(layer):
         This function assumes that the names of geometry attributes
         in the mapped class are the same as those of geometry columns.
         """
-        mapped_class = get_class(layer.geo_table)
+        mapped_class = get_layer_class(layer)
         for p in class_mapper(mapped_class).iterate_properties:
             if not isinstance(p, ColumnProperty):
                 continue  # pragma: no cover
@@ -134,7 +134,7 @@ def _get_layer_for_request(self):
 
     def _get_protocol_for_layer(self, layer, **kwargs):
         """ Returns a papyrus ``Protocol`` for the ``Layer`` object. """
-        cls = get_class(layer.geo_table)
+        cls = get_layer_class(layer)
         geom_attr = self._get_geom_col_info(layer)[0]
         return Protocol(models.DBSession, cls, geom_attr, **kwargs)
 
@@ -354,16 +354,16 @@ def _validate_geometry(geom):
                 raise TopologicalError(reason)
 
     def _log_last_update(self, layer, feature):
-        last_update_date = self._get_metadata(layer, "lastUpdateDateColumn")
+        last_update_date = self.get_metadata(layer, "lastUpdateDateColumn")
         if last_update_date is not None:
             setattr(feature, last_update_date, datetime.now())
 
-        last_update_user = self._get_metadata(layer, "lastUpdateUserColumn")
+        last_update_user = self.get_metadata(layer, "lastUpdateUserColumn")
         if last_update_user is not None:
             setattr(feature, last_update_user, self.request.user.id)
 
     @staticmethod
-    def _get_metadata(layer, key, default=None):
+    def get_metadata(layer, key, default=None):
         metadatas = layer.get_metadatas(key)
         if len(metadatas) == 1:
             metadata = metadatas[0]
@@ -372,7 +372,7 @@ def _get_metadata(layer, key, default=None):
 
     def _get_validation_setting(self, layer):
         # The validation UIMetadata is stored as a string, not a boolean
-        should_validate = self._get_metadata(layer, "geometryValidation", None)
+        should_validate = self.get_metadata(layer, "geometryValidation", None)
         if should_validate:
             return should_validate.lower() != "false"
         return self.settings.get("geometry_validation", False)
@@ -416,19 +416,7 @@ def metadata(self):
         if not layer.public and self.request.user is None:
             raise HTTPForbidden()
 
-        # exclude the columns used to record the last features update
-        exclude = [] if layer.exclude_properties is None else layer.exclude_properties.split(",")
-        last_update_date = self._get_metadata(layer, "lastUpdateDateColumn")
-        if last_update_date is not None:
-            exclude.append(last_update_date)
-        last_update_user = self._get_metadata(layer, "lastUpdateUserColumn")
-        if last_update_user is not None:
-            exclude.append(last_update_user)
-
-        return get_class(
-            layer.geo_table,
-            exclude_properties=exclude
-        )
+        return get_layer_class(layer)
 
     @view_config(route_name="layers_enumerate_attribute_values", renderer="json")
     def enumerate_attribute_values(self):
@@ -481,24 +469,26 @@ def query_enumerate_attribute_values(dbsession, layerinfos, fieldname):
         return dbsession.query(distinct(attribute)).order_by(attribute).all()
 
 
-def get_layer_metadatas(layer):
+def get_layer_class(layer):
     # exclude the columns used to record the last features update
     exclude = [] if layer.exclude_properties is None else layer.exclude_properties.split(",")
-
-    date_metadata = layer.get_metadatas("lastUpdateDateColumn")
-    last_update_date = date_metadata[0] if len(date_metadata) == 1 else None
+    last_update_date = Layers.get_metadata(layer, "lastUpdateDateColumn")
     if last_update_date is not None:
-        exclude.append(last_update_date.value)
-    user_metadata = layer.get_metadatas("lastUpdateUserColumn")
-    last_update_user = user_metadata[0] if len(user_metadata) == 1 else None
+        exclude.append(last_update_date)
+    last_update_user = Layers.get_metadata(layer, "lastUpdateUserColumn")
     if last_update_user is not None:
-        exclude.append(last_update_user.value)
+        exclude.append(last_update_user)
 
-    cls = get_class(
-        layer.geo_table,
-        exclude_properties=exclude
+    primary_key = Layers.get_metadata(layer, "geotablePrimaryKey")
+    return get_class(
+        str(layer.geo_table),
+        exclude_properties=exclude,
+        primary_key=primary_key
     )
 
+
+def get_layer_metadatas(layer):
+    cls = get_layer_class(layer)
     edit_columns = []
 
     for column_property in class_mapper(cls).iterate_properties:

geoportal/c2cgeoportal_geoportal/views/proxy.py

@@ -29,7 +29,6 @@
 
 
 import sys
-import httplib2
 import urllib.request
 import urllib.parse
 import urllib.error
@@ -40,6 +39,7 @@
 from pyramid.response import Response
 from pyramid.httpexceptions import HTTPBadGateway, exception_response
 
+from c2cgeoportal_geoportal.lib import get_http
 from c2cgeoportal_geoportal.lib.caching import get_region, \
     set_common_headers, NO_CACHE, PUBLIC_CACHE, PRIVATE_CACHE
 
@@ -80,7 +80,7 @@ def _proxy(self, url, params=None, method=None, cache=False, body=None, headers=
             method = self.request.method
 
         # forward request to target (without Host Header)
-        http = httplib2.Http()
+        http = get_http(self.request)
 
         if headers is None:  # pragma: no cover
             headers = dict(self.request.headers)