diff --git a/doc/administrator/editing.rst b/doc/administrator/editing.rst index 6174057f71..38f8e7c6da 100644 --- a/doc/administrator/editing.rst +++ b/doc/administrator/editing.rst @@ -133,3 +133,24 @@ Example: * Source and destination layers must have the same geometry type. * Only the geometry will be copied, the attributes will not be. + +Edit views +---------- + +To be able to edit PostgreSQL views a primary key must be manually configured. +Add a layer metadata ``geotablePrimaryKey`` with value the name of the column to use as primary key. +That column must be of type ``Integer``. + +Example: + + * geotablePrimaryKey: ``id`` + +Enable snapping +--------------- + +To be able to snap while editing, the ``snappingConfig`` must be set on the layer metadata. +The value is a ``json`` object containing the following optional properties: + + * edge (boolean): whether to allow snapping on edges or not; + * vertex (boolean): whether to allow snapping on vertices or not; + * tolerance (number): the pixel tolerance. diff --git a/doc/developer/debugging.rst b/doc/developer/debugging.rst index 82c03df886..e548444cc1 100644 --- a/doc/developer/debugging.rst +++ b/doc/developer/debugging.rst @@ -205,7 +205,7 @@ Performance or network error ---------------------------- For performance and proxy issues make sure that all internal URLs in the config file -use localhost (use ``curl "http://localhost/" -H Host:`` +use localhost (use ``curl "http://localhost/" --header Host:`` to test it). Tilecloud chain diff --git a/doc/integrator/checker.rst b/doc/integrator/checker.rst index 5d1ce0bb18..c6bce2aea1 100644 --- a/doc/integrator/checker.rst +++ b/doc/integrator/checker.rst @@ -68,6 +68,29 @@ The checker use the following configuration structure in ``vars_.yaml`` checker: forward_headers: ['Cookie', 'Authorisation'] +.. note:: + + The checker assumes that it can access the c2cgeoportal services via ``http://localhost``. + If this is not allowed on your server, you can override this behaviour as follows. + In your ``vars`` file, add the following: + + .. code:: yaml + + vars: + checker: + rewrite_as_http_localhost: False + + Now, in your configuration file ``project.yaml.mako``, instead of defining the ``checker_path``, + define a ``checker_url`` with the full URL to be used, for example: + + .. code:: yaml + + ... + host: ${host} + checker_url: https://${host}/${instanceid}/wsgi/check_collector? + ... + + ``print`` ~~~~~~~~~ diff --git a/doc/integrator/https.rst b/doc/integrator/https.rst index fe323103da..838a39c679 100644 --- a/doc/integrator/https.rst +++ b/doc/integrator/https.rst @@ -78,3 +78,17 @@ Then you can access resources by building urls using the following schema: For example: ``http://geoportail.camptocamp.com/main/wsgi/resourceproxy?target=rfinfo&values=(175,2633)`` + +Local certificate checks +~~~~~~~~~~~~~~~~~~~~~~~~ + +Certain c2cgeoportal features open a http session to your c2cgeoportal services, +for example the ``checker`` or the ``lingua_extractor``. +If you are running your server in https and wish to disable certificate checks in these +connections, you can achieve this by adding the following configuration element to your ``vars`` file: + +.. code:: yaml + + vars: + http_options: + disable_ssl_certificate_validation: True diff --git a/geoportal/c2cgeoportal_geoportal/lib/__init__.py b/geoportal/c2cgeoportal_geoportal/lib/__init__.py index a38d64d929..3ae3a218ec 100644 --- a/geoportal/c2cgeoportal_geoportal/lib/__init__.py +++ b/geoportal/c2cgeoportal_geoportal/lib/__init__.py @@ -30,6 +30,7 @@ import datetime import dateutil +import httplib2 import json import re import urllib.request @@ -55,6 +56,10 @@ def get_types_map(types_array): return types_map +def get_http(request): + return httplib2.Http(**request.registry.settings.get("http_options", {})) + + def get_url(url, request, default=None, errors=None): if url is None: return default diff --git a/geoportal/c2cgeoportal_geoportal/lib/dbreflection.py b/geoportal/c2cgeoportal_geoportal/lib/dbreflection.py index e1ad59451f..bfcdf70943 100644 --- a/geoportal/c2cgeoportal_geoportal/lib/dbreflection.py +++ b/geoportal/c2cgeoportal_geoportal/lib/dbreflection.py @@ -31,7 +31,7 @@ import warnings from typing import Dict, Tuple # noqa, pylint: disable=unused-import -from sqlalchemy import Table, MetaData +from sqlalchemy import Table, MetaData, Column, Integer from sqlalchemy.orm import relationship from sqlalchemy.orm.util import class_mapper from sqlalchemy.exc import SAWarning @@ -125,7 +125,7 @@ def _get_schema(tablename): return tablename, schema -def get_table(tablename, schema=None, session=None): +def get_table(tablename, schema=None, session=None, primary_key=None): if schema is None: tablename, schema = _get_schema(tablename) @@ -144,16 +144,20 @@ def get_table(tablename, schema=None, session=None): "ignore", "Did not recognize type 'geometry' of column", SAWarning) + args = [tablename, metadata] + if primary_key is not None: + # Ensure we have a primary key to be able to edit views + args.append(Column(primary_key, Integer, primary_key=True)) table = Table( - tablename, metadata, + *args, schema=schema, autoload=True, - autoload_with=engine, + autoload_with=engine ) return table -def get_class(tablename, session=None, exclude_properties=None): +def get_class(tablename, session=None, exclude_properties=None, primary_key=None): """ Get the SQLAlchemy mapped class for "tablename". If no class exists for "tablename" one is created, and added to the cache. "tablename" @@ -164,12 +168,12 @@ def get_class(tablename, session=None, exclude_properties=None): if exclude_properties is None: exclude_properties = [] tablename, schema = _get_schema(tablename) - cache_key = (schema, tablename, ",".join(exclude_properties)) + cache_key = (schema, tablename, ",".join(exclude_properties), primary_key) if cache_key in _class_cache: return _class_cache[cache_key] - table = get_table(tablename, schema, session) + table = get_table(tablename, schema, session, primary_key=primary_key) # create the mapped class cls = _create_class(table, exclude_properties) diff --git a/geoportal/c2cgeoportal_geoportal/lib/filter_capabilities.py b/geoportal/c2cgeoportal_geoportal/lib/filter_capabilities.py index 1e2d5b43f0..b3c6a961df 100644 --- a/geoportal/c2cgeoportal_geoportal/lib/filter_capabilities.py +++ b/geoportal/c2cgeoportal_geoportal/lib/filter_capabilities.py @@ -29,7 +29,6 @@ import logging -import httplib2 import copy from io import StringIO from urllib.parse import urlsplit, urljoin @@ -47,7 +46,7 @@ from c2cgeoportal_geoportal.lib import caching, get_protected_layers_query, \ get_writable_layers_query, add_url_params, get_ogc_server_wms_url_ids,\ - get_ogc_server_wfs_url_ids + get_ogc_server_wfs_url_ids, get_http from c2cgeoportal_commons.models import DBSession from c2cgeoportal_commons.models.main import LayerWMS, OGCServer @@ -83,7 +82,7 @@ def get_writable_layers(role_id, ogc_server_ids): @cache_region.cache_on_arguments() -def wms_structure(wms_url, host): +def wms_structure(wms_url, host, request): url = urlsplit(wms_url) wms_url = add_url_params(wms_url, { "SERVICE": "WMS", @@ -92,7 +91,7 @@ def wms_structure(wms_url, host): }) # Forward request to target (without Host Header) - http = httplib2.Http() + http = get_http(request) headers = dict() if url.hostname == "localhost" and host is not None: # pragma: no cover headers["Host"] = host @@ -174,7 +173,7 @@ def filter_capabilities(content, role_id, wms, url, headers, proxies, request): if proxies: # pragma: no cover enable_proxies(proxies) - wms_structure_ = wms_structure(url, headers.get("Host")) + wms_structure_ = wms_structure(url, headers.get("Host"), request) ogc_server_ids = ( get_ogc_server_wms_url_ids(request) if wms else diff --git a/geoportal/c2cgeoportal_geoportal/lib/lingua_extractor.py b/geoportal/c2cgeoportal_geoportal/lib/lingua_extractor.py index 2a66c1a21d..2092eecd8c 100644 --- a/geoportal/c2cgeoportal_geoportal/lib/lingua_extractor.py +++ b/geoportal/c2cgeoportal_geoportal/lib/lingua_extractor.py @@ -58,7 +58,6 @@ from c2cgeoportal_geoportal import init_dbsessions from c2cgeoportal_geoportal.lib import add_url_params, get_url2 from c2cgeoportal_geoportal.lib.bashcolor import colorize, RED -from c2cgeoportal_geoportal.lib.dbreflection import get_class from c2cgeoportal_geoportal.lib.caching import init_region from c2cgeoportal_geoportal.lib.print_ import * # noqa @@ -421,15 +420,9 @@ def _import_layer_wms(self, layer, messages): for wms_layer in layer.layer.split(","): self._import_layer_attributes(url, wms_layer, layer.item_type, layer.name, messages) if layer.geo_table is not None and layer.geo_table != "": - exclude = [] if layer.exclude_properties is None else layer.exclude_properties.split(",") - last_update_date = layer.get_metadatas("lastUpdateDateColumn") - if len(last_update_date) == 1: - exclude.append(last_update_date[0].value) - last_update_user = layer.get_metadatas("lastUpdateUserColumn") - if len(last_update_user) == 1: - exclude.append(last_update_user[0].value) try: - cls = get_class(layer.geo_table, exclude_properties=exclude) + from c2cgeoportal_geoportal.views.layers import get_layer_class + cls = get_layer_class(layer) for column_property in class_mapper(cls).iterate_properties: if isinstance(column_property, ColumnProperty) and len(column_property.columns) == 1: column = column_property.columns[0] diff --git a/geoportal/c2cgeoportal_geoportal/scripts/c2cupgrade.py b/geoportal/c2cgeoportal_geoportal/scripts/c2cupgrade.py index 3e9cf7d71e..4000675b93 100644 --- a/geoportal/c2cgeoportal_geoportal/scripts/c2cupgrade.py +++ b/geoportal/c2cgeoportal_geoportal/scripts/c2cupgrade.py @@ -276,7 +276,7 @@ def test_checkers(self): "Run `curl {} '{}'` for more information." ]).format( ' '.join([ - '--header={}={}'.format(*i) for i in self.project.get("checker_headers", {}).items() + '--header {}={}'.format(*i) for i in self.project.get("checker_headers", {}).items() ]), self.project["checker_url"], ) diff --git a/geoportal/c2cgeoportal_geoportal/views/entry.py b/geoportal/c2cgeoportal_geoportal/views/entry.py index 547b0469e6..8a8fdd39a7 100644 --- a/geoportal/c2cgeoportal_geoportal/views/entry.py +++ b/geoportal/c2cgeoportal_geoportal/views/entry.py @@ -28,7 +28,6 @@ # either expressed or implied, of the FreeBSD Project. -import httplib2 import logging import json import sys @@ -55,7 +54,7 @@ from c2cgeoportal_commons import models from c2cgeoportal_commons.models import main, static from c2cgeoportal_geoportal.lib import get_setting, get_protected_layers_query, \ - get_url2, get_url, get_typed, get_types_map, add_url_params + get_url2, get_url, get_typed, get_types_map, add_url_params, get_http from c2cgeoportal_geoportal.lib.cacheversion import get_cache_version from c2cgeoportal_geoportal.lib.caching import get_region, \ set_common_headers, NO_CACHE, PUBLIC_CACHE, PRIVATE_CACHE @@ -235,7 +234,7 @@ def _wms_getcap(self, ogc_server=None): @cache_region.cache_on_arguments() def get_http_cached(self, url, headers): - http = httplib2.Http() + http = get_http(self.request) return http.request(url, method="GET", headers=headers) @cache_region.cache_on_arguments() diff --git a/geoportal/c2cgeoportal_geoportal/views/layers.py b/geoportal/c2cgeoportal_geoportal/views/layers.py index d911cae0e9..7c87054b76 100644 --- a/geoportal/c2cgeoportal_geoportal/views/layers.py +++ b/geoportal/c2cgeoportal_geoportal/views/layers.py @@ -80,7 +80,7 @@ def _get_geom_col_info(layer): This function assumes that the names of geometry attributes in the mapped class are the same as those of geometry columns. """ - mapped_class = get_class(layer.geo_table) + mapped_class = get_layer_class(layer) for p in class_mapper(mapped_class).iterate_properties: if not isinstance(p, ColumnProperty): continue # pragma: no cover @@ -134,7 +134,7 @@ def _get_layer_for_request(self): def _get_protocol_for_layer(self, layer, **kwargs): """ Returns a papyrus ``Protocol`` for the ``Layer`` object. """ - cls = get_class(layer.geo_table) + cls = get_layer_class(layer) geom_attr = self._get_geom_col_info(layer)[0] return Protocol(models.DBSession, cls, geom_attr, **kwargs) @@ -354,16 +354,16 @@ def _validate_geometry(geom): raise TopologicalError(reason) def _log_last_update(self, layer, feature): - last_update_date = self._get_metadata(layer, "lastUpdateDateColumn") + last_update_date = self.get_metadata(layer, "lastUpdateDateColumn") if last_update_date is not None: setattr(feature, last_update_date, datetime.now()) - last_update_user = self._get_metadata(layer, "lastUpdateUserColumn") + last_update_user = self.get_metadata(layer, "lastUpdateUserColumn") if last_update_user is not None: setattr(feature, last_update_user, self.request.user.id) @staticmethod - def _get_metadata(layer, key, default=None): + def get_metadata(layer, key, default=None): metadatas = layer.get_metadatas(key) if len(metadatas) == 1: metadata = metadatas[0] @@ -372,7 +372,7 @@ def _get_metadata(layer, key, default=None): def _get_validation_setting(self, layer): # The validation UIMetadata is stored as a string, not a boolean - should_validate = self._get_metadata(layer, "geometryValidation", None) + should_validate = self.get_metadata(layer, "geometryValidation", None) if should_validate: return should_validate.lower() != "false" return self.settings.get("geometry_validation", False) @@ -416,19 +416,7 @@ def metadata(self): if not layer.public and self.request.user is None: raise HTTPForbidden() - # exclude the columns used to record the last features update - exclude = [] if layer.exclude_properties is None else layer.exclude_properties.split(",") - last_update_date = self._get_metadata(layer, "lastUpdateDateColumn") - if last_update_date is not None: - exclude.append(last_update_date) - last_update_user = self._get_metadata(layer, "lastUpdateUserColumn") - if last_update_user is not None: - exclude.append(last_update_user) - - return get_class( - layer.geo_table, - exclude_properties=exclude - ) + return get_layer_class(layer) @view_config(route_name="layers_enumerate_attribute_values", renderer="json") def enumerate_attribute_values(self): @@ -481,24 +469,26 @@ def query_enumerate_attribute_values(dbsession, layerinfos, fieldname): return dbsession.query(distinct(attribute)).order_by(attribute).all() -def get_layer_metadatas(layer): +def get_layer_class(layer): # exclude the columns used to record the last features update exclude = [] if layer.exclude_properties is None else layer.exclude_properties.split(",") - - date_metadata = layer.get_metadatas("lastUpdateDateColumn") - last_update_date = date_metadata[0] if len(date_metadata) == 1 else None + last_update_date = Layers.get_metadata(layer, "lastUpdateDateColumn") if last_update_date is not None: - exclude.append(last_update_date.value) - user_metadata = layer.get_metadatas("lastUpdateUserColumn") - last_update_user = user_metadata[0] if len(user_metadata) == 1 else None + exclude.append(last_update_date) + last_update_user = Layers.get_metadata(layer, "lastUpdateUserColumn") if last_update_user is not None: - exclude.append(last_update_user.value) + exclude.append(last_update_user) - cls = get_class( - layer.geo_table, - exclude_properties=exclude + primary_key = Layers.get_metadata(layer, "geotablePrimaryKey") + return get_class( + str(layer.geo_table), + exclude_properties=exclude, + primary_key=primary_key ) + +def get_layer_metadatas(layer): + cls = get_layer_class(layer) edit_columns = [] for column_property in class_mapper(cls).iterate_properties: diff --git a/geoportal/c2cgeoportal_geoportal/views/proxy.py b/geoportal/c2cgeoportal_geoportal/views/proxy.py index 2a8e19d103..170e17edeb 100644 --- a/geoportal/c2cgeoportal_geoportal/views/proxy.py +++ b/geoportal/c2cgeoportal_geoportal/views/proxy.py @@ -29,7 +29,6 @@ import sys -import httplib2 import urllib.request import urllib.parse import urllib.error @@ -40,6 +39,7 @@ from pyramid.response import Response from pyramid.httpexceptions import HTTPBadGateway, exception_response +from c2cgeoportal_geoportal.lib import get_http from c2cgeoportal_geoportal.lib.caching import get_region, \ set_common_headers, NO_CACHE, PUBLIC_CACHE, PRIVATE_CACHE @@ -80,7 +80,7 @@ def _proxy(self, url, params=None, method=None, cache=False, body=None, headers= method = self.request.method # forward request to target (without Host Header) - http = httplib2.Http() + http = get_http(self.request) if headers is None: # pragma: no cover headers = dict(self.request.headers) diff --git a/geoportal/tests/functional/test_dbreflection.py b/geoportal/tests/functional/test_dbreflection.py index ad9dc1f102..61ae3608b2 100644 --- a/geoportal/tests/functional/test_dbreflection.py +++ b/geoportal/tests/functional/test_dbreflection.py @@ -168,7 +168,7 @@ def test_get_class(self): # the class should now be in the cache self.assertTrue( - ("public", "table_a", "") in + ("public", "table_a", "", None) in c2cgeoportal_geoportal.lib.dbreflection._class_cache ) _modelclass = get_class("table_a") @@ -213,7 +213,7 @@ def test_get_class_exclude_properties(self): # the class should now be in the cache self.assertTrue( - ("public", "table_d", "foo,bar") in + ("public", "table_d", "foo,bar", None) in c2cgeoportal_geoportal.lib.dbreflection._class_cache )