From 2619f7741ee4d701bd41be0a8fd76837500f3293 Mon Sep 17 00:00:00 2001 From: Wolfgang Kaltz Date: Fri, 29 Jun 2018 11:40:55 +0200 Subject: [PATCH 1/9] add configuration parameter to http session creation --- c2cgeoportal/lib/lingua_extractor.py | 12 ++++++++++-- c2cgeoportal/scaffolds/update/CONST_Makefile_tmpl | 2 +- c2cgeoportal/scaffolds/update/CONST_vars.yaml_tmpl | 4 ++++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/c2cgeoportal/lib/lingua_extractor.py b/c2cgeoportal/lib/lingua_extractor.py index 4b3b30f249..117eae9ff1 100644 --- a/c2cgeoportal/lib/lingua_extractor.py +++ b/c2cgeoportal/lib/lingua_extractor.py @@ -369,6 +369,14 @@ def _import(object_type, messages, callback=None): if callback is not None: callback(item, messages) + def _get_http(self): + http_options = self.config.get("http_options") + disable_ssl_certificate_validation = False + if http_options is not None: + disable_ssl_certificate_validation = http_options.get("disable_ssl_certificate_validation", False) + http = httplib2.Http(disable_ssl_certificate_validation = disable_ssl_certificate_validation) + return http + def _import_layer_wms(self, layer, messages): server = layer.ogc_server url = server.url_wfs or server.url @@ -472,7 +480,7 @@ def _layer_attributes(self, url, layer): self.wmscap_cache[url] = None # forward request to target (without Host Header) - http = httplib2.Http() + http = self._get_http() h = {} if hostname == "localhost": # pragma: no cover h["Host"] = self.package["host"] @@ -515,7 +523,7 @@ def _layer_attributes(self, url, layer): self.featuretype_cache[url] = None # forward request to target (without Host Header) - http = httplib2.Http() + http = self._get_http() h = {} if hostname == "localhost": # pragma: no cover h["Host"] = self.package["host"] diff --git a/c2cgeoportal/scaffolds/update/CONST_Makefile_tmpl b/c2cgeoportal/scaffolds/update/CONST_Makefile_tmpl index 173b969957..9d0008187d 100644 --- a/c2cgeoportal/scaffolds/update/CONST_Makefile_tmpl +++ b/c2cgeoportal/scaffolds/update/CONST_Makefile_tmpl @@ -491,7 +491,7 @@ CONFIG_VARS += instanceid sqlalchemy.url schema parentschema enable_admin_interf raster shortener hide_capabilities mapserverproxy tinyowsproxy resourceproxy print_url \ tiles_url checker check_collector default_max_age jsbuild package srid \ reset_password fulltextsearch headers authorized_referers hooks stats db_chooser \ - ogcproxy_enable dbsessions urllogin host_forward_host smtp + ogcproxy_enable dbsessions urllogin host_forward_host smtp http_options MAKE_FILES = $(shell ls -1 *.mk) CONST_Makefile # Disabling Make built-in rules to speed up execution time diff --git a/c2cgeoportal/scaffolds/update/CONST_vars.yaml_tmpl b/c2cgeoportal/scaffolds/update/CONST_vars.yaml_tmpl index aac2aeb514..b0e648e37d 100644 --- a/c2cgeoportal/scaffolds/update/CONST_vars.yaml_tmpl +++ b/c2cgeoportal/scaffolds/update/CONST_vars.yaml_tmpl @@ -245,6 +245,10 @@ vars: # architecture. external_themes_url: + # Available options for http sessions created within c2cgeoportal + http_options: + disable_ssl_certificate_validation: False + urllogin: {} mapserverproxy: From 26f3d2b48bba88de3847e8107860f361ee56c01d Mon Sep 17 00:00:00 2001 From: Wolfgang Kaltz Date: Fri, 29 Jun 2018 14:27:53 +0200 Subject: [PATCH 2/9] make flake8 happy --- c2cgeoportal/lib/lingua_extractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c2cgeoportal/lib/lingua_extractor.py b/c2cgeoportal/lib/lingua_extractor.py index 117eae9ff1..a1a9bdab3f 100644 --- a/c2cgeoportal/lib/lingua_extractor.py +++ b/c2cgeoportal/lib/lingua_extractor.py @@ -374,7 +374,7 @@ def _get_http(self): disable_ssl_certificate_validation = False if http_options is not None: disable_ssl_certificate_validation = http_options.get("disable_ssl_certificate_validation", False) - http = httplib2.Http(disable_ssl_certificate_validation = disable_ssl_certificate_validation) + http = httplib2.Http(disable_ssl_certificate_validation=disable_ssl_certificate_validation) return http def _import_layer_wms(self, layer, messages): From 8319137220707c9074108e95b7e1c2f9fc82e7d4 Mon Sep 17 00:00:00 2001 From: Wolfgang Kaltz Date: Fri, 29 Jun 2018 15:18:46 +0200 Subject: [PATCH 3/9] review feedback --- c2cgeoportal/lib/lingua_extractor.py | 7 +------ c2cgeoportal/scaffolds/update/CONST_config-schema.yaml | 7 +++++++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/c2cgeoportal/lib/lingua_extractor.py b/c2cgeoportal/lib/lingua_extractor.py index a1a9bdab3f..d8567d3f1a 100644 --- a/c2cgeoportal/lib/lingua_extractor.py +++ b/c2cgeoportal/lib/lingua_extractor.py @@ -370,12 +370,7 @@ def _import(object_type, messages, callback=None): callback(item, messages) def _get_http(self): - http_options = self.config.get("http_options") - disable_ssl_certificate_validation = False - if http_options is not None: - disable_ssl_certificate_validation = http_options.get("disable_ssl_certificate_validation", False) - http = httplib2.Http(disable_ssl_certificate_validation=disable_ssl_certificate_validation) - return http + return httplib2.Http(**self.config.get("http_options", {})) def _import_layer_wms(self, layer, messages): server = layer.ogc_server diff --git a/c2cgeoportal/scaffolds/update/CONST_config-schema.yaml b/c2cgeoportal/scaffolds/update/CONST_config-schema.yaml index dd5a56658b..129674cad7 100644 --- a/c2cgeoportal/scaffolds/update/CONST_config-schema.yaml +++ b/c2cgeoportal/scaffolds/update/CONST_config-schema.yaml @@ -165,6 +165,13 @@ mapping: required: True sequence: - type: str + http_options: + type: map + required: False + mapping: + disable_ssl_certificate_validation: + type: bool + required: False layers: type: map required: True From f195596f3537f68838751cbcf26f222d0a38f2ce Mon Sep 17 00:00:00 2001 From: Guillaume Beraudo Date: Mon, 2 Jul 2018 17:13:11 +0200 Subject: [PATCH 4/9] Add mechanism to edit views --- c2cgeoportal/lib/dbreflection.py | 16 +++++++----- .../scaffolds/update/CONST_vars.yaml_tmpl | 1 + c2cgeoportal/views/layers.py | 26 +++++++++++-------- doc/administrator/editing.rst | 21 +++++++++++++++ 4 files changed, 47 insertions(+), 17 deletions(-) diff --git a/c2cgeoportal/lib/dbreflection.py b/c2cgeoportal/lib/dbreflection.py index 9a7446f3d0..410d085608 100644 --- a/c2cgeoportal/lib/dbreflection.py +++ b/c2cgeoportal/lib/dbreflection.py @@ -30,7 +30,7 @@ import warnings -from sqlalchemy import Table, sql, MetaData +from sqlalchemy import Table, sql, MetaData, Column, Integer from sqlalchemy.orm import relationship from sqlalchemy.orm.util import class_mapper from sqlalchemy.exc import SAWarning @@ -144,7 +144,7 @@ def _get_schema(tablename): return tablename, schema -def get_table(tablename, schema=None, session=None): +def get_table(tablename, schema=None, session=None, primary_key=None): if schema is None: tablename, schema = _get_schema(tablename) @@ -161,16 +161,20 @@ def get_table(tablename, schema=None, session=None): "ignore", "Did not recognize type 'geometry' of column", SAWarning) + args = [tablename, metadata] + if primary_key is not None: + # Ensure we have a primary key to be able to edit views + args.append(Column(primary_key, Integer, primary_key=True)) table = Table( - tablename, metadata, + *args, schema=schema, autoload=True, - autoload_with=engine, + autoload_with=engine ) return table -def get_class(tablename, session=None, exclude_properties=None): +def get_class(tablename, session=None, exclude_properties=None, primary_key=None): """ Get the SQLAlchemy mapped class for "tablename". If no class exists for "tablename" one is created, and added to the cache. "tablename" @@ -186,7 +190,7 @@ def get_class(tablename, session=None, exclude_properties=None): if cache_key in _class_cache: return _class_cache[cache_key] - table = get_table(tablename, schema, session) + table = get_table(tablename, schema, session, primary_key=primary_key) # create the mapped class cls = _create_class(table, exclude_properties) diff --git a/c2cgeoportal/scaffolds/update/CONST_vars.yaml_tmpl b/c2cgeoportal/scaffolds/update/CONST_vars.yaml_tmpl index b0e648e37d..4502a6451a 100644 --- a/c2cgeoportal/scaffolds/update/CONST_vars.yaml_tmpl +++ b/c2cgeoportal/scaffolds/update/CONST_vars.yaml_tmpl @@ -185,6 +185,7 @@ vars: - name: lastUpdateUserColumn - name: snappingConfig type: json + - name: geotablePrimaryKey - name: geometry_validation # V1 type: boolean - name: excludeProperties # not yet used diff --git a/c2cgeoportal/views/layers.py b/c2cgeoportal/views/layers.py index 4ec385879e..6fd9e999d3 100644 --- a/c2cgeoportal/views/layers.py +++ b/c2cgeoportal/views/layers.py @@ -71,8 +71,7 @@ def __init__(self, request): self.settings = request.registry.settings.get("layers", {}) self.layers_enum_config = self.settings.get("enum") - @staticmethod - def _get_geom_col_info(layer): + def _get_geom_col_info(self, layer): """ Return information about the layer's geometry column, namely a ``(name, srid)`` tuple, where ``name`` is the name of the geometry column, and ``srid`` its srid. @@ -80,7 +79,8 @@ def _get_geom_col_info(layer): This function assumes that the names of geometry attributes in the mapped class are the same as those of geometry columns. """ - mapped_class = get_class(layer.geo_table) + primary_key = self.get_metadata(layer, "geotablePrimaryKey") + mapped_class = get_class(layer.geo_table, primary_key=primary_key) for p in class_mapper(mapped_class).iterate_properties: if not isinstance(p, ColumnProperty): continue # pragma: no cover @@ -354,16 +354,16 @@ def _validate_geometry(geom): raise TopologicalError(reason) def _log_last_update(self, layer, feature): - last_update_date = self._get_metadata(layer, "lastUpdateDateColumn") + last_update_date = self.get_metadata(layer, "lastUpdateDateColumn") if last_update_date is not None: setattr(feature, last_update_date, datetime.now()) - last_update_user = self._get_metadata(layer, "lastUpdateUserColumn") + last_update_user = self.get_metadata(layer, "lastUpdateUserColumn") if last_update_user is not None: setattr(feature, last_update_user, self.request.user.id) @staticmethod - def _get_metadata(layer, key, default=None): + def get_metadata(layer, key, default=None): metadatas = layer.get_metadatas(key) if len(metadatas) == 1: metadata = metadatas[0] @@ -372,7 +372,7 @@ def _get_metadata(layer, key, default=None): def _get_validation_setting(self, layer): # The validation UIMetadata is stored as a string, not a boolean - should_validate = self._get_metadata(layer, "geometry_validation", None) + should_validate = self.get_metadata(layer, "geometry_validation", None) if should_validate: return should_validate.lower() != "false" return self.settings.get("geometry_validation", False) @@ -418,16 +418,18 @@ def metadata(self): # exclude the columns used to record the last features update exclude = [] if layer.exclude_properties is None else layer.exclude_properties.split(",") - last_update_date = self._get_metadata(layer, "lastUpdateDateColumn") + last_update_date = self.get_metadata(layer, "lastUpdateDateColumn") if last_update_date is not None: exclude.append(last_update_date) - last_update_user = self._get_metadata(layer, "lastUpdateUserColumn") + last_update_user = self.get_metadata(layer, "lastUpdateUserColumn") if last_update_user is not None: exclude.append(last_update_user) + primary_key = self.get_metadata(layer, "geotablePrimaryKey") return get_class( str(layer.geo_table), - exclude_properties=exclude + exclude_properties=exclude, + primary_key=primary_key ) @view_config(route_name="layers_enumerate_attribute_values", renderer="json") @@ -495,9 +497,11 @@ def get_layer_metadatas(layer): if last_update_user is not None: exclude.append(last_update_user.value) + primary_key = Layers.get_metadata(layer, "geotablePrimaryKey") cls = get_class( str(layer.geo_table), - exclude_properties=exclude + exclude_properties=exclude, + primary_key=primary_key ) edit_columns = [] diff --git a/doc/administrator/editing.rst b/doc/administrator/editing.rst index 810783d0b4..ef6dc55dec 100644 --- a/doc/administrator/editing.rst +++ b/doc/administrator/editing.rst @@ -132,3 +132,24 @@ Example: * Source and destination layers must have the same geometry type. * Only the geometry will be copied, the attributes will not be. + +Edit views +---------- + +To be able to edit PostgreSQL views a primary key must be manually configured. +Add a layer metadata ``geotablePrimaryKey`` with value the name of the column to use as primary key. +That column must be of type ``Integer``. + +Example: + + * geotablePrimaryKey: ``id`` + +Enable snapping +--------------- + +To be able to snap while editing, the ``snappingConfig`` must be set on the layer metadata. +The value is a ``json`` object containing the following optional properties: + + * edge (boolean): whether to allow snapping on edges or not; + * vertex (boolean): whether to allow snapping on vertices or not; + * tolerance (number): the pixel tolerance. From 943cfd8772ce03fca4cf50d9a1d4d145544269ce Mon Sep 17 00:00:00 2001 From: Wolfgang Kaltz Date: Mon, 2 Jul 2018 11:52:06 +0200 Subject: [PATCH 5/9] add configuration for checker to be able to use https and disabled ssl cert; add configuration to disable ssl cert for other http session creations by c2cgeoportal --- c2cgeoportal/lib/__init__.py | 5 +++ c2cgeoportal/lib/filter_capabilities.py | 9 +++-- c2cgeoportal/lib/lingua_extractor.py | 10 ++---- .../scaffolds/create/project.yaml.mako_tmpl | 3 ++ .../scaffolds/update/CONST_config-schema.yaml | 3 ++ .../scaffolds/update/CONST_vars.yaml_tmpl | 1 + c2cgeoportal/scripts/c2ctool.py | 25 ++++++++++---- c2cgeoportal/views/check_collector.py | 4 +-- c2cgeoportal/views/checker.py | 34 ++++++++++--------- c2cgeoportal/views/entry.py | 9 +++-- c2cgeoportal/views/proxy.py | 4 +-- 11 files changed, 63 insertions(+), 44 deletions(-) diff --git a/c2cgeoportal/lib/__init__.py b/c2cgeoportal/lib/__init__.py index 6284587ae3..1d70807753 100644 --- a/c2cgeoportal/lib/__init__.py +++ b/c2cgeoportal/lib/__init__.py @@ -30,6 +30,7 @@ import datetime import dateutil +import httplib2 import json import re import urllib @@ -53,6 +54,10 @@ def get_types_map(types_array): return types_map +def get_http(request): + return httplib2.Http(**request.registry.settings.get("http_options", {})) + + def get_url(url, request, default=None, errors=None): if url is None: return default diff --git a/c2cgeoportal/lib/filter_capabilities.py b/c2cgeoportal/lib/filter_capabilities.py index 50ca6175ec..55c2e68676 100644 --- a/c2cgeoportal/lib/filter_capabilities.py +++ b/c2cgeoportal/lib/filter_capabilities.py @@ -29,7 +29,6 @@ import logging -import httplib2 import copy from StringIO import StringIO from urlparse import urlsplit, urljoin @@ -47,7 +46,7 @@ from c2cgeoportal.lib import caching, get_protected_layers_query, \ get_writable_layers_query, add_url_params, get_ogc_server_wms_url_ids,\ - get_ogc_server_wfs_url_ids + get_ogc_server_wfs_url_ids, get_http from c2cgeoportal.models import DBSession, LayerWMS, OGCServer cache_region = caching.get_region() @@ -82,7 +81,7 @@ def get_writable_layers(role_id, ogc_server_ids): @cache_region.cache_on_arguments() -def wms_structure(wms_url, host): +def wms_structure(wms_url, host, request): url = urlsplit(wms_url) wms_url = add_url_params(wms_url, { "SERVICE": "WMS", @@ -91,7 +90,7 @@ def wms_structure(wms_url, host): }) # Forward request to target (without Host Header) - http = httplib2.Http() + http = get_http(request) headers = dict() if url.hostname == "localhost" and host is not None: # pragma: no cover headers["Host"] = host @@ -171,7 +170,7 @@ def filter_capabilities(content, role_id, wms, url, headers, proxies, request): if proxies: # pragma: no cover enable_proxies(proxies) - wms_structure_ = wms_structure(url, headers.get("Host")) + wms_structure_ = wms_structure(url, headers.get("Host"), request) ogc_server_ids = ( get_ogc_server_wms_url_ids(request) if wms else diff --git a/c2cgeoportal/lib/lingua_extractor.py b/c2cgeoportal/lib/lingua_extractor.py index d8567d3f1a..b8d08393a6 100644 --- a/c2cgeoportal/lib/lingua_extractor.py +++ b/c2cgeoportal/lib/lingua_extractor.py @@ -28,7 +28,6 @@ # either expressed or implied, of the FreeBSD Project. -import httplib2 import subprocess import os import yaml @@ -53,7 +52,7 @@ from mako.lookup import TemplateLookup from owslib.wms import WebMapService -from c2cgeoportal.lib import add_url_params, get_url2 +from c2cgeoportal.lib import add_url_params, get_url2, get_http from c2cgeoportal.lib.bashcolor import colorize, RED from c2cgeoportal.lib.dbreflection import get_class from c2cgeoportal.lib.caching import init_region @@ -369,9 +368,6 @@ def _import(object_type, messages, callback=None): if callback is not None: callback(item, messages) - def _get_http(self): - return httplib2.Http(**self.config.get("http_options", {})) - def _import_layer_wms(self, layer, messages): server = layer.ogc_server url = server.url_wfs or server.url @@ -475,7 +471,7 @@ def _layer_attributes(self, url, layer): self.wmscap_cache[url] = None # forward request to target (without Host Header) - http = self._get_http() + http = get_http(request) h = {} if hostname == "localhost": # pragma: no cover h["Host"] = self.package["host"] @@ -518,7 +514,7 @@ def _layer_attributes(self, url, layer): self.featuretype_cache[url] = None # forward request to target (without Host Header) - http = self._get_http() + http = get_http(request) h = {} if hostname == "localhost": # pragma: no cover h["Host"] = self.package["host"] diff --git a/c2cgeoportal/scaffolds/create/project.yaml.mako_tmpl b/c2cgeoportal/scaffolds/create/project.yaml.mako_tmpl index 146bf119d7..092ae32080 100644 --- a/c2cgeoportal/scaffolds/create/project.yaml.mako_tmpl +++ b/c2cgeoportal/scaffolds/create/project.yaml.mako_tmpl @@ -1,7 +1,10 @@ project_folder: {{project}} project_package: ${package} host: ${host} +# Legacy configuration option for checker checker_path: /${instanceid}/wsgi/check_collector? +# New configuration option for checker +#checker_url: https://${host}/${instanceid}/wsgi/check_collector? template_vars: package: {{package}} srid: {{srid}} diff --git a/c2cgeoportal/scaffolds/update/CONST_config-schema.yaml b/c2cgeoportal/scaffolds/update/CONST_config-schema.yaml index 129674cad7..0a3198a219 100644 --- a/c2cgeoportal/scaffolds/update/CONST_config-schema.yaml +++ b/c2cgeoportal/scaffolds/update/CONST_config-schema.yaml @@ -436,6 +436,9 @@ mapping: regex;(.+): type: any + rewrite_as_http_localhost: + type: bool + required: False check_collector: type: map required: True diff --git a/c2cgeoportal/scaffolds/update/CONST_vars.yaml_tmpl b/c2cgeoportal/scaffolds/update/CONST_vars.yaml_tmpl index b0e648e37d..74d106c1cf 100644 --- a/c2cgeoportal/scaffolds/update/CONST_vars.yaml_tmpl +++ b/c2cgeoportal/scaffolds/update/CONST_vars.yaml_tmpl @@ -426,6 +426,7 @@ vars: routes: - name: apijs - name: printproxy_capabilities + rewrite_as_http_localhost: True # Check collector configuration check_collector: diff --git a/c2cgeoportal/scripts/c2ctool.py b/c2cgeoportal/scripts/c2ctool.py index 11fe862fb5..bb13d4e2ab 100644 --- a/c2cgeoportal/scripts/c2ctool.py +++ b/c2cgeoportal/scripts/c2ctool.py @@ -214,21 +214,32 @@ def get_project(): return yaml.safe_load(f) def test_checkers(self): - http = httplib2.Http() + # Read project options for checker http session connection + checker_url_base = self.project.get("checker_url") + # If parameter not present, use legacy parameter + if checker_url_base is None: + print("test_checkers missing configuration setting checker_url, using legacy configuration") + checker_url_base = "http://localhost{}".format(self.project["checker_path"]) + + http = httplib2.Http(disable_ssl_certificate_validation=True) for check_type in ("", "type=all"): + checker_url = "{}{}".format(checker_url_base, check_type) + checker_headers = {"Host": self.project["host"]} + print("Calling checker via URL {}, headers {}".format(checker_url, checker_headers)) resp, _ = http.request( - "http://localhost{}{}".format(self.project["checker_path"], check_type), + checker_url, method="GET", - headers={ - "Host": self.project["host"] - } + headers=checker_headers ) if resp.status < 200 or resp.status >= 300: return False, "\n".join([ "Checker error:", - "Open `http://{}{}{}` for more informations." + "Run `curl {} '{}'` for more information." ]).format( - self.project["host"], self.project["checker_path"], check_type + ' '.join([ + '--header={}={}'.format(*i) for i in checker_headers.items() + ]), + checker_url ) return True, None diff --git a/c2cgeoportal/views/check_collector.py b/c2cgeoportal/views/check_collector.py index f60309cefb..eee88294ef 100644 --- a/c2cgeoportal/views/check_collector.py +++ b/c2cgeoportal/views/check_collector.py @@ -30,12 +30,12 @@ import logging import httplib -from httplib2 import Http from time import time from pyramid.view import view_config from pyramid.response import Response +from c2cgeoportal.lib import get_http from c2cgeoportal.views.checker import build_url log = logging.getLogger(__name__) @@ -84,7 +84,7 @@ def check_collector(self): def _testurl(self, url): url, headers = build_url("Collect", url, self.request) - h = Http() + h = get_http(self.request) resp, content = h.request(url, headers=headers) if resp.status != httplib.OK: diff --git a/c2cgeoportal/views/checker.py b/c2cgeoportal/views/checker.py index 26a3ead098..6ceedf5e47 100644 --- a/c2cgeoportal/views/checker.py +++ b/c2cgeoportal/views/checker.py @@ -32,7 +32,6 @@ import urllib import httplib import logging -from httplib2 import Http from json import dumps, loads from time import sleep from urlparse import urlsplit, urlunsplit @@ -41,7 +40,7 @@ from pyramid.view import view_config from pyramid.response import Response -from c2cgeoportal.lib import add_url_params +from c2cgeoportal.lib import add_url_params, get_http log = logging.getLogger(__name__) @@ -51,17 +50,20 @@ def build_url(name, url, request, headers=None): headers = {} headers["Cache-Control"] = "no-cache" - urlfragments = urlsplit(url) - if urlfragments.netloc == request.environ.get("SERVER_NAME") or \ - urlfragments.netloc.startswith("localhost:"): - url_ = urlunsplit(( - "http", "localhost", urlfragments.path, urlfragments.query, urlfragments.fragment - )) - headers["Host"] = urlfragments.netloc - else: - url_ = url - settings = request.registry.settings.get("checker", {}) + rewrite_as_http_localhost = settings.get("rewrite_as_http_localhost", True) + log.info("build_url rewrite_as_http_localhost: {}".format(rewrite_as_http_localhost)) + + url_ = url + if rewrite_as_http_localhost: + urlfragments = urlsplit(url) + if urlfragments.netloc == request.environ.get("SERVER_NAME") or \ + urlfragments.netloc.startswith("localhost:"): + url_ = urlunsplit(( + "http", "localhost", urlfragments.path, urlfragments.query, urlfragments.fragment + )) + headers["Host"] = urlfragments.netloc + for header in settings.get("forward_headers", []): value = request.headers.get(header) if value is not None: @@ -96,7 +98,7 @@ def make_response(self, msg): def testurl(self, url): url, headers = build_url("Check", url, self.request) - h = Http() + h = get_http(self.request) resp, content = h.request(url, headers=headers) if resp.status != httplib.OK: @@ -142,7 +144,7 @@ def _pdf3(self): "Content-Type": "application/json;charset=utf-8", }) - h = Http() + h = get_http(self.request) resp, content = h.request(url, "POST", headers=headers, body=body) if resp.status != httplib.OK: @@ -187,7 +189,7 @@ def _fts(self): }) url, headers = build_url("Check the fulltextsearch", url, self.request) - h = Http() + h = get_http(self.request) resp, content = h.request(url, headers=headers) if resp.status != httplib.OK: @@ -209,7 +211,7 @@ def themes_errors(self): settings = self.settings.get("themes", {}) url = self.request.route_url("themes") - h = Http() + h = get_http(self.request) default_params = settings.get("default", {}).get("params", {}) results = [] for interface, in DBSession.query(Interface.name).all(): diff --git a/c2cgeoportal/views/entry.py b/c2cgeoportal/views/entry.py index 81aefbd24a..10804e7f44 100644 --- a/c2cgeoportal/views/entry.py +++ b/c2cgeoportal/views/entry.py @@ -28,7 +28,6 @@ # either expressed or implied, of the FreeBSD Project. -import httplib2 import logging import json import sys @@ -52,7 +51,7 @@ from owslib.wms import WebMapService from c2cgeoportal.lib import get_setting, get_protected_layers_query, \ - get_url2, get_url, get_typed, get_types_map, add_url_params + get_url2, get_url, get_typed, get_types_map, add_url_params, get_http from c2cgeoportal.lib.cacheversion import get_cache_version from c2cgeoportal.lib.caching import get_region, invalidate_region, \ set_common_headers, NO_CACHE, PUBLIC_CACHE, PRIVATE_CACHE @@ -232,7 +231,7 @@ def _wms_getcap_cached(self, ogc_server, role_id): log.info(u"Get WMS GetCapabilities for url: {0!s}".format(url)) # forward request to target (without Host Header) - http = httplib2.Http() + http = get_http(self.request) headers = dict(self.request.headers) role = None if self.request.user is None else self.request.user.role @@ -1088,7 +1087,7 @@ def _wfs_types_cached(self, wfs_url): log.info(u"WFS GetCapabilities for base url: {0!s}".format(wfsgc_url)) # forward request to target (without Host Header) - http = httplib2.Http() + http = get_http(self.request) headers = dict(self.request.headers) if urlparse.urlsplit(wfsgc_url).hostname != "localhost" and "Host" in headers: # pragma: no cover headers.pop("Host") @@ -1156,7 +1155,7 @@ def _external_themes_role(self, interface, role_id): # pragma nocover ]) # forward request to target (without Host Header) - http = httplib2.Http() + http = get_http(self.request) headers = dict(self.request.headers) if urlparse.urlsplit(ext_url).hostname != "localhost" and "Host" in headers: # pragma: no cover headers.pop("Host") diff --git a/c2cgeoportal/views/proxy.py b/c2cgeoportal/views/proxy.py index 734c6c97d2..7900f4baca 100644 --- a/c2cgeoportal/views/proxy.py +++ b/c2cgeoportal/views/proxy.py @@ -29,7 +29,6 @@ import sys -import httplib2 import urllib import logging @@ -39,6 +38,7 @@ from pyramid.response import Response from pyramid.httpexceptions import HTTPBadGateway, HTTPInternalServerError +from c2cgeoportal.lib import get_http from c2cgeoportal.lib.caching import get_region, \ set_common_headers, NO_CACHE, PUBLIC_CACHE, PRIVATE_CACHE @@ -82,7 +82,7 @@ def _proxy(self, url, params=None, method=None, cache=False, body=None, headers= method = self.request.method # forward request to target (without Host Header) - http = httplib2.Http() + http = get_http(self.request) if headers is None: # pragma: no cover headers = dict(self.request.headers) From 496e2bb23194c23b99c72ed377714fc3610369dc Mon Sep 17 00:00:00 2001 From: Guillaume Beraudo Date: Tue, 3 Jul 2018 17:01:22 +0200 Subject: [PATCH 6/9] Centralize the get_layer_class code in a unique place In addition, this should ensure all created classes for a layer are consistent. --- c2cgeoportal/lib/lingua_extractor.py | 11 ++------ c2cgeoportal/views/layers.py | 40 +++++++++------------------- 2 files changed, 14 insertions(+), 37 deletions(-) diff --git a/c2cgeoportal/lib/lingua_extractor.py b/c2cgeoportal/lib/lingua_extractor.py index d8567d3f1a..235eb00a6a 100644 --- a/c2cgeoportal/lib/lingua_extractor.py +++ b/c2cgeoportal/lib/lingua_extractor.py @@ -55,7 +55,6 @@ from c2cgeoportal.lib import add_url_params, get_url2 from c2cgeoportal.lib.bashcolor import colorize, RED -from c2cgeoportal.lib.dbreflection import get_class from c2cgeoportal.lib.caching import init_region @@ -380,15 +379,9 @@ def _import_layer_wms(self, layer, messages): for wms_layer in layer.layer.split(","): self._import_layer_attributes(url, wms_layer, layer.item_type, layer.name, messages) if layer.geo_table is not None and layer.geo_table != "": - exclude = [] if layer.exclude_properties is None else layer.exclude_properties.split(",") - last_update_date = layer.get_metadatas("lastUpdateDateColumn") - if len(last_update_date) == 1: - exclude.append(last_update_date[0].value) - last_update_user = layer.get_metadatas("lastUpdateUserColumn") - if len(last_update_user) == 1: - exclude.append(last_update_user[0].value) try: - cls = get_class(layer.geo_table, exclude_properties=exclude) + from c2cgeoportal.views.layers import get_layer_class + cls = get_layer_class(layer) for column_property in class_mapper(cls).iterate_properties: if isinstance(column_property, ColumnProperty) and len(column_property.columns) == 1: column = column_property.columns[0] diff --git a/c2cgeoportal/views/layers.py b/c2cgeoportal/views/layers.py index 6fd9e999d3..7edd121d8d 100644 --- a/c2cgeoportal/views/layers.py +++ b/c2cgeoportal/views/layers.py @@ -79,8 +79,7 @@ def _get_geom_col_info(self, layer): This function assumes that the names of geometry attributes in the mapped class are the same as those of geometry columns. """ - primary_key = self.get_metadata(layer, "geotablePrimaryKey") - mapped_class = get_class(layer.geo_table, primary_key=primary_key) + mapped_class = get_layer_class(layer) for p in class_mapper(mapped_class).iterate_properties: if not isinstance(p, ColumnProperty): continue # pragma: no cover @@ -134,7 +133,7 @@ def _get_layer_for_request(self): def _get_protocol_for_layer(self, layer, **kwargs): """ Returns a papyrus ``Protocol`` for the ``Layer`` object. """ - cls = get_class(layer.geo_table) + cls = get_layer_class(layer) geom_attr = self._get_geom_col_info(layer)[0] return Protocol(DBSession, cls, geom_attr, **kwargs) @@ -415,22 +414,7 @@ def metadata(self): layer = self._get_layer_for_request() if not layer.public and self.request.user is None: raise HTTPForbidden() - - # exclude the columns used to record the last features update - exclude = [] if layer.exclude_properties is None else layer.exclude_properties.split(",") - last_update_date = self.get_metadata(layer, "lastUpdateDateColumn") - if last_update_date is not None: - exclude.append(last_update_date) - last_update_user = self.get_metadata(layer, "lastUpdateUserColumn") - if last_update_user is not None: - exclude.append(last_update_user) - - primary_key = self.get_metadata(layer, "geotablePrimaryKey") - return get_class( - str(layer.geo_table), - exclude_properties=exclude, - primary_key=primary_key - ) + return get_layer_class(layer) @view_config(route_name="layers_enumerate_attribute_values", renderer="json") def enumerate_attribute_values(self): @@ -484,26 +468,26 @@ def query_enumerate_attribute_values(dbsession, layerinfos, fieldname): return dbsession.query(distinct(attribute)).order_by(attribute).all() -def get_layer_metadatas(layer): +def get_layer_class(layer): # exclude the columns used to record the last features update exclude = [] if layer.exclude_properties is None else layer.exclude_properties.split(",") - - date_metadata = layer.get_metadatas("lastUpdateDateColumn") - last_update_date = date_metadata[0] if len(date_metadata) == 1 else None + last_update_date = Layers.get_metadata(layer, "lastUpdateDateColumn") if last_update_date is not None: - exclude.append(last_update_date.value) - user_metadata = layer.get_metadatas("lastUpdateUserColumn") - last_update_user = user_metadata[0] if len(user_metadata) == 1 else None + exclude.append(last_update_date) + last_update_user = Layers.get_metadata(layer, "lastUpdateUserColumn") if last_update_user is not None: - exclude.append(last_update_user.value) + exclude.append(last_update_user) primary_key = Layers.get_metadata(layer, "geotablePrimaryKey") - cls = get_class( + return get_class( str(layer.geo_table), exclude_properties=exclude, primary_key=primary_key ) + +def get_layer_metadatas(layer): + cls = get_layer_class(layer) edit_columns = [] for column_property in class_mapper(cls).iterate_properties: From 3ab75f0d1a23c1261e49d7f08b3394d52986ca7b Mon Sep 17 00:00:00 2001 From: Guillaume Beraudo Date: Wed, 4 Jul 2018 10:17:08 +0200 Subject: [PATCH 7/9] Correctly compute the cache key --- c2cgeoportal/lib/dbreflection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c2cgeoportal/lib/dbreflection.py b/c2cgeoportal/lib/dbreflection.py index 410d085608..9a0591d7d8 100644 --- a/c2cgeoportal/lib/dbreflection.py +++ b/c2cgeoportal/lib/dbreflection.py @@ -185,7 +185,7 @@ def get_class(tablename, session=None, exclude_properties=None, primary_key=None if exclude_properties is None: exclude_properties = [] tablename, schema = _get_schema(tablename) - cache_key = (schema, tablename, ",".join(exclude_properties)) + cache_key = (schema, tablename, ",".join(exclude_properties), primary_key) if cache_key in _class_cache: return _class_cache[cache_key] From 0d45681254cfd0de6654e704a82886d90ad2116e Mon Sep 17 00:00:00 2001 From: Wolfgang Kaltz Date: Wed, 4 Jul 2018 16:51:45 +0200 Subject: [PATCH 8/9] document new configuration options for local http sessions (starting gmf 2.2.5) --- doc/integrator/checker.rst | 25 ++++++++++++++++++++++++- doc/integrator/https.rst | 14 ++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/doc/integrator/checker.rst b/doc/integrator/checker.rst index d66d16adc5..65f85bdf6e 100644 --- a/doc/integrator/checker.rst +++ b/doc/integrator/checker.rst @@ -26,7 +26,7 @@ The return code are: Checker ------- -The checker use the following configuration structure: +The checker uses the following configuration structure: .. code:: yaml @@ -50,6 +50,29 @@ The checker use the following configuration structure: checker: forward_headers: ['Cookie', 'Authorisation'] +.. note:: + + The checker assumes that it can access the c2cgeoportal services via ``http://localhost``. + If this is not allowed on your server, you can override this behaviour as follows. + In your ``vars`` file, add the following: + + .. code:: yaml + + vars: + checker: + rewrite_as_http_localhost: False + + Now, in your configuration file ``project.yaml.mako``, instead of defining the ``checker_path``, + define a ``checker_url`` with the full URL to be used, for example: + + .. code:: yaml + + ... + host: ${host} + checker_url: https://${host}/${instanceid}/wsgi/check_collector? + ... + + ``checker_pdf`` ~~~~~~~~~~~~~~~ diff --git a/doc/integrator/https.rst b/doc/integrator/https.rst index 1380eec08e..8f120aa975 100644 --- a/doc/integrator/https.rst +++ b/doc/integrator/https.rst @@ -81,3 +81,17 @@ Then you can access resources by building urls using the following schema: For example: ``http://geoportail.camptocamp.com/main/wsgi/resourceproxy?target=rfinfo&values=(175,2633)`` + +Local certificate checks +~~~~~~~~~~~~~~~~~~~~~~~~ + +Certain c2cgeoportal features open a http session to your c2cgeoportal services, +for example the ``checker`` or the ``lingua_extractor``. +If you are running your server in https and wish to disable certificate checks in these +connections, you can achieve this by adding the following configuration element to your ``vars`` file: + +.. code:: yaml + + vars: + http_options: + disable_ssl_certificate_validation: True From 12f39f3eed48c4d16e64c954ac89535192ea9836 Mon Sep 17 00:00:00 2001 From: Guillaume Beraudo Date: Wed, 4 Jul 2018 13:55:26 +0200 Subject: [PATCH 9/9] Fix tests --- c2cgeoportal/tests/functional/test_dbreflection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/c2cgeoportal/tests/functional/test_dbreflection.py b/c2cgeoportal/tests/functional/test_dbreflection.py index 91fe12f31c..c7674cee17 100644 --- a/c2cgeoportal/tests/functional/test_dbreflection.py +++ b/c2cgeoportal/tests/functional/test_dbreflection.py @@ -175,7 +175,7 @@ def test_get_class(self): # the class should now be in the cache self.assertTrue( - ("public", "table_a", "") in + ("public", "table_a", "", None) in c2cgeoportal.lib.dbreflection._class_cache ) _modelclass = get_class("table_a") @@ -220,7 +220,7 @@ def test_get_class_exclude_properties(self): # the class should now be in the cache self.assertTrue( - ("public", "table_d", "foo,bar") in + ("public", "table_d", "foo,bar", None) in c2cgeoportal.lib.dbreflection._class_cache )