From abab474ea2fad73da10dcddea71499f82faee065 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Bovard?= Date: Mon, 13 May 2019 14:57:54 +0200 Subject: [PATCH 1/6] Add doc for DEM nodata value --- doc/integrator/raster.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/integrator/raster.rst b/doc/integrator/raster.rst index 099bbc2c4e..4be8064c35 100644 --- a/doc/integrator/raster.rst +++ b/doc/integrator/raster.rst @@ -41,6 +41,9 @@ We recommend to use a `vrt `_ file built ``round`` specifies how the result values should be rounded. For instance '1': round to the unit, '0.01': round to the hundredth, etc. +``nodata`` specifies the nodata value. +By default the nodata value is directly read from the source. + The application viewer should be configured with one (or more) of the ``ContextualData`` and the ``Profile`` ``CGXP`` plugins for the DEM data to be viewable in the web application. From 5ff697e5953449ea1988b73d812e39f6e0d50f2c Mon Sep 17 00:00:00 2001 From: CI Date: Mon, 13 May 2019 14:36:07 +0000 Subject: [PATCH 2/6] Update the minor version --- travis/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis/ci.yaml b/travis/ci.yaml index 309bcd1c1e..b17f0c66c7 100644 --- a/travis/ci.yaml +++ b/travis/ci.yaml @@ -1,2 +1,2 @@ --- -minor: 17 +minor: 18 From 4668e284effb97c5bffff499f12b7033912ffc43 Mon Sep 17 00:00:00 2001 From: "arnaud.morvan@camptocamp.com" Date: Wed, 13 Mar 2019 17:21:49 +0100 Subject: [PATCH 3/6] Allow static urls in url metadatas --- admin/acceptance_tests/metadatas_test.py | 8 ++++++-- admin/c2cgeoportal_admin/schemas/metadata.py | 3 ++- commons/c2cgeoportal_commons/lib/validators.py | 6 ++++++ commons/tests/validators_test.py | 17 +++++++++++++++++ 4 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 commons/c2cgeoportal_commons/lib/validators.py create mode 100644 commons/tests/validators_test.py diff --git a/admin/acceptance_tests/metadatas_test.py b/admin/acceptance_tests/metadatas_test.py index ee846a2415..369a6638e8 100644 --- a/admin/acceptance_tests/metadatas_test.py +++ b/admin/acceptance_tests/metadatas_test.py @@ -192,13 +192,17 @@ def test_invalid_url_metadata(self, test_app, metadatas_test_data): 'gnagnagna', 'Must be a URL') - def test_valid_url_metadata(self, test_app, metadatas_test_data): + @pytest.mark.parametrize("url", [ + 'www.111111111111111111111111111111111111111111111111111111111111.com', + 'static://static/img/cadastre.jpeg', + ]) + def test_valid_url_metadata(self, test_app, metadatas_test_data, url): self._post_metadata( test_app, '/layers_wms/new', self._base_metadata_params(metadatas_test_data), '_url', - 'www.111111111111111111111111111111111111111111111111111111111111.com', + url, 302) def test_invalid_json_metadata(self, test_app, metadatas_test_data): diff --git a/admin/c2cgeoportal_admin/schemas/metadata.py b/admin/c2cgeoportal_admin/schemas/metadata.py index f8993db465..bb7a31c326 100644 --- a/admin/c2cgeoportal_admin/schemas/metadata.py +++ b/admin/c2cgeoportal_admin/schemas/metadata.py @@ -2,6 +2,7 @@ import colander from deform.widget import MappingWidget, SelectWidget, SequenceWidget, TextAreaWidget from c2cgeoform.schema import GeoFormSchemaNode +from c2cgeoportal_commons.lib.validators import url from c2cgeoportal_commons.models.main import Metadata from c2cgeoportal_admin import _ @@ -59,7 +60,7 @@ def __init__(self, *args, **kw): self._add_value_node('boolean', colander.Boolean()) self._add_value_node('int', colander.Int()) self._add_value_node('float', colander.Float()) - self._add_value_node('url', colander.String(), validator=colander.url) + self._add_value_node('url', colander.String(), validator=url) self._add_value_node( 'json', colander.String(), diff --git a/commons/c2cgeoportal_commons/lib/validators.py b/commons/c2cgeoportal_commons/lib/validators.py new file mode 100644 index 0000000000..92e9ff03e7 --- /dev/null +++ b/commons/c2cgeoportal_commons/lib/validators.py @@ -0,0 +1,6 @@ +import colander +import re + +# Custom url validator that allow c2cgeoportal static urls. +URL_REGEX = r'(?:{}|^(:?static|config)://\S+$)'.format(colander.URL_REGEX) +url = colander.Regex(URL_REGEX, msg=colander._('Must be a URL'), flags=re.IGNORECASE) diff --git a/commons/tests/validators_test.py b/commons/tests/validators_test.py new file mode 100644 index 0000000000..6cb71ea0c2 --- /dev/null +++ b/commons/tests/validators_test.py @@ -0,0 +1,17 @@ +# pylint: disable=no-self-use + +import pytest +from c2cgeoportal_commons.lib.validators import url + + +class TestUrl(): + + @pytest.mark.parametrize("valid_url", [ + 'http://geomapfish.org', + 'https://geomapfish.org/functionalities', + 'geomapfish.org/functionalities', + 'static://img/cadastre.jpeg', + 'config://internal/mapserv' + ]) + def test_valid_url(self, valid_url): + url(None, valid_url) From 4dbe572e65ad8f0f411cf767a86838d6c9374068 Mon Sep 17 00:00:00 2001 From: CI Date: Wed, 15 May 2019 14:30:27 +0000 Subject: [PATCH 4/6] Update the minor version --- travis/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis/ci.yaml b/travis/ci.yaml index b17f0c66c7..2abfcb4088 100644 --- a/travis/ci.yaml +++ b/travis/ci.yaml @@ -1,2 +1,2 @@ --- -minor: 18 +minor: 19 From 7592b2dca55a7df2f8fc314eed1111163a18faac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Wed, 15 May 2019 11:46:17 +0200 Subject: [PATCH 5/6] Fix the cache when we call the application from different URL --- .../c2cgeoportal_geoportal/views/entry.py | 50 +++++++++++-------- geoportal/tests/functional/test_entry.py | 36 ++++++------- geoportal/tests/functional/test_looptheme.py | 2 +- 3 files changed, 47 insertions(+), 41 deletions(-) diff --git a/geoportal/c2cgeoportal_geoportal/views/entry.py b/geoportal/c2cgeoportal_geoportal/views/entry.py index 2842c861a5..616fa96f51 100644 --- a/geoportal/c2cgeoportal_geoportal/views/entry.py +++ b/geoportal/c2cgeoportal_geoportal/views/entry.py @@ -783,8 +783,9 @@ def _is_internal_wms(layer): isinstance(layer, main.LayerV1) and layer.layer_type == "internal WMS" @cache_region.cache_on_arguments() - def _get_ogc_servers(self, group, depth): + def _get_ogc_servers(self, host_url, group, depth): """ Recurse on all children to get unique identifier for each child. """ + ogc_servers = set() # escape loop @@ -795,7 +796,7 @@ def _get_ogc_servers(self, group, depth): # recurse on children if isinstance(group, main.LayerGroup) and len(group.children) > 0: for tree_item in group.children: - ogc_servers.update(self._get_ogc_servers(tree_item, depth + 1)) + ogc_servers.update(self._get_ogc_servers(host_url, tree_item, depth + 1)) if isinstance(group, main.LayerWMS): ogc_servers.add(group.ogc_server.name) @@ -806,7 +807,7 @@ def _get_ogc_servers(self, group, depth): return ogc_servers def _group( - self, path, group, layers, depth=1, min_levels=1, + self, host_url, path, group, layers, depth=1, min_levels=1, catalogue=True, role_id=None, version=1, mixed=True, time=None, dim=None, wms_layers=None, layers_name=None, **kwargs ): @@ -830,7 +831,7 @@ def _group( ogc_servers = None org_depth = depth if depth == 1: - ogc_servers = list(self._get_ogc_servers(group, 1)) + ogc_servers = list(self._get_ogc_servers(host_url, group, 1)) # check if mixed content mixed = len(ogc_servers) != 1 or ogc_servers[0] is False if not mixed: @@ -842,7 +843,7 @@ def _group( if isinstance(group, main.Theme) or catalogue or \ group.is_internal_wms == tree_item.is_internal_wms: gp, gp_errors = self._group( - u"{0!s}/{1!s}".format(path, tree_item.name), + host_url, u"{0!s}/{1!s}".format(path, tree_item.name), tree_item, layers, depth=depth + 1, min_levels=min_levels, catalogue=catalogue, role_id=role_id, version=version, mixed=mixed, time=time, dim=dim, wms_layers=wms_layers, layers_name=layers_name, **kwargs @@ -922,7 +923,8 @@ def _group( return None, errors @cache_region.cache_on_arguments() - def _layers(self, role_id, version, interface): + def _layers(self, host_url, role_id, version, interface): + del host_url # Only for cache query = self._create_layer_query(role_id, version, interface=interface) return [name for (name,) in query.all()] @@ -960,7 +962,7 @@ def _load_tree_items(self): @cache_region.cache_on_arguments() def _themes( - self, role_id, interface="desktop", filter_themes=True, version=1, + self, host_url, role_id, interface="desktop", filter_themes=True, version=1, catalogue=False, min_levels=1 ): """ @@ -969,7 +971,7 @@ def _themes( """ self._load_tree_items() errors = set() - layers = self._layers(role_id, version, interface) + layers = self._layers(host_url, role_id, version, interface) themes = models.DBSession.query(main.Theme) themes = themes.filter(main.Theme.public.is_(True)) @@ -994,7 +996,7 @@ def _themes( continue children, children_errors = self._get_children( - theme, layers, version, catalogue, min_levels, role_id + host_url, theme, layers, version, catalogue, min_levels, role_id ) errors |= children_errors @@ -1041,12 +1043,13 @@ def invalidate_cache(): # pragma: no cover "success": True } - def _get_children(self, theme, layers, version, catalogue, min_levels, role_id): + def _get_children(self, host_url, theme, layers, version, catalogue, min_levels, role_id): children = [] errors = set() for item in theme.children: if isinstance(item, main.LayerGroup): gp, gp_errors = self._group( + host_url, "{0!s}/{1!s}".format(theme.name, item.name), item, layers, role_id=role_id, version=version, catalogue=catalogue, @@ -1161,14 +1164,15 @@ def _wfs_types_cached(self, wfs_url): except Exception: # pragma: no cover return get_capabilities_xml, errors - def _functionality(self): + def _functionality(self, host_url): return self._functionality_cached( - self.request.user.role.name if self.request.user is not None else None + host_url, self.request.user.role.name if self.request.user is not None else None ) @cache_region.cache_on_arguments() - def _functionality_cached(self, role): + def _functionality_cached(self, host_url, role): del role # Just for caching + del host_url # Just for caching functionality = {} for func_ in get_setting( self.settings, @@ -1235,7 +1239,7 @@ def get_cgxp_viewer_vars(self): role_id = self._get_role_id() interface = self.request.interface_name - themes, errors = self._themes(role_id, interface) + themes, errors = self._themes(self.request.host_url, role_id, interface) wfs_types, add_errors = self._internal_wfs_types(role_id) errors |= add_errors @@ -1255,7 +1259,7 @@ def get_cgxp_viewer_vars(self): "user": self.request.user, "WFSTypes": json.dumps(wfs_types), "tiles_url": json.dumps(self.settings.get("tiles_url")), - "functionality": self._functionality(), + "functionality": self._functionality(self.request.host_url), "queryer_attribute_urls": json.dumps(self._get_layers_enum()), "serverError": json.dumps(list(errors)), "version_params": version_params, @@ -1433,7 +1437,7 @@ def themes(self): } if export_themes: themes, errors = self._themes( - role_id, interface, True, version, catalogue, min_levels + self.request.host_url, role_id, interface, True, version, catalogue, min_levels ) if version == 1: @@ -1443,13 +1447,15 @@ def themes(self): all_errors |= errors if export_group: - group, errors = self._get_group(group, role_id, interface, version) + group, errors = self._get_group(self.request.host_url, group, role_id, interface, version) if group is not None: result["group"] = group all_errors |= errors if export_background: - group, errors = self._get_group(background_layers_group, role_id, interface, version) + group, errors = self._get_group( + self.request.host_url, background_layers_group, role_id, interface, version + ) result["background_layers"] = group["children"] if group is not None else [] all_errors |= errors @@ -1458,12 +1464,12 @@ def themes(self): log.info("Theme errors:\n{}".format("\n".join(all_errors))) return result - def _get_group(self, group, role_id, interface, version): - layers = self._layers(role_id, version, interface) + def _get_group(self, host_url, group, role_id, interface, version): + layers = self._layers(host_url, role_id, version, interface) try: lg = models.DBSession.query(main.LayerGroup).filter(main.LayerGroup.name == group).one() return self._group( - lg.name, lg, layers, + host_url, lg.name, lg, layers, role_id=role_id, version=version ) except NoResultFound: # pragma: no cover @@ -1566,7 +1572,7 @@ def _user(self, user=None): "role_id": user.role.id } if user else {} - result["functionalities"] = self._functionality() + result["functionalities"] = self._functionality(self.request.host_url) return result diff --git a/geoportal/tests/functional/test_entry.py b/geoportal/tests/functional/test_entry.py index 8b76717c5f..fabcbbbcb5 100644 --- a/geoportal/tests/functional/test_entry.py +++ b/geoportal/tests/functional/test_entry.py @@ -426,7 +426,7 @@ def test_theme(self): entry = Entry(request) # unautenticated - themes, errors = entry._themes(None, "desktop") + themes, errors = entry._themes('', None, "desktop") self.assertEqual(errors, { "The layer '__test_layer_in_group' (__test_layer_in_group) is not defined in WMS capabilities from '__test_ogc_server'", }) @@ -452,7 +452,7 @@ def test_theme(self): # authenticated request.params = {} request.user = DBSession.query(User).filter_by(username="__test_user1").one() - themes, errors = entry._themes(request.user.role.id) + themes, errors = entry._themes('', request.user.role.id) self.assertEqual(errors, { "The layer '__test_layer_in_group' (__test_layer_in_group) is not defined in WMS capabilities from '__test_ogc_server'", }) @@ -478,7 +478,7 @@ def test_notmapfile(self): from c2cgeoportal_geoportal.lib import caching caching.invalidate_region() - _, errors = entry._themes(None, "desktop") + _, errors = entry._themes('', None, "desktop") self.assertEqual({e[:43] for e in errors}, { "The layer '__test_public_layer' (__test_pub", "The layer '__test_layer_in_group' (__test_l", @@ -541,7 +541,7 @@ def test_theme_geoserver(self): entry = Entry(request) # unautenticated v1 - themes, errors = entry._themes(None, "desktop") + themes, errors = entry._themes('', None, "desktop") self.assertEqual(errors, set()) self.assertEqual(len(themes), 1) layers = {l["name"] for l in themes[0]["children"][0]["children"]} @@ -554,7 +554,7 @@ def test_theme_geoserver(self): # authenticated v1 request.params = {} request.user = DBSession.query(User).filter_by(username="__test_user1").one() - themes, errors = entry._themes(request.user.role.id) + themes, errors = entry._themes('', request.user.role.id) self.assertEqual(errors, set()) self.assertEqual(len(themes), 1) layers = {l["name"] for l in themes[0]["children"][0]["children"]} @@ -935,7 +935,7 @@ def test_layer(self): entry = Entry(request) self.assertEqual(entry._group( - "", LayerGroup(), layers=[]), (None, set()) + '', "", LayerGroup(), layers=[]), (None, set()) ) layer = LayerV1() @@ -1234,7 +1234,7 @@ def test_layer(self): group.children = [layer_t1, layer_t2] time = TimeInformation() entry._group( - "", group, [layer_t1.name, layer_t2.name], + '', "", group, [layer_t1.name, layer_t2.name], time=time, mixed=False, depth=2 ) self.assertEqual(time.to_dict(), { @@ -1355,7 +1355,7 @@ def test_layer(self): group2.children = [layer] self.assertEqual(entry._group( - "", group1, [layer.name], + '', "", group1, [layer.name], ), ({ "id": 11, "isExpanded": False, @@ -1409,7 +1409,7 @@ def test_internalwms(self): group2 = LayerGroup() group2.is_internal_wms = False group1.children = [group2] - _, errors = entry._group("", group1, [], catalogue=False) + _, errors = entry._group('', "", group1, [], catalogue=False) self._assert_has_error(errors, "Group '' cannot be in group '' (internal/external mix).") group1 = LayerGroup() @@ -1417,7 +1417,7 @@ def test_internalwms(self): group2 = LayerGroup() group2.is_internal_wms = True group1.children = [group2] - _, errors = entry._group("", group1, [], catalogue=False) + _, errors = entry._group('', "", group1, [], catalogue=False) self._assert_has_error(errors, "Group '' cannot be in group '' (internal/external mix).") group = LayerGroup() @@ -1425,7 +1425,7 @@ def test_internalwms(self): layer = LayerV1() layer.layer_type = "internal WMS" group.children = [layer] - _, errors = entry._group("", group, [layer.name], catalogue=False) + _, errors = entry._group('', "", group, [layer.name], catalogue=False) self.assertEqual(errors, { "The layer '' () is not defined in WMS capabilities from '__test_ogc_server'", }) @@ -1435,7 +1435,7 @@ def test_internalwms(self): layer = LayerV1() layer.layer_type = "external WMS" group.children = [layer] - _, errors = entry._group("", group, [layer.name], catalogue=False) + _, errors = entry._group('', "", group, [layer.name], catalogue=False) self._assert_has_error(errors, "Layer '' cannot be in the group '' (internal/external mix).") group = LayerGroup() @@ -1443,7 +1443,7 @@ def test_internalwms(self): layer = LayerV1() layer.layer_type = "WMTS" group.children = [layer] - _, errors = entry._group("", group, [layer.name], catalogue=False) + _, errors = entry._group('', "", group, [layer.name], catalogue=False) self._assert_has_error(errors, "Layer '' cannot be in the group '' (internal/external mix).") group = LayerGroup() @@ -1451,7 +1451,7 @@ def test_internalwms(self): layer = LayerV1() layer.layer_type = "no 2D" group.children = [layer] - _, errors = entry._group("", group, [layer.name], catalogue=False) + _, errors = entry._group('', "", group, [layer.name], catalogue=False) self._assert_has_error(errors, "Layer '' cannot be in the group '' (internal/external mix).") group = LayerGroup() @@ -1459,7 +1459,7 @@ def test_internalwms(self): layer = LayerV1() layer.layer_type = "internal WMS" group.children = [layer] - _, errors = entry._group("", group, [layer.name], catalogue=False) + _, errors = entry._group('', "", group, [layer.name], catalogue=False) self._assert_has_error(errors, "Layer '' cannot be in the group '' (internal/external mix).") group = LayerGroup() @@ -1467,7 +1467,7 @@ def test_internalwms(self): layer = LayerV1() layer.layer_type = "external WMS" group.children = [layer] - _, errors = entry._group("", group, [layer.name], catalogue=False, min_levels=0) + _, errors = entry._group('', "", group, [layer.name], catalogue=False, min_levels=0) self.assertEqual(errors, set()) group = LayerGroup() @@ -1475,7 +1475,7 @@ def test_internalwms(self): layer = LayerV1() layer.layer_type = "WMTS" group.children = [layer] - _, errors = entry._group("", group, [layer.name], catalogue=False, min_levels=0) + _, errors = entry._group('', "", group, [layer.name], catalogue=False, min_levels=0) self.assertEqual(errors, set()) group = LayerGroup() @@ -1483,7 +1483,7 @@ def test_internalwms(self): layer = LayerV1() layer.layer_type = "no 2D" group.children = [layer] - _, errors = entry._group("", group, [layer.name], catalogue=False, min_levels=0) + _, errors = entry._group('', "", group, [layer.name], catalogue=False, min_levels=0) self.assertEqual(errors, set()) def test_loginchange_no_params(self): diff --git a/geoportal/tests/functional/test_looptheme.py b/geoportal/tests/functional/test_looptheme.py index 4543ebb2b1..c080d7efd8 100644 --- a/geoportal/tests/functional/test_looptheme.py +++ b/geoportal/tests/functional/test_looptheme.py @@ -97,5 +97,5 @@ def test_theme(self): request.client_addr = None request.user = None entry = Entry(request) - _, errors = entry._themes(None, "desktop2", True, 2) + _, errors = entry._themes('', None, "desktop2", True, 2) self.assertEqual(len([e for e in errors if e == "Too many recursions with group '__test_layer_group'"]), 1) From 85e3c2af435e16dd759c9c776400553a12a51655 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Thu, 16 May 2019 16:42:53 +0200 Subject: [PATCH 6/6] Replace the entrypoint also in the css and html files --- docker/build/bin/eval-templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/build/bin/eval-templates b/docker/build/bin/eval-templates index e6bc4504c1..c1dad8ca9d 100755 --- a/docker/build/bin/eval-templates +++ b/docker/build/bin/eval-templates @@ -6,7 +6,7 @@ do envsubst < ${file} > ${file%.tmpl} done -find /app/*_geoportal/static-ngeo/build/ -name '*.js' -print | while read file +find /app/*_geoportal/static-ngeo/build/ \( -name '*.js' -or -name '*.css' -or -name '*.html' \) -print | while read file do echo "Evaluate: ${file}" sed --in-place --expression="s#__ENTRY_POINT__#${VISIBLE_ENTRY_POINT}#g" "${file}"