refactor: interface query on m-m joins and select specific columns (#…

…1398)

* refactor: interface query on m-m joins and select specific columns

* mssql hack

* add tests with many children

* a bit of refactor on the refactor

* remove prints

* fix and cleanup

* fix and cleanup

* type annotations and code quality

* type annotations and code quality

* type annotations and code quality

* lint

* more type annotations

* only use from_self if necessary

* fix column model query loading for non inner queries

* Drying and fix get with improved performance

* MVC starts using select columns, maybe revert

* revert select_columns from mvc

* feat: list_select_columns to enable wider config

* one or none instead of first

* doc show_select_columns and list_select_columns

* fix, one or none instead of first

* fix, docs

docs/rest_api.rst

@@ -138,7 +138,7 @@ so data can be translated back and forth without loss or guesswork::
             if 'name' in kwargs['rison']:
                 return self.response(
                     200,
-                    message="Hello {}".format(kwargs['rison']['name'])
+                    message=f"Hello {kwargs['rison']['name']}"
                 )
             return self.response_400(message="Please send your name")
 
@@ -238,7 +238,7 @@ validate your Rison arguments, this way you can implement a very strict API easi
     def greeting4(self, **kwargs):
         return self.response(
             200,
-            message="Hello {}".format(kwargs['rison']['name'])
+            message=f"Hello {kwargs['rison']['name']}"
         )
 
 Finally to properly handle all possible exceptions use the ``safe`` decorator,
@@ -396,7 +396,7 @@ easily reference them::
             """
             return self.response(
                 200,
-                message="Hello {}".format(kwargs['rison']['name'])
+                message=f"Hello {kwargs['rison']['name']}"
             )
 
 
@@ -1015,6 +1015,33 @@ the ``show_columns`` property. This takes precedence from the *Rison* arguments:
         datamodel = SQLAInterface(Contact)
         show_columns = ['name']
 
+By default FAB will issue a query containing the exact fields for `show_columns`, but these are also associated with
+the response object. Sometimes it's useful to distinguish between the query select columns and the response itself.
+Imagine the case you want to use a `@property` to further transform the output, and that transformation implies
+two model fields (concat or sum for example)::
+
+    class ContactModelApi(ModelRestApi):
+        resource_name = 'contact'
+        datamodel = SQLAInterface(Contact)
+        show_columns = ['name', 'age']
+        show_select_columns = ['name', 'birthday']
+
+
+The Model::
+
+    class Contact(Model):
+        id = Column(Integer, primary_key=True)
+        name = Column(String(150), unique=True, nullable=False)
+        ...
+        birthday = Column(Date, nullable=True)
+        ...
+
+        @property
+        def age(self):
+            return date.today().year - self.birthday.year
+
+Note: The same logic is applied on `list_select_columns`
+
 We can add fields that are python functions also, for this on the SQLAlchemy definition,
 let's add a new function::
 
@@ -1034,7 +1061,7 @@ let's add a new function::
             return self.name
 
         def some_function(self):
-            return "Hello {}".format(self.name)
+            return f"Hello {self.name}"
 
 And then on the REST API::
 

flask_appbuilder/api/__init__.py

@@ -3,15 +3,15 @@
 import logging
 import re
 import traceback
-from typing import Dict, Optional
+from typing import Callable, Dict, List, Optional, Set
 import urllib.parse
 
 from apispec import APISpec, yaml_utils
 from apispec.exceptions import DuplicateComponentNameError
 from flask import Blueprint, current_app, jsonify, make_response, request, Response
 from flask_babel import lazy_gettext as _
 import jsonschema
-from marshmallow import ValidationError
+from marshmallow import Schema, ValidationError
 from marshmallow_sqlalchemy.fields import Related, RelatedList
 import prison
 from sqlalchemy.exc import IntegrityError
@@ -214,39 +214,39 @@ class BaseApi(object):
 
     appbuilder = None
     blueprint = None
-    endpoint = None
+    endpoint: Optional[str] = None
 
-    version = "v1"
+    version: Optional[str] = "v1"
     """
         Define the Api version for this resource/class
     """
-    route_base = None
+    route_base: Optional[str] = None
     """
         Define the route base where all methods will suffix from
     """
-    resource_name = None
+    resource_name: Optional[str] = None
     """
         Defines a custom resource name, overrides the inferred from Class name
         makes no sense to use it with route base
     """
-    base_permissions = None
+    base_permissions: Optional[List[str]] = None
     """
         A list of allowed base permissions::
 
             class ExampleApi(BaseApi):
                 base_permissions = ['can_get']
 
     """
-    class_permission_name = None
+    class_permission_name: Optional[str] = None
     """
         Override class permission name default fallback to self.__class__.__name__
     """
-    previous_class_permission_name = None
+    previous_class_permission_name: Optional[str] = None
     """
         If set security converge will replace all permissions tuples
         with this name by the class_permission_name or self.__class__.__name__
     """
-    method_permission_name = None
+    method_permission_name: Optional[Dict[str, str]] = None
     """
         Override method permission names, example::
 
@@ -258,7 +258,7 @@ class ExampleApi(BaseApi):
                 'delete': 'write'
             }
     """
-    previous_method_permission_name = None
+    previous_method_permission_name: Optional[Dict[str, str]] = None
     """
         Use same structure as method_permission_name. If set security converge
         will replace all method permissions by the new ones
@@ -272,7 +272,7 @@ class ExampleApi(BaseApi):
     """
         If using flask-wtf CSRFProtect exempt the API from check
     """
-    apispec_parameter_schemas = None
+    apispec_parameter_schemas: Optional[Dict[str, Dict]] = None
     """
         Set your custom Rison parameter schemas here so that
         they get registered on the OpenApi spec::
@@ -377,7 +377,7 @@ class ContactModelView(ModelRestApi):
 
         The previous examples will only register the `put`, `post` and `delete` routes
     """
-    include_route_methods = None
+    include_route_methods: Set[str] = None
     """
         If defined will assume a white list setup, where all endpoints are excluded
         except those define on this attribute
@@ -412,7 +412,7 @@ class GreetingApi(BaseApi):
         Use this attribute to override the tag name
     """
 
-    def __init__(self):
+    def __init__(self) -> None:
         """
             Initialization of base permissions
             based on exposed methods and actions
@@ -855,72 +855,83 @@ class ModelRestApi(BaseModelApi):
         List Title, if not configured the default is
         'List ' with pretty model name
     """
-    show_title = ""
+    show_title: Optional[str] = ""
     """
         Show Title , if not configured the default is
         'Show ' with pretty model name
     """
-    add_title = ""
+    add_title: Optional[str] = ""
     """
         Add Title , if not configured the default is
         'Add ' with pretty model name
     """
-    edit_title = ""
+    edit_title: Optional[str] = ""
     """
         Edit Title , if not configured the default is
         'Edit ' with pretty model name
     """
-
-    list_columns = None
+    list_select_columns: Optional[List[str]] = None
+    """
+        A List of column names that will be included on the SQL select.
+        This is useful for including all necessary columns that are referenced
+        by properties listed on `list_columns` without generating N+1 queries.
+    """
+    list_columns: Optional[List[str]] = None
     """
         A list of columns (or model's methods) to be displayed on the list view.
         Use it to control the order of the display
     """
-    show_columns = None
+    show_select_columns: Optional[List[str]] = None
+    """
+        A List of column names that will be included on the SQL select.
+        This is useful for including all necessary columns that are referenced
+        by properties listed on `show_columns` without generating N+1 queries.
+    """
+    show_columns: Optional[List[str]] = None
     """
         A list of columns (or model's methods) for the get item endpoint.
         Use it to control the order of the results
     """
-    add_columns = None
+    add_columns: Optional[List[str]] = None
     """
         A list of columns (or model's methods) to be allowed to post
     """
-    edit_columns = None
+    edit_columns: Optional[List[str]] = None
     """
         A list of columns (or model's methods) to be allowed to update
     """
-    list_exclude_columns = None
+    list_exclude_columns: Optional[List[str]] = None
     """
         A list of columns to exclude from the get list endpoint.
         By default all columns are included.
     """
-    show_exclude_columns = None
+    show_exclude_columns: Optional[List[str]] = None
     """
         A list of columns to exclude from the get item endpoint.
         By default all columns are included.
     """
-    add_exclude_columns = None
+    add_exclude_columns: Optional[List[str]] = None
     """
         A list of columns to exclude from the add endpoint.
         By default all columns are included.
     """
-    edit_exclude_columns = None
+    edit_exclude_columns: Optional[List[str]] = None
     """
         A list of columns to exclude from the edit endpoint.
         By default all columns are included.
     """
-    order_columns = None
+    order_columns: Optional[List[str]] = None
     """ Allowed order columns """
     page_size = 20
     """
         Use this property to change default page size
     """
-    max_page_size = None
+    max_page_size: Optional[int] = None
     """
         class override for the FAB_API_MAX_SIZE, use special -1 to allow for any page
         size
     """
-    description_columns = None
+    description_columns: Optional[Dict[str, str]] = None
     """
         Dictionary with column descriptions that will be shown on the forms::
 
@@ -930,8 +941,8 @@ class MyView(ModelView):
                 description_columns = {'name':'your models name column',
                                         'address':'the address column'}
     """
-    validators_columns = None
-    """ Dictionary to add your own validators for forms """
+    validators_columns: Optional[Dict[str, Callable]] = None
+    """ Dictionary to add your own marshmallow validators """
 
     add_query_rel_fields = None
     """
@@ -973,22 +984,22 @@ class ContactModelView(ModelRestApi):
                     'gender': ('name', 'asc')
                 }
     """
-    list_model_schema = None
+    list_model_schema: Optional[Schema] = None
     """
         Override to provide your own marshmallow Schema
         for JSON to SQLA dumps
     """
-    add_model_schema = None
+    add_model_schema: Optional[Schema] = None
     """
         Override to provide your own marshmallow Schema
         for JSON to SQLA dumps
     """
-    edit_model_schema = None
+    edit_model_schema: Optional[Schema] = None
     """
         Override to provide your own marshmallow Schema
         for JSON to SQLA dumps
     """
-    show_model_schema = None
+    show_model_schema: Optional[Schema] = None
     """
         Override to provide your own marshmallow Schema
         for JSON to SQLA dumps
@@ -1069,7 +1080,7 @@ def _init_titles(self):
             self.show_title = "Show " + self._prettify_name(class_name)
         self.title = self.list_title
 
-    def _init_properties(self):
+    def _init_properties(self) -> None:
         """
             Init Properties
         """
@@ -1091,6 +1102,7 @@ def _init_properties(self):
                 for x in self.datamodel.get_user_columns_list()
                 if x not in self.list_exclude_columns
             ]
+        self.list_select_columns = self.list_select_columns or self.list_columns
 
         self.order_columns = (
             self.order_columns
@@ -1101,6 +1113,8 @@ def _init_properties(self):
             self.show_columns = [
                 x for x in list_cols if x not in self.show_exclude_columns
             ]
+        self.show_select_columns = self.show_select_columns or self.show_columns
+
         if not self.add_columns:
             self.add_columns = [
                 x for x in list_cols if x not in self.add_exclude_columns
@@ -1302,7 +1316,7 @@ def get_headless(self, pk, **kwargs) -> Response:
         :param kwargs: Query string parameter arguments
         :return: HTTP Response
         """
-        item = self.datamodel.get(pk, self._base_filters)
+        item = self.datamodel.get(pk, self._base_filters, self.show_select_columns)
         if not item:
             return self.response_404()
 
@@ -1417,13 +1431,15 @@ def get_list_headless(self, **kwargs) -> Response:
         # handle select columns
         select_cols = _args.get(API_SELECT_COLUMNS_RIS_KEY, [])
         _pruned_select_cols = [col for col in select_cols if col in self.list_columns]
+        # map decorated metadata
         self.set_response_key_mappings(
             _response,
             self.get_list,
             _args,
             **{API_SELECT_COLUMNS_RIS_KEY: _pruned_select_cols},
         )
-
+        # Create a response schema with the computed response columns,
+        # defined or requested
         if _pruned_select_cols:
             _list_model_schema = self.model2schemaconverter.convert(_pruned_select_cols)
         else:
@@ -1441,14 +1457,13 @@ def get_list_headless(self, **kwargs) -> Response:
         # handle pagination
         page_index, page_size = self._handle_page_args(_args)
         # Make the query
-        query_select_columns = _pruned_select_cols or self.list_columns
         count, lst = self.datamodel.query(
             joined_filters,
             order_column,
             order_direction,
             page=page_index,
             page_size=page_size,
-            select_columns=query_select_columns,
+            select_columns=self.list_select_columns,
         )
         pks = self.datamodel.get_keys(lst)
         _response[API_RESULT_RES_KEY] = _list_model_schema.dump(lst, many=True)

flask_appbuilder/exceptions.py

@@ -20,3 +20,9 @@ class InvalidOrderByColumnFABException(FABException):
     """Invalid order by column"""
 
     pass
+
+
+class InterfaceQueryWithoutSession(FABException):
+    """You need to setup a session on the interface to perform queries"""
+
+    pass