[Network] Add new commands to manage flow-log and deprecate old configure command

src/azure-cli/azure/cli/command_modules/network/_client_factory.py

@@ -32,6 +32,10 @@ def cf_connection_monitor(cli_ctx, _):
     return network_client_factory(cli_ctx).connection_monitors
 
 
+def cf_flow_logs(cli_ctx, _):
+    return network_client_factory(cli_ctx).flow_logs
+
+
 def cf_ddos_protection_plans(cli_ctx, _):
     return network_client_factory(cli_ctx).ddos_protection_plans
 

src/azure-cli/azure/cli/command_modules/network/_help.py

@@ -5329,12 +5329,49 @@
     text: az network watcher flow-log configure -g MyResourceGroup --enabled false --nsg MyNsg
 """
 
+helps['network watcher flow-log create'] = """
+type: command
+short-summary: Create a flow log on a network security group.
+"""
+
+helps['network watcher flow-log list'] = """
+type: command
+short-summary: List all flow log resources for the specified Network Watcher
+"""
+
+helps['network watcher flow-log delete'] = """
+type: command
+short-summary: Delete the specified flow log resource.
+"""
+
 helps['network watcher flow-log show'] = """
 type: command
 short-summary: Get the flow log configuration of a network security group.
 examples:
-  - name: Show NSG flow logs.
+  - name: Show NSG flow logs. (Deprecated)
     text: az network watcher flow-log show -g MyResourceGroup --nsg MyNsg
+  - name: Show NSG flow logs with Azure Resource Management formatted.
+    text: az network watcher flow-log show --watcher MyNetworkWatcher --name true
+"""
+
+helps['network watcher flow-log update'] = """
+type: command
+short-summary: Update the flow log configuration of a network security group
+examples:
+  - name: Update storage account with name to let resource group identify the storage account and network watcher
+    text: >
+      az network watcher flow-log update
+      --resource-group MyResourceGroup
+      --watcher MyNetworkWatcher
+      --name MyFlowLog
+      --storage-account mystorageaccountname
+  - name: Update storage account with ID to let location identify the network watcher
+    text: >
+      az network watcher flow-log update
+      --resource-group MyResourceGroup
+      --watcher MyNetworkWatcher
+      --name MyFlowLog
+      --storage-account mystorageaccountid
 """
 
 helps['network watcher list'] = """

src/azure-cli/azure/cli/command_modules/network/_params.py

@@ -1348,8 +1348,26 @@ def load_arguments(self, _):
         c.argument('filters', type=get_json_object)
 
     with self.argument_context('network watcher flow-log') as c:
+        c.argument('network_watcher_name', options_list='--watcher')
+        c.argument('location', get_location_type(self.cli_ctx))
+        c.argument('flow_log_name', name_arg_type, help='The name of the flow logger', min_api='2019-11-01')
         c.argument('nsg', help='Name or ID of the network security group.')
-        c.argument('enabled', arg_type=get_three_state_flag())
+        c.argument('enabled', arg_type=get_three_state_flag(), help='Enable logging', default='true')
+        c.argument('retention', type=int, help='Number of days to retain logs')
+        c.argument('storage_account', help='Name or ID of the storage account in which to save the flow logs')
+
+    # temporary solution for compatible with old show command's parameter
+    # after old show command's parameter is deprecated and removed,
+    # this argument group "network watcher flow-log show" should be removed
+    with self.argument_context('network watcher flow-log show') as c:
+        c.argument('nsg',
+                   deprecate_info=c.deprecate(redirect='--watcher and --name combination', hide=False),
+                   help='Name or ID of the network security group.')
+
+    with self.argument_context('network watcher flow-log show',
+                               arg_group='Result with Azure Management Resource (ARM) Formatted') as c:
+        c.argument('network_watcher_name', options_list='--watcher')
+        c.argument('flow_log_name', name_arg_type, help='The name of the flow logger', min_api='2019-11-01')
 
     with self.argument_context('network watcher flow-log', arg_group='Format', min_api='2018-10-01') as c:
         c.argument('log_format', options_list='--format', help='File type of the flow log.', arg_type=get_enum_type(FlowLogFormatType))

src/azure-cli/azure/cli/command_modules/network/_validators.py

@@ -1379,7 +1379,7 @@ def process_nw_test_connectivity_namespace(cmd, namespace):
         namespace.headers = headers
 
 
-def process_nw_flow_log_set_namespace(cmd, namespace):
+def process_nw_flow_log_set_namespace(cmd, namespace, remove_location=True):
     from msrestazure.tools import is_valid_resource_id, resource_id
     if namespace.storage_account and not is_valid_resource_id(namespace.storage_account):
         namespace.storage_account = resource_id(
@@ -1396,24 +1396,55 @@ def process_nw_flow_log_set_namespace(cmd, namespace):
             type='workspaces',
             name=namespace.traffic_analytics_workspace)
 
-    process_nw_flow_log_show_namespace(cmd, namespace)
+    process_nw_flow_log_show_namespace(cmd, namespace, remove_location)
 
 
-def process_nw_flow_log_show_namespace(cmd, namespace):
+def process_nw_flow_log_create_namespace(cmd, namespace):
+    """
+    Flow Log is the sub-resource of Network Watcher, they must be in the same region and subscription.
+    If we can get location from NSG, we will use it to identify the Flow Log and Network Watcher.
+    If user provide --location, we respect it.
+    If user don't provide --location, we try to retrieve region info from resource group.
+    Otherwise, we raise CLIError
+    """
+    process_nw_flow_log_set_namespace(cmd, namespace, remove_location=False)    # keep location
+
+    # set location for the sake of watcher if it's unset
+    if namespace.location is None and namespace.resource_group_name is not None:
+        get_default_location_from_resource_group(cmd, namespace)
+        get_network_watcher_from_location(remove=False)(cmd, namespace)
+
+    if namespace.location is None:
+        raise CLIError('usage error: require --location/--resource-group to help identify Network Watcher. '
+                       'Network Watcher and Flow Log must be in the same region and subscription. '
+                       'While the other resources are not required. '
+                       'If we can get location from NSG, we will use it to identify the Flow Log and Network Watcher. '
+                       'If location is provided, we will use it to identify the Flow Log and Network Watcher. '
+                       'If location is missing, we will try to retrieve location from resource group. ')
+
+    validate_tags(namespace)
+
+
+def process_nw_flow_log_show_namespace(cmd, namespace, remove_location=True):
     from msrestazure.tools import is_valid_resource_id, resource_id
     from azure.cli.core.commands.arm import get_arm_resource_by_id
 
-    if not is_valid_resource_id(namespace.nsg):
-        namespace.nsg = resource_id(
-            subscription=get_subscription_id(cmd.cli_ctx),
-            resource_group=namespace.resource_group_name,
-            namespace='Microsoft.Network',
-            type='networkSecurityGroups',
-            name=namespace.nsg)
+    if hasattr(namespace, 'nsg') and namespace.nsg is not None:
+        if not is_valid_resource_id(namespace.nsg):
+            namespace.nsg = resource_id(
+                subscription=get_subscription_id(cmd.cli_ctx),
+                resource_group=namespace.resource_group_name,
+                namespace='Microsoft.Network',
+                type='networkSecurityGroups',
+                name=namespace.nsg)
 
-    nsg = get_arm_resource_by_id(cmd.cli_ctx, namespace.nsg)
-    namespace.location = nsg.location  # pylint: disable=no-member
-    get_network_watcher_from_location(remove=True)(cmd, namespace)
+        nsg = get_arm_resource_by_id(cmd.cli_ctx, namespace.nsg)
+        namespace.location = nsg.location  # pylint: disable=no-member
+        get_network_watcher_from_location(remove=remove_location)(cmd, namespace)
+    elif namespace.flow_log_name is not None and namespace.network_watcher_name is not None:
+        pass
+    else:
+        raise CLIError('usage error: --nsg NSG | --watcher NETWORK_WATCHER_NAME --name FLOW_LOW_NAME')
 
 
 def process_nw_topology_namespace(cmd, namespace):

src/azure-cli/azure/cli/command_modules/network/commands.py

@@ -27,7 +27,7 @@
     cf_express_route_circuit_connections, cf_express_route_gateways, cf_express_route_connections,
     cf_express_route_ports, cf_express_route_port_locations, cf_express_route_links, cf_app_gateway_waf_policy,
     cf_service_tags, cf_private_link_services, cf_private_endpoint_types, cf_peer_express_route_circuit_connections,
-    cf_virtual_router, cf_virtual_router_peering, cf_service_aliases, cf_bastion_hosts)
+    cf_virtual_router, cf_virtual_router_peering, cf_service_aliases, cf_bastion_hosts, cf_flow_logs)
 from azure.cli.command_modules.network._util import (
     list_network_resource_property, get_network_resource_property_entry, delete_network_resource_property_entry)
 from azure.cli.command_modules.network._format import (
@@ -50,7 +50,7 @@
     process_nw_cm_create_namespace,
     process_nw_cm_v2_endpoint_namespace, process_nw_cm_v2_test_configuration_namespace,
     process_nw_cm_v2_test_group, process_nw_cm_v2_output_namespace,
-    process_nw_flow_log_set_namespace, process_nw_flow_log_show_namespace,
+    process_nw_flow_log_set_namespace, process_nw_flow_log_create_namespace, process_nw_flow_log_show_namespace,
     process_nw_packet_capture_create_namespace, process_nw_test_connectivity_namespace, process_nw_topology_namespace,
     process_nw_troubleshooting_start_namespace, process_nw_troubleshooting_show_namespace,
     process_public_ip_create_namespace, process_tm_endpoint_create_namespace,
@@ -300,6 +300,16 @@ def load_command_table(self, _):
         client_factory=cf_connection_monitor
     )
 
+    network_watcher_flow_log_sdk = CliCommandType(
+        operations_tmpl='azure.mgmt.network.operations#FlowLogsOperations.{}',
+        client_factory=cf_flow_logs,
+    )
+
+    network_watcher_flow_log_update_sdk = CliCommandType(
+        operations_tmpl='azure.cli.command_modules.network.custom#{}',
+        client_factory=cf_flow_logs,
+    )
+
     network_watcher_pc_sdk = CliCommandType(
         operations_tmpl='azure.mgmt.network.operations#PacketCapturesOperations.{}',
         client_factory=cf_packet_capture
@@ -910,9 +920,33 @@ def _make_singular(value):
         g.command('list', 'list')
 
     with self.command_group('network watcher flow-log', client_factory=cf_network_watcher, min_api='2016-09-01') as g:
-        g.custom_command('configure', 'set_nsg_flow_logging', validator=process_nw_flow_log_set_namespace)
+        g.custom_command('configure',
+                         'set_nsg_flow_logging',
+                         validator=process_nw_flow_log_set_namespace,
+                         deprecate_info=self.deprecate(redirect='network watcher flow-log create', hide=False))
         g.custom_show_command('show', 'show_nsg_flow_logging', validator=process_nw_flow_log_show_namespace)
 
+    with self.command_group('network watcher flow-log', network_watcher_flow_log_sdk, min_api='2019-11-01') as g:
+        g.custom_command('create',
+                         'create_nw_flow_log',
+                         client_factory=cf_flow_logs,
+                         validator=process_nw_flow_log_create_namespace)
+        # show command implementation is substituted by show_nsg_flow_logging()
+        # after old show command's parameter is deprecated and removed, should refactor this show command implementation
+        # g.custom_show_command('show',
+        #                       'show_nw_flow_log',
+        #                       client_factory=cf_flow_logs,
+        #                       validator=process_nw_flow_log_show_namespace)
+        g.command('list', 'list')
+        g.command('delete', 'delete')
+        g.generic_update_command('update',
+                                 getter_name='update_nw_flow_log_getter',
+                                 getter_type=network_watcher_flow_log_update_sdk,
+                                 setter_name='update_nw_flow_log_setter',
+                                 setter_type=network_watcher_flow_log_update_sdk,
+                                 custom_func_name='update_nw_flow_log',
+                                 validator=process_nw_flow_log_create_namespace)
+
     with self.command_group('network watcher troubleshooting', client_factory=cf_network_watcher, min_api='2016-09-01') as g:
         g.custom_command('start', 'start_nw_troubleshooting', supports_no_wait=True, validator=process_nw_troubleshooting_start_namespace)
         g.custom_show_command('show', 'show_nw_troubleshooting_result', validator=process_nw_troubleshooting_show_namespace)

src/azure-cli/azure/cli/command_modules/network/custom.py

@@ -4381,10 +4381,131 @@ def set_nsg_flow_logging(cmd, client, watcher_rg, watcher_name, nsg, storage_acc
     return client.set_flow_log_configuration(watcher_rg, watcher_name, config)
 
 
-def show_nsg_flow_logging(client, watcher_rg, watcher_name, nsg, resource_group_name=None):
+# why need 2 different names (watcher_name and network_watcher_name) for watcher?
+# It's temporary solution for compatible with old show command's parameter.
+# After old show command's parameter is deprecated, those parameters for should be removed.
+def show_nsg_flow_logging(cmd, client, watcher_rg, watcher_name, resource_group_name=None, nsg=None,
+                          network_watcher_name=None, flow_log_name=None):
+    # new approach to show flow log
+    if all([resource_group_name, network_watcher_name, flow_log_name]):
+        from ._client_factory import cf_flow_logs
+        client = cf_flow_logs(cmd.cli_ctx, None)
+        return client.get(resource_group_name, network_watcher_name, flow_log_name)
+
+    # deprecated approach to show flow log
     return client.get_flow_log_status(watcher_rg, watcher_name, nsg)
 
 
+def create_nw_flow_log(cmd,
+                       client,
+                       watcher_rg,
+                       watcher_name,
+                       flow_log_name,
+                       nsg,
+                       storage_account=None,
+                       resource_group_name=None,
+                       location=None,
+                       enabled=None,
+                       retention=0,
+                       log_format=None,
+                       log_version=None,
+                       traffic_analytics_workspace=None,
+                       traffic_analytics_interval=60,
+                       traffic_analytics_enabled=None,
+                       tags=None):
+    FlowLog = cmd.get_models('FlowLog')
+    flow_log = FlowLog(location=location,
+                       target_resource_id=nsg,
+                       storage_id=storage_account,
+                       enabled=enabled,
+                       tags=tags)
+
+    if retention > 0:
+        RetentionPolicyParameters = cmd.get_models('RetentionPolicyParameters')
+        retention_policy = RetentionPolicyParameters(days=retention, enabled=(retention > 0))
+        flow_log.retention_policy = retention_policy
+
+    if log_format is not None or log_version is not None:
+        FlowLogFormatParameters = cmd.get_models('FlowLogFormatParameters')
+        format_config = FlowLogFormatParameters(type=log_format, version=log_version)
+        flow_log.format = format_config
+
+    if traffic_analytics_workspace is not None:
+        TrafficAnalyticsProperties, TrafficAnalyticsConfigurationProperties = \
+            cmd.get_models('TrafficAnalyticsProperties', 'TrafficAnalyticsConfigurationProperties')
+
+        from azure.cli.core.commands.arm import get_arm_resource_by_id
+        workspace = get_arm_resource_by_id(cmd.cli_ctx, traffic_analytics_workspace)
+        if not workspace:
+            raise CLIError('Name or ID of workspace is invalid')
+
+        traffic_analytics_config = TrafficAnalyticsConfigurationProperties(
+            enabled=traffic_analytics_enabled,
+            workspace_id=workspace.properties['customerId'],
+            workspace_region=workspace.location,
+            workspace_resource_id=workspace.id,
+            traffic_analytics_interval=traffic_analytics_interval
+        )
+        traffic_analytics = TrafficAnalyticsProperties(
+            network_watcher_flow_analytics_configuration=traffic_analytics_config
+        )
+
+        flow_log.flow_analytics_configuration = traffic_analytics
+
+    return client.create_or_update(watcher_rg, watcher_name, flow_log_name, flow_log)
+
+
+def update_nw_flow_log_getter(client, watcher_rg, network_watcher_name, flow_log_name):
+    return client.get(watcher_rg, network_watcher_name, flow_log_name)
+
+
+def update_nw_flow_log_setter(client, watcher_rg, network_watcher_name, flow_log_name, parameters):
+    return client.create_or_update(watcher_rg, network_watcher_name, flow_log_name, parameters)
+
+
+def update_nw_flow_log(cmd,
+                       instance,
+                       location=None,   # dummy parameter to let it appear in command
+                       resource_group_name=None,    # dummy parameter to let it appear in command
+                       enabled=None,
+                       storage_account=None,
+                       retention=0,
+                       log_format=None,
+                       log_version=None,
+                       traffic_analytics_workspace=None,
+                       traffic_analytics_interval=60,
+                       traffic_analytics_enabled=None,
+                       tags=None):
+    with cmd.update_context(instance) as c:
+        c.set_param('enabled', enabled)
+        c.set_param('tags', tags)
+        c.set_param('storage_id', storage_account)
+
+    with cmd.update_context(instance.retention_policy) as c:
+        c.set_param('days', retention)
+        c.set_param('enabled', retention > 0)
+
+    with cmd.update_context(instance.format) as c:
+        c.set_param('type', log_format)
+        c.set_param('version', log_version)
+
+    if traffic_analytics_workspace is not None:
+        from azure.cli.core.commands.arm import get_arm_resource_by_id
+        workspace = get_arm_resource_by_id(cmd.cli_ctx, traffic_analytics_workspace)
+        if not workspace:
+            raise CLIError('Name or ID of workspace is invalid')
+
+        with cmd.update_context(
+                instance.flow_analytics_configuration.network_watcher_flow_analytics_configuration) as c:
+            c.set_param('enabled', traffic_analytics_enabled)
+            c.set_param('workspace_id', workspace.properties['customerId'])
+            c.set_param('workspace_region', workspace.location)
+            c.set_param('workspace_resource_id', workspace.id)
+            c.set_param('traffic_analytics_interval', traffic_analytics_interval)
+
+    return instance
+
+
 def start_nw_troubleshooting(cmd, client, watcher_name, watcher_rg, resource, storage_account,
                              storage_path, resource_type=None, resource_group_name=None,
                              no_wait=False):