[Network] Add new commands to manage flow-log and deprecate old configure command

src/azure-cli/azure/cli/command_modules/network/_client_factory.py

@@ -32,6 +32,10 @@ def cf_connection_monitor(cli_ctx, _):
     return network_client_factory(cli_ctx).connection_monitors
 
 
+def cf_flow_logs(cli_ctx, _):
+    return network_client_factory(cli_ctx).flow_logs
+
+
 def cf_ddos_protection_plans(cli_ctx, _):
     return network_client_factory(cli_ctx).ddos_protection_plans
 

src/azure-cli/azure/cli/command_modules/network/_help.py

@@ -5329,12 +5329,79 @@
     text: az network watcher flow-log configure -g MyResourceGroup --enabled false --nsg MyNsg
 """
 
+helps['network watcher flow-log create'] = """
+type: command
+short-summary: Create a flow log on a network security group.
+examples:
+  - name: Create a flow log with Network Security Group name
+    text: >
+      az network watcher flow-log create
+      --location westus
+      --resource-group MyResourceGroup
+      --name MyFlowLog
+      --nsg MyNetworkSecurityGroupName
+      --storage-account account
+  - name: Create a flow log with Network Security Group ID (could be in other resource group)
+    text: >
+      az network watcher flow-log create
+      --location westus
+      --name MyFlowLog
+      --nsg MyNetworkSecurityGroupID
+      --storage-account account
+"""
+
+helps['network watcher flow-log list'] = """
+type: command
+short-summary: List all flow log resources for the specified Network Watcher
+"""
+
+helps['network watcher flow-log delete'] = """
+type: command
+short-summary: Delete the specified flow log resource.
+"""
+
 helps['network watcher flow-log show'] = """
 type: command
 short-summary: Get the flow log configuration of a network security group.
 examples:
-  - name: Show NSG flow logs.
+  - name: Show NSG flow logs. (Deprecated)
     text: az network watcher flow-log show -g MyResourceGroup --nsg MyNsg
+  - name: Show NSG flow logs with Azure Resource Management formatted.
+    text: az network watcher flow-log show --location MyNetworkWatcher --name MyFlowLog
+"""
+
+helps['network watcher flow-log update'] = """
+type: command
+short-summary: Update the flow log configuration of a network security group
+examples:
+  - name: Update storage account with name to let resource group identify the storage account and network watcher
+    text: >
+      az network watcher flow-log update
+      --location westus
+      --resource-group MyResourceGroup
+      --name MyFlowLog
+      --storage-account accountname
+  - name: Update storage account with ID to let location identify the network watcher
+    text: >
+      az network watcher flow-log update
+      --location westus
+      --resource-group MyResourceGroup
+      --name MyFlowLog
+      --storage-account accountid
+  - name: Update Network Security Group on another resource group
+    text: >
+      az network watcher flow-log update
+      --location westus
+      --resource-group MyAnotherResourceGroup
+      --name MyFlowLog
+      --nsg MyNSG
+  - name: Update Workspace on another resource group
+    text: >
+      az network watcher flow-log update
+      --location westus
+      --resource-group MyAnotherResourceGroup
+      --name MyFlowLog
+      --workspace MyAnotherLogAnalyticWorkspace
 """
 
 helps['network watcher list'] = """

src/azure-cli/azure/cli/command_modules/network/_params.py

@@ -1348,16 +1348,33 @@ def load_arguments(self, _):
         c.argument('filters', type=get_json_object)
 
     with self.argument_context('network watcher flow-log') as c:
+        c.argument('location', get_location_type(self.cli_ctx),
+                   help='Location to identify the exclusive Network Watcher under a region. '
+                        'Only one Network Watcher can be existed per subscription and region.')
+        c.argument('flow_log_name', name_arg_type, help='The name of the flow logger', min_api='2019-11-01')
         c.argument('nsg', help='Name or ID of the network security group.')
-        c.argument('enabled', arg_type=get_three_state_flag())
+        c.argument('enabled', arg_type=get_three_state_flag(), help='Enable logging', default='true')
+        c.argument('retention', type=int, help='Number of days to retain logs')
+        c.argument('storage_account', help='Name or ID of the storage account in which to save the flow logs. '
+                                           'Must be in the same region of flow log.')
+
+    # temporary solution for compatible with old show command's parameter
+    # after old show command's parameter is deprecated and removed,
+    # this argument group "network watcher flow-log show" should be removed
+    with self.argument_context('network watcher flow-log show') as c:
+        c.argument('nsg',
+                   deprecate_info=c.deprecate(redirect='--location and --watcher combination', hide=False),
+                   help='Name or ID of the network security group.')
 
     with self.argument_context('network watcher flow-log', arg_group='Format', min_api='2018-10-01') as c:
         c.argument('log_format', options_list='--format', help='File type of the flow log.', arg_type=get_enum_type(FlowLogFormatType))
         c.argument('log_version', help='Version (revision) of the flow log.', type=int)
 
     with self.argument_context('network watcher flow-log', arg_group='Traffic Analytics', min_api='2018-10-01') as c:
         c.argument('traffic_analytics_interval', type=int, options_list='--interval', help='Interval in minutes at which to conduct flow analytics. Temporarily allowed values are 10 and 60.', min_api='2018-12-01')
-        c.argument('traffic_analytics_workspace', options_list='--workspace', help='Name or ID of a Log Analytics workspace.')
+        c.argument('traffic_analytics_workspace',
+                   options_list='--workspace',
+                   help='Name or ID of a Log Analytics workspace. Must be in the same region of flow log')
         c.argument('traffic_analytics_enabled', options_list='--traffic-analytics', arg_type=get_three_state_flag(), help='Enable traffic analytics. Defaults to true if `--workspace` is provided.')
 
     for item in ['list', 'stop', 'delete', 'show', 'show-status']:

src/azure-cli/azure/cli/command_modules/network/_validators.py

@@ -1379,6 +1379,66 @@ def process_nw_test_connectivity_namespace(cmd, namespace):
         namespace.headers = headers
 
 
+def process_nw_flow_log_create_namespace(cmd, namespace):
+    """
+    Flow Log is the sub-resource of Network Watcher, they must be in the same region and subscription.
+    """
+    from msrestazure.tools import is_valid_resource_id, resource_id
+
+    # for both create and update
+    if namespace.resource_group_name is None:
+        err_tpl, err_body = 'usage error: use {} instead.', None
+
+        if namespace.nsg and not is_valid_resource_id(namespace.nsg):
+            err_body = '--nsg ID / --nsg NSD_NAME --resource-group NSD_RESOURCE_GROUP'
+
+        if namespace.storage_account and not is_valid_resource_id(namespace.storage_account):
+            err_body = '--storage-account ID / --storage-account NAME --resource_group STORAGE_ACCOUNT_RESOURCE_GROUP'
+
+        if namespace.traffic_analytics_workspace and not is_valid_resource_id(namespace.traffic_analytics_workspace):
+            err_body = '--workspace ID / --workspace NAME --resource-group WORKSPACE_RESOURCE_GROUP'
+
+        if err_body is not None:
+            raise CLIError(err_tpl.format(err_body))
+
+    # for both create and update
+    if namespace.nsg and not is_valid_resource_id(namespace.nsg):
+        kwargs = {
+            'subscription': get_subscription_id(cmd.cli_ctx),
+            'resource_group': namespace.resource_group_name,
+            'namespace': 'Microsoft.Network',
+            'type': 'networkSecurityGroups',
+            'name': namespace.nsg
+        }
+        namespace.nsg = resource_id(**kwargs)
+
+    # for both create and update
+    if namespace.storage_account and not is_valid_resource_id(namespace.storage_account):
+        kwargs = {
+            'subscription': get_subscription_id(cmd.cli_ctx),
+            'resource_group': namespace.resource_group_name,
+            'namespace': 'Microsoft.Storage',
+            'type': 'storageAccounts',
+            'name': namespace.storage_account
+        }
+        namespace.storage_account = resource_id(**kwargs)
+
+    # for both create and update
+    if namespace.traffic_analytics_workspace and not is_valid_resource_id(namespace.traffic_analytics_workspace):
+        kwargs = {
+            'subscription': get_subscription_id(cmd.cli_ctx),
+            'resource_group': namespace.resource_group_name,
+            'namespace': 'Microsoft.OperationalInsights',
+            'type': 'workspaces',
+            'name': namespace.traffic_analytics_workspace
+        }
+        namespace.traffic_analytics_workspace = resource_id(**kwargs)
+
+    get_network_watcher_from_location(remove=False)(cmd, namespace)
+
+    validate_tags(namespace)
+
+
 def process_nw_flow_log_set_namespace(cmd, namespace):
     from msrestazure.tools import is_valid_resource_id, resource_id
     if namespace.storage_account and not is_valid_resource_id(namespace.storage_account):
@@ -1403,17 +1463,22 @@ def process_nw_flow_log_show_namespace(cmd, namespace):
     from msrestazure.tools import is_valid_resource_id, resource_id
     from azure.cli.core.commands.arm import get_arm_resource_by_id
 
-    if not is_valid_resource_id(namespace.nsg):
-        namespace.nsg = resource_id(
-            subscription=get_subscription_id(cmd.cli_ctx),
-            resource_group=namespace.resource_group_name,
-            namespace='Microsoft.Network',
-            type='networkSecurityGroups',
-            name=namespace.nsg)
-
-    nsg = get_arm_resource_by_id(cmd.cli_ctx, namespace.nsg)
-    namespace.location = nsg.location  # pylint: disable=no-member
-    get_network_watcher_from_location(remove=True)(cmd, namespace)
+    if hasattr(namespace, 'nsg') and namespace.nsg is not None:
+        if not is_valid_resource_id(namespace.nsg):
+            namespace.nsg = resource_id(
+                subscription=get_subscription_id(cmd.cli_ctx),
+                resource_group=namespace.resource_group_name,
+                namespace='Microsoft.Network',
+                type='networkSecurityGroups',
+                name=namespace.nsg)
+
+        nsg = get_arm_resource_by_id(cmd.cli_ctx, namespace.nsg)
+        namespace.location = nsg.location  # pylint: disable=no-member
+        get_network_watcher_from_location(remove=True)(cmd, namespace)
+    elif namespace.flow_log_name is not None and namespace.location is not None:
+        get_network_watcher_from_location(remove=False)(cmd, namespace)
+    else:
+        raise CLIError('usage error: --nsg NSG | --location NETWORK_WATCHER_LOCATION --name FLOW_LOW_NAME')
 
 
 def process_nw_topology_namespace(cmd, namespace):

src/azure-cli/azure/cli/command_modules/network/commands.py

@@ -27,7 +27,7 @@
     cf_express_route_circuit_connections, cf_express_route_gateways, cf_express_route_connections,
     cf_express_route_ports, cf_express_route_port_locations, cf_express_route_links, cf_app_gateway_waf_policy,
     cf_service_tags, cf_private_link_services, cf_private_endpoint_types, cf_peer_express_route_circuit_connections,
-    cf_virtual_router, cf_virtual_router_peering, cf_service_aliases, cf_bastion_hosts)
+    cf_virtual_router, cf_virtual_router_peering, cf_service_aliases, cf_bastion_hosts, cf_flow_logs)
 from azure.cli.command_modules.network._util import (
     list_network_resource_property, get_network_resource_property_entry, delete_network_resource_property_entry)
 from azure.cli.command_modules.network._format import (
@@ -43,14 +43,15 @@
     transform_vnet_table_output, transform_effective_route_table, transform_effective_nsg,
     transform_vnet_gateway_routes_table, transform_vnet_gateway_bgp_peer_table)
 from azure.cli.command_modules.network._validators import (
+    get_network_watcher_from_location,
     process_ag_create_namespace, process_ag_listener_create_namespace, process_ag_http_settings_create_namespace,
     process_ag_rule_create_namespace, process_ag_ssl_policy_set_namespace, process_ag_url_path_map_create_namespace,
     process_ag_url_path_map_rule_create_namespace, process_auth_create_namespace, process_nic_create_namespace,
     process_lb_create_namespace, process_lb_frontend_ip_namespace, process_local_gateway_create_namespace,
     process_nw_cm_create_namespace,
     process_nw_cm_v2_endpoint_namespace, process_nw_cm_v2_test_configuration_namespace,
     process_nw_cm_v2_test_group, process_nw_cm_v2_output_namespace,
-    process_nw_flow_log_set_namespace, process_nw_flow_log_show_namespace,
+    process_nw_flow_log_set_namespace, process_nw_flow_log_create_namespace, process_nw_flow_log_show_namespace,
     process_nw_packet_capture_create_namespace, process_nw_test_connectivity_namespace, process_nw_topology_namespace,
     process_nw_troubleshooting_start_namespace, process_nw_troubleshooting_show_namespace,
     process_public_ip_create_namespace, process_tm_endpoint_create_namespace,
@@ -300,6 +301,16 @@ def load_command_table(self, _):
         client_factory=cf_connection_monitor
     )
 
+    network_watcher_flow_log_sdk = CliCommandType(
+        operations_tmpl='azure.mgmt.network.operations#FlowLogsOperations.{}',
+        client_factory=cf_flow_logs,
+    )
+
+    network_watcher_flow_log_update_sdk = CliCommandType(
+        operations_tmpl='azure.cli.command_modules.network.custom#{}',
+        client_factory=cf_flow_logs,
+    )
+
     network_watcher_pc_sdk = CliCommandType(
         operations_tmpl='azure.mgmt.network.operations#PacketCapturesOperations.{}',
         client_factory=cf_packet_capture
@@ -910,9 +921,31 @@ def _make_singular(value):
         g.command('list', 'list')
 
     with self.command_group('network watcher flow-log', client_factory=cf_network_watcher, min_api='2016-09-01') as g:
-        g.custom_command('configure', 'set_nsg_flow_logging', validator=process_nw_flow_log_set_namespace)
+        g.custom_command('configure',
+                         'set_nsg_flow_logging',
+                         validator=process_nw_flow_log_set_namespace,
+                         deprecate_info=self.deprecate(redirect='network watcher flow-log create', hide=False))
         g.custom_show_command('show', 'show_nsg_flow_logging', validator=process_nw_flow_log_show_namespace)
 
+    with self.command_group('network watcher flow-log',
+                            network_watcher_flow_log_sdk,
+                            client_factory=cf_flow_logs,
+                            min_api='2019-11-01',
+                            validator=get_network_watcher_from_location(remove=False)) as g:
+        g.custom_command('list', 'list_nw_flow_log', validator=get_network_watcher_from_location(remove=False))
+        g.custom_command('delete', 'delete_nw_flow_log', validator=get_network_watcher_from_location(remove=False))
+        g.custom_command('create',
+                         'create_nw_flow_log',
+                         client_factory=cf_flow_logs,
+                         validator=process_nw_flow_log_create_namespace)
+        g.generic_update_command('update',
+                                 getter_name='update_nw_flow_log_getter',
+                                 getter_type=network_watcher_flow_log_update_sdk,
+                                 setter_name='update_nw_flow_log_setter',
+                                 setter_type=network_watcher_flow_log_update_sdk,
+                                 custom_func_name='update_nw_flow_log',
+                                 validator=process_nw_flow_log_create_namespace)
+
     with self.command_group('network watcher troubleshooting', client_factory=cf_network_watcher, min_api='2016-09-01') as g:
         g.custom_command('start', 'start_nw_troubleshooting', supports_no_wait=True, validator=process_nw_troubleshooting_start_namespace)
         g.custom_show_command('show', 'show_nw_troubleshooting_result', validator=process_nw_troubleshooting_show_namespace)