Azdatalake stg 78-82 (#22192)

* STG 82 swagger update, azcore, azidentity dependancy updates (#22000)

* corrected release date in changelog

* STG84 swagger and azcore, azidentity, azblob dependency updates

* dependancy updates

* indirect dependancy

* update code generator version

* using STG 82 swagger, and revert code gen version

* adding transforms for missing header files

* re-generate zz_options.go file

* reverting to azblob 1.1.0

* reverting to azblob 1.0.0

* Updated input file from main to latest commit id

* Encryption scope SAS (#22099)

* corrected release date in changelog

* STG84 swagger and azcore, azidentity, azblob dependency updates

* dependancy updates

* Initial changes for encryption scope sas

* indirect dependancy

* update code generator version

* using STG 82 swagger, and revert code gen version

* adding transforms for missing header files

* re-generate zz_options.go file

* reverting to azblob 1.1.0

* reverting to azblob 1.0.0

* Updated input file from main to latest commit id

* added tests for encryption scope sas

* add test for user delegation sas

* lint error fix

* Fix lint error in test

* created new datalake encryption scope var

* removed blob account encryption scope

* push recording

* add encryption scope in user delegation

* recordings

* linter error

* minor comments resolve

* arm template changes

* arm template changes

* test fix

---------

Co-authored-by: Sourav Gupta <souravgupta@microsoft.com>

* List system containers (#22108)

* corrected release date in changelog

* list system containers test

* push recording

* push recording

* recording

* recordings

* Datalake CPK (#22159)

* corrected release date in changelog

* datalake cpk

* Update CHANGELOG.md

add newline changelog

* download file & stream tests

* recordings

* recordings

* change to unrecorded test

* add negative tests for CPK

* [AzDatalake] Set Expiry (#22170)

* corrected release date in changelog

* setExpiry for files

* lint error

* recordings

* create new sub directory (#22177)

* Implement NewSubdirectoryClient

* Lowercase d in subDirectoryName

Co-authored-by: Gauri Prasad <51212198+gapra-msft@users.noreply.github.com>

* Make test recorded and add API calls

* Extend testing

* Add recorded tests

* Update recorded test

* doc

* Update assets.json

* create subdirectory client

* changelog

* revert to unrecorded test

---------

Co-authored-by: Adele Reed <adreed@microsoft.com>
Co-authored-by: adreed-msft <49764384+adreed-msft@users.noreply.github.com>
Co-authored-by: Gauri Prasad <51212198+gapra-msft@users.noreply.github.com>

* [AzDatalake] Blob swagger update to latest 2021-12-02 (#22178)

* blob swagger update to latest 2021-12-02

* Update sdk/storage/azdatalake/internal/generated_blob/autorest.md

Co-authored-by: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com>

* unmarshall-xml list blobs

* unmarshall-xml list blobs

---------

Co-authored-by: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com>

* service version update

* service version upgrade

---------

Co-authored-by: Sourav Gupta <souravgupta@microsoft.com>
Co-authored-by: Adele Reed <adreed@microsoft.com>
Co-authored-by: adreed-msft <49764384+adreed-msft@users.noreply.github.com>
Co-authored-by: Gauri Prasad <51212198+gapra-msft@users.noreply.github.com>
Co-authored-by: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com>

sdk/storage/azdatalake/CHANGELOG.md

@@ -3,6 +3,10 @@
 ## 1.0.1 (Unreleased)
 
 ### Features Added
+* Encryption Scope For SAS
+* CPK for Datalake
+* Create SubDirectory Client
+* Service Version upgrade to 2021-06-08
 
 ### Breaking Changes
 
@@ -20,4 +24,4 @@
 
 ### Features Added
 
-* This is the initial preview release of the `azdatalake` library
+* This is the initial preview release of the `azdatalake` library

sdk/storage/azdatalake/README.md

@@ -1,6 +1,6 @@
 # ADLS Gen2 Storage SDK for Go
 
-> Service Version: 2020-10-02
+> Service Version: 2021-06-08
 
 Azure Data Lake Storage Gen2 (ADLS Gen2) is Microsoft's hierarchical object storage solution for the cloud with converged capabilities with Azure Blob Storage. 
 For example, Data Lake Storage Gen2 provides file system semantics, file-level security, and scale. 

sdk/storage/azdatalake/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "go",
   "TagPrefix": "go/storage/azdatalake",
-  "Tag": "go/storage/azdatalake_7c0fb050cb"
+  "Tag": "go/storage/azdatalake_ba5bfaa832"
 }

sdk/storage/azdatalake/directory/client.go

@@ -228,6 +228,29 @@ func (d *Client) NewFileClient(fileName string) (*file.Client, error) {
 	return (*file.Client)(base.NewPathClient(fileURL, newBlobURL, newBlobClient, d.generatedDirClientWithDFS().InternalClient().WithClientName(shared.FileClient), d.sharedKey(), d.identityCredential(), d.getClientOptions())), nil
 }
 
+// NewSubdirectoryClient creates a new directory.Client object by concatenating subdirectoryName to the end of this Client's URL.
+// The new directory.Client uses the same request policy pipeline as the Client.
+func (d *Client) NewSubdirectoryClient(subdirectoryName string) (*Client, error) {
+	subdirectoryName = url.PathEscape(strings.TrimRight(subdirectoryName, "/"))
+	subDirectoryURL := runtime.JoinPaths(d.DFSURL(), subdirectoryName)
+	newBlobURL, subDirectoryURL := shared.GetURLs(subDirectoryURL)
+	var newBlobClient *blockblob.Client
+	clientOptions := &blockblob.ClientOptions{ClientOptions: d.getClientOptions().ClientOptions}
+	var err error
+	if d.identityCredential() != nil {
+		newBlobClient, err = blockblob.NewClient(newBlobURL, *d.identityCredential(), clientOptions)
+	} else if d.sharedKey() != nil {
+		blobSharedKey, _ := exported.ConvertToBlobSharedKey(d.sharedKey())
+		newBlobClient, err = blockblob.NewClientWithSharedKeyCredential(newBlobURL, blobSharedKey, clientOptions)
+	} else {
+		newBlobClient, err = blockblob.NewClientWithNoCredential(newBlobURL, clientOptions)
+	}
+	if err != nil {
+		return nil, exported.ConvertToDFSError(err)
+	}
+	return (*Client)(base.NewPathClient(subDirectoryURL, newBlobURL, newBlobClient, d.generatedDirClientWithDFS().InternalClient().WithClientName(shared.DirectoryClient), d.sharedKey(), d.identityCredential(), d.getClientOptions())), nil
+}
+
 // Create creates a new directory.
 func (d *Client) Create(ctx context.Context, options *CreateOptions) (CreateResponse, error) {
 	lac, mac, httpHeaders, createOpts, cpkOpts := options.format()
@@ -295,7 +318,7 @@ func (d *Client) Rename(ctx context.Context, destinationPath string, options *Re
 		newPathURL = strings.Split(newPathURL, "?")[0] + "?" + newDestQuery
 	}
 	newBlobURL, _ := shared.GetURLs(newPathURL)
-	lac, mac, smac, createOpts := path.FormatRenameOptions(options, newSrcPath)
+	lac, mac, smac, createOpts, cpkOpts := path.FormatRenameOptions(options, newSrcPath)
 
 	if d.identityCredential() != nil {
 		newBlobClient, err = blockblob.NewClient(newBlobURL, *d.identityCredential(), nil)
@@ -310,7 +333,7 @@ func (d *Client) Rename(ctx context.Context, destinationPath string, options *Re
 		return RenameResponse{}, exported.ConvertToDFSError(err)
 	}
 	newDirClient := (*Client)(base.NewPathClient(newPathURL, newBlobURL, newBlobClient, d.generatedDirClientWithDFS().InternalClient().WithClientName(shared.DirectoryClient), d.sharedKey(), d.identityCredential(), d.getClientOptions()))
-	resp, err := newDirClient.generatedDirClientWithDFS().Create(ctx, createOpts, nil, lac, mac, smac, nil)
+	resp, err := newDirClient.generatedDirClientWithDFS().Create(ctx, createOpts, nil, lac, mac, smac, cpkOpts)
 	//return RenameResponse{
 	//	Response:           resp,
 	//	NewDirectoryClient: newDirClient,

sdk/storage/azdatalake/directory/client_test.go

@@ -72,7 +72,6 @@ type UnrecordedTestSuite struct {
 }
 
 func (s *UnrecordedTestSuite) TestCreateDirAndDeleteWithConnectionString() {
-
 	_require := require.New(s.T())
 	testName := s.T().Name()
 
@@ -147,6 +146,31 @@ func (s *RecordedTestSuite) TestCreateDirAndDelete() {
 	_require.NotNil(resp)
 }
 
+func (s *RecordedTestSuite) TestCreateDirUsingCPK() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+
+	filesystemName := testcommon.GenerateFileSystemName(testName)
+	fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	_, err = fsClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	dirName := testcommon.GenerateDirName(testName)
+	dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+
+	dirOpts := &directory.CreateOptions{CPKInfo: &testcommon.TestCPKByValue}
+	resp, err := dirClient.Create(context.Background(), dirOpts)
+	_require.NoError(err)
+	_require.NotNil(resp)
+
+	_require.Equal(true, *(resp.IsServerEncrypted))
+	_require.Equal(testcommon.TestCPKByValue.EncryptionKeySHA256, resp.EncryptionKeySHA256)
+}
+
 func (s *RecordedTestSuite) TestGetAndCreateFileClient() {
 	_require := require.New(s.T())
 	testName := s.T().Name()
@@ -177,6 +201,70 @@ func (s *RecordedTestSuite) TestGetAndCreateFileClient() {
 	_require.NoError(err)
 }
 
+func (s *RecordedTestSuite) TestCreateNewSubdirectoryClient() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+
+	accountName, _ := testcommon.GetGenericAccountInfo(testcommon.TestAccountDatalake)
+	_require.Greater(len(accountName), 0)
+
+	svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+
+	fsName := testcommon.GenerateFileSystemName(testName)
+	fsClient := svcClient.NewFileSystemClient(fsName)
+
+	_, err = fsClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	dirName := testcommon.GenerateDirName(testName)
+	dirClient := fsClient.NewDirectoryClient(dirName)
+
+	_, err = dirClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	subdirName := testcommon.GenerateSubDirName(testName)
+	subdirClient, err := dirClient.NewSubdirectoryClient(subdirName)
+	_require.NoError(err)
+
+	perm := "r-xr-x---"
+
+	_, err = subdirClient.Create(context.Background(), &directory.CreateOptions{
+		Permissions: &perm,
+		CPKInfo:     &testcommon.TestCPKByValue,
+	})
+	_require.NoError(err)
+
+	resp, err := subdirClient.GetProperties(context.Background(), &directory.GetPropertiesOptions{CPKInfo: &testcommon.TestCPKByValue})
+	_require.NoError(err)
+	_require.NotNil(resp.Permissions)
+	_require.Equal(perm, *resp.Permissions)
+	_require.Equal(*(resp.IsServerEncrypted), true)
+	_require.Equal(resp.EncryptionKeySHA256, testcommon.TestCPKByValue.EncryptionKeySHA256)
+
+	// Create a file under the new directory just to make sure we're not secretly targeting the parent
+	fileName := testcommon.GenerateFileName("newFile")
+	subdirFileClient, err := subdirClient.NewFileClient(fileName)
+	_require.NoError(err)
+
+	_, err = subdirFileClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	fileContents := []byte(testcommon.DefaultData)
+	err = subdirFileClient.UploadBuffer(context.Background(), fileContents, nil)
+	_require.NoError(err)
+
+	// check for the file at the parent directory
+	dirFileClient, err := dirClient.NewFileClient(fileName)
+	_require.NoError(err)
+
+	_, err = dirFileClient.GetProperties(context.Background(), nil)
+	_require.Error(err) // we should get back a 404
+	_require.True(datalakeerror.HasCode(err, datalakeerror.PathNotFound))
+}
+
 func (s *RecordedTestSuite) TestCreateDirWithNilAccessConditions() {
 	_require := require.New(s.T())
 	testName := s.T().Name()
@@ -1912,6 +2000,65 @@ func (s *RecordedTestSuite) TestDirSetMetadataWithAccessConditions() {
 	_require.NoError(err)
 }
 
+func (s *RecordedTestSuite) TestDirSetMetadataWithCPK() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+
+	filesystemName := testcommon.GenerateFileSystemName(testName)
+	fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	_, err = fsClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	dirName := testcommon.GenerateDirName(testName)
+	dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+
+	defer testcommon.DeleteDir(context.Background(), _require, dirClient)
+
+	resp, err := dirClient.Create(context.Background(), &directory.CreateOptions{CPKInfo: &testcommon.TestCPKByValue})
+	_require.NoError(err)
+	_require.NotNil(resp)
+
+	opts := &directory.SetMetadataOptions{
+		CPKInfo: &testcommon.TestCPKByValue,
+	}
+	_, err = dirClient.SetMetadata(context.Background(), testcommon.BasicMetadata, opts)
+	_require.NoError(err)
+}
+
+func (s *RecordedTestSuite) TestDirSetMetadataWithCPKNegative() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+
+	filesystemName := testcommon.GenerateFileSystemName(testName)
+	fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	_, err = fsClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	dirName := testcommon.GenerateDirName(testName)
+	dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+
+	defer testcommon.DeleteDir(context.Background(), _require, dirClient)
+
+	resp, err := dirClient.Create(context.Background(), nil)
+	_require.NoError(err)
+	_require.NotNil(resp)
+
+	opts := &directory.SetMetadataOptions{
+		CPKInfo: &testcommon.TestCPKByValue,
+	}
+	_, err = dirClient.SetMetadata(context.Background(), testcommon.BasicMetadata, opts)
+	_require.Error(err)
+	_require.ErrorContains(err, "PathDoesNotUseCustomerSpecifiedEncryption")
+}
+
 func validatePropertiesSet(_require *require.Assertions, dirClient *directory.Client, disposition string) {
 	resp, err := dirClient.GetProperties(context.Background(), nil)
 	_require.NoError(err)
@@ -2243,6 +2390,38 @@ func (s *RecordedTestSuite) TestDirRenameNoOptions() {
 	//_require.Contains(resp1.NewDirectoryClient.DFSURL(), "newName")
 }
 
+func (s *RecordedTestSuite) TestDirRenameRequestWithCPK() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+
+	filesystemName := testcommon.GenerateFileSystemName(testName)
+	fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	_, err = fsClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	dirName := testcommon.GenerateDirName(testName)
+	dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+
+	createOpts := &directory.CreateOptions{
+		CPKInfo: &testcommon.TestCPKByValue,
+	}
+
+	resp, err := dirClient.Create(context.Background(), createOpts)
+	_require.NoError(err)
+	_require.NotNil(resp)
+
+	renameFileOpts := &directory.RenameOptions{
+		CPKInfo: &testcommon.TestCPKByValue,
+	}
+
+	_, err = dirClient.Rename(context.Background(), "newName", renameFileOpts)
+	_require.NoError(err)
+}
+
 func (s *RecordedTestSuite) TestRenameDirWithNilAccessConditions() {
 	_require := require.New(s.T())
 	testName := s.T().Name()
@@ -2548,6 +2727,35 @@ func (s *RecordedTestSuite) TestDirGetPropertiesResponseCapture() {
 	_require.Equal("directory", respFromCtxService.Header.Get("x-ms-resource-type"))
 }
 
+func (s *RecordedTestSuite) TestDirGetPropertiesWithCPK() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+
+	filesystemName := testcommon.GenerateFileSystemName(testName)
+	fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	_, err = fsClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	dirName := testcommon.GenerateDirName(testName)
+	dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+
+	dirOpts := &directory.CreateOptions{CPKInfo: &testcommon.TestCPKByValue}
+	resp, err := dirClient.Create(context.Background(), dirOpts)
+	_require.NoError(err)
+	_require.NotNil(resp)
+
+	getPropertiesOpts := &directory.GetPropertiesOptions{CPKInfo: &testcommon.TestCPKByValue}
+	response, err := dirClient.GetProperties(context.Background(), getPropertiesOpts)
+	_require.NoError(err)
+	_require.NotNil(response)
+	_require.Equal(*(resp.IsServerEncrypted), true)
+	_require.Equal(resp.EncryptionKeySHA256, testcommon.TestCPKByValue.EncryptionKeySHA256)
+}
+
 func (s *UnrecordedTestSuite) TestDirCreateDeleteUsingOAuth() {
 	_require := require.New(s.T())
 	testName := s.T().Name()

sdk/storage/azdatalake/file/client.go

@@ -278,7 +278,7 @@ func (f *Client) Rename(ctx context.Context, destinationPath string, options *Re
 		newPathURL = strings.Split(newPathURL, "?")[0] + "?" + newDestQuery
 	}
 	newBlobURL, _ := shared.GetURLs(newPathURL)
-	lac, mac, smac, createOpts := path.FormatRenameOptions(options, newSrcPath)
+	lac, mac, smac, createOpts, cpkOpts := path.FormatRenameOptions(options, newSrcPath)
 
 	if f.identityCredential() != nil {
 		newBlobClient, err = blockblob.NewClient(newBlobURL, *f.identityCredential(), nil)
@@ -293,7 +293,7 @@ func (f *Client) Rename(ctx context.Context, destinationPath string, options *Re
 		return RenameResponse{}, exported.ConvertToDFSError(err)
 	}
 	newFileClient := (*Client)(base.NewPathClient(newPathURL, newBlobURL, newBlobClient, f.generatedFileClientWithDFS().InternalClient().WithClientName(shared.FileClient), f.sharedKey(), f.identityCredential(), f.getClientOptions()))
-	resp, err := newFileClient.generatedFileClientWithDFS().Create(ctx, createOpts, nil, lac, mac, smac, nil)
+	resp, err := newFileClient.generatedFileClientWithDFS().Create(ctx, createOpts, nil, lac, mac, smac, cpkOpts)
 
 	//return RenameResponse{
 	//	Response:      resp,