[Internal] Pipelines: Adds nightly build to produce packages (#3802)

* Support cleaning

* wire previous content delete

* as text

* with variable

* another test

* param with types

* as string

* no delete

* no quotes

* undoing

* re-adding quotes

* testing empty

* trying another test

* readding version

* fixing publishing artifacts

* fixing parameter

* Fixing official pipeline

* version 5

* fixing main pipeline

* test with true

* using start time

* nightly preview

* passing parameters to pack

* Fixing nuget version

* arguments on the nuget pack

* folder structure

* testing v5

* Using only content

* Removing currentDate

Microsoft.Azure.Cosmos/src/Microsoft.Azure.Cosmos.csproj

@@ -9,8 +9,9 @@
 		<CurrentDate>$([System.DateTime]::Now.ToString(yyyyMMdd))</CurrentDate>
 		<ClientVersion Condition=" '$(IsPreview)' != 'true' ">$(ClientOfficialVersion)</ClientVersion>
 		<ClientVersion Condition=" '$(IsPreview)' == 'true' ">$(ClientPreviewVersion)</ClientVersion>
-		<VersionSuffix Condition=" '$(IsNightly)' == 'true' ">nightly-$(CurrentDate)</VersionSuffix>
 		<VersionSuffix Condition=" '$(IsPreview)' == 'true' ">$(ClientPreviewSuffixVersion)</VersionSuffix>
+		<VersionSuffix Condition=" '$(IsNightly)' == 'true' AND '$(IsPreview)' != 'true' ">nightly-$(CurrentDate)</VersionSuffix>
+		<VersionSuffix Condition=" '$(IsNightly)' == 'true' AND '$(IsPreview)' == 'true' ">$(VersionSuffix)-nightly-$(CurrentDate)</VersionSuffix>
 		<Version Condition=" '$(VersionSuffix)' == '' ">$(ClientVersion)</Version>
 		<Version Condition=" '$(VersionSuffix)' != '' ">$(ClientVersion)-$(VersionSuffix)</Version>
 		<FileVersion>$(ClientVersion)</FileVersion>

azure-pipelines-nightly.yml

@@ -13,64 +13,32 @@ schedules:
 
 
 variables:
-  ReleaseArguments: ' --filter "TestCategory!=Quarantine" --verbosity normal ' 
   VmImage: windows-latest # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops 
-  BuildConfiguration: Release 
-  IsNightly: true 
-
+  BuildConfiguration: Release
+  Packaging.EnableSBOMSigning: true
 
 stages:
 - stage:
-  displayName: Gate 
+  displayName: Generate nightly GA 
   jobs:
-    - template:  templates/static-tools.yml
-      parameters:
-        BuildConfiguration: $(BuildConfiguration)
-        VmImage: $(VmImage)
-
-
-    - template:  templates/build-test.yml
+    - template:  templates/nuget-pack.yml
       parameters:
-        BuildConfiguration: $(BuildConfiguration)
-        Arguments: $(ReleaseArguments) /p:IsNightly=true /p:GeneratePackageOnBuild=true 
+        BuildConfiguration: Release
+        Arguments: /p:IsNightly=true
         VmImage: $(VmImage)
-
-
+        ReleasePackage: true
+        OutputPath: '$(Build.ArtifactStagingDirectory)/bin/AnyCPU/$(BuildConfiguration)/Microsoft.Azure.Cosmos'
+        BlobVersion: 'nightly'
+        CleanupFolder: true
 - stage:
-  displayName: Publish 
+  displayName: Generate nightly preview 
   jobs:
-    - job:
-      pool:
-        vmImage: $(VmImage)
-
-      steps:
-      - task: DotNetCoreCLI@2
-        displayName: Build Microsoft.Azure.Cosmos
-        inputs: 
-          command: build  
-          configuration: $(BuildConfiguration)
-          nugetConfigPath: NuGet.config
-          projects: Microsoft.Azure.Cosmos/src/Microsoft.Azure.Cosmos.csproj 
-          arguments: --configuration $(BuildConfiguration) 
-          versioningScheme: OFF
-
-      - task: DotNetCoreCLI@2
-        displayName: Pack Microsoft.Azure.Cosmos
-        inputs: 
-          command: pack 
-          configuration: $(BuildConfiguration)
-          searchPatternPack: Microsoft.Azure.Cosmos/src/Microsoft.Azure.Cosmos.csproj
-          arguments: --configuration $(BuildConfiguration) 
-          versioningScheme: OFF
-
-      - task: DotNetCoreCLI@2
-        displayName: Push Microsoft.Azure.Cosmos
-        inputs: 
-          command: push 
-          configuration: $(BuildConfiguration)
-          searchPatternPack: Microsoft.Azure.Cosmos/src/Microsoft.Azure.Cosmos.csproj
-          arguments: --configuration $(BuildConfiguration) 
-          packagesToPush: '$(Build.ArtifactStagingDirectory)/*.nupkg'
-          nuGetFeedType: 'internal'
-          publishVstsFeed: '4000bd49-81c3-47f2-94d8-d1392b95c228/04efb628-f46d-4b48-ac4d-5af5b1c75043' #azure-cosmos-dotnet
-          versioningScheme: OFF
+    - template:  templates/nuget-pack.yml
+      parameters:
+        BuildConfiguration: Release
+        Arguments: /p:IsNightly=true /p:IsPreview=true
+        VmImage: $(VmImage)
+        ReleasePackage: true
+        OutputPath: '$(Build.ArtifactStagingDirectory)/bin/AnyCPU/$(BuildConfiguration)/Microsoft.Azure.Cosmos'
+        BlobVersion: 'nightly-preview'
+        CleanupFolder: true

azure-pipelines-official.yml

@@ -29,8 +29,7 @@ stages:
     - template:  templates/nuget-pack.yml
       parameters:
         BuildConfiguration: Release
-        Arguments: $(ReleaseArguments)
         VmImage: $(VmImage)
         ReleasePackage: true
         OutputPath: '$(Build.ArtifactStagingDirectory)/bin/AnyCPU/$(BuildConfiguration)/Microsoft.Azure.Cosmos'
-        BlobVersion: variables['BlobVersion']
+        BlobVersion: $(BlobVersion)

azure-pipelines.yml

@@ -29,7 +29,6 @@ jobs:
 - template:  templates/nuget-pack.yml
   parameters:
     BuildConfiguration: Release
-    Arguments: $(ReleaseArguments)
     VmImage: $(VmImage)
     ReleasePackage: false
     OutputPath: '$(Build.ArtifactStagingDirectory)/bin/AnyCPU/Release/Microsoft.Azure.Cosmos'

templates/nuget-pack.yml

@@ -1,13 +1,27 @@
 # File: templates/nuget-pack.yml
 
 parameters:
-  BuildConfiguration: ''
-  Arguments: ''
-  VmImage: '' # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops
-  OS: 'Windows'
-  OutputPath: ''
-  ReleasePackage: false
-  BlobVersion: ''
+  - name: BuildConfiguration
+    type: string
+    default: ''
+  - name: Arguments
+    type: string
+    default: ''
+  - name: VmImage
+    type: string
+    default: ''
+  - name: OutputPath
+    type: string
+    default: ''
+  - name: BlobVersion
+    type: string
+    default: ''
+  - name: ReleasePackage
+    type: boolean
+    default: false
+  - name: CleanupFolder
+    type: boolean
+    default: false
 
 jobs:
 - job: GenerateNugetPackages
@@ -23,7 +37,7 @@ jobs:
       configuration: $(BuildConfiguration)
       nugetConfigPath: NuGet.config
       projects: Microsoft.Azure.Cosmos/src/Microsoft.Azure.Cosmos.csproj 
-      arguments: --configuration ${{ parameters.BuildConfiguration }} -p:Optimize=true 
+      arguments: --configuration ${{ parameters.BuildConfiguration }} -p:Optimize=true ${{ parameters.Arguments }}
       versioningScheme: OFF
 
   - task: DotNetCoreCLI@2
@@ -32,7 +46,7 @@ jobs:
       command: custom
       projects: 'Microsoft.Azure.Cosmos\src\Microsoft.Azure.Cosmos.csproj'
       custom: pack
-      arguments: '-v detailed -c ${{ parameters.BuildConfiguration }} --no-build --no-restore -o "${{ parameters.OutputPath }}"'
+      arguments: '-v detailed -c ${{ parameters.BuildConfiguration }} --no-build ${{ parameters.Arguments }} --no-restore -o "${{ parameters.OutputPath }}"'
 
   - ${{ if eq(parameters.ReleasePackage, true) }}:
     - task: DotNetCoreCLI@2
@@ -41,26 +55,25 @@ jobs:
         command: custom
         projects: 'Microsoft.Azure.Cosmos\src\Microsoft.Azure.Cosmos.csproj'
         custom: pack
-        arguments: '-v detailed -c ${{ parameters.BuildConfiguration }} --no-build --include-symbols /p:SymbolPackageFormat=snupkg --no-restore -o "${{ parameters.OutputPath }}"'
+        arguments: '-v detailed -c ${{ parameters.BuildConfiguration }} --no-build --include-symbols /p:SymbolPackageFormat=snupkg ${{ parameters.Arguments }} --no-restore -o "${{ parameters.OutputPath }}"'
     - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
       inputs:
         BuildDropPath: '$(Build.ArtifactStagingDirectory)/bin/AnyCPU/$(BuildConfiguration)/Microsoft.Azure.Cosmos'
 
-    - task: AzureFileCopy@2
-      displayName: ' Copy Artifacts to Azure SDK Release blob storage'
-      condition: and(succeeded(),ne(${{ parameters.BlobVersion }}, ''))
-      inputs:
-        SourcePath: '$(Build.ArtifactStagingDirectory)/bin/AnyCPU/$(BuildConfiguration)/Microsoft.Azure.Cosmos'
-        azureSubscription: azuresdkpartnerdrops
-        Destination: AzureBlob
-        storage: azuresdkpartnerdrops
-        ContainerName: 'drops'
-        BlobPrefix: 'cosmosdb/csharp/$(BlobVersion)'
-
-    - task: PublishBuildArtifacts@1
-      displayName: 'Publish Artifacts: Microsoft.Azure.Cosmos'
-      inputs:
-        artifactName: Microsoft.Azure.Cosmos
+    - ${{ if ne(parameters.BlobVersion, '') }}:
+      - task: AzureFileCopy@5
+        displayName: 'Copy Artifacts to Azure SDK Release blob storage'
+        condition: succeeded()
+        inputs:
+          SourcePath: '$(Build.ArtifactStagingDirectory)/bin/AnyCPU/$(BuildConfiguration)/Microsoft.Azure.Cosmos/**'
+          azureSubscription: azuresdkpartnerdrops
+          Destination: AzureBlob
+          storage: azuresdkpartnerdrops
+          ContainerName: 'drops'
+          BlobPrefix: 'cosmosdb/csharp/${{ parameters.BlobVersion }}'
+          CleanTargetBeforeCopy: ${{ parameters.CleanupFolder }}
 
-    - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
-      displayName: 'Component Detection'
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifacts: Microsoft.Azure.Cosmos'
+    inputs:
+      artifactName: Microsoft.Azure.Cosmos