changes to implement Resource concept

Signed-off-by: Nader Ziada <nziada@pivotal.io>

README.md

@@ -26,6 +26,7 @@ The CRDs involved are:
 * [PipelineParams](#pipelineparams)
 * [TaskRun](#taskrun)
 * [PipelineRun](#pipelinerun)
+* [Resources](#resources)
 
 High level details of this design:
 
@@ -37,6 +38,7 @@ High level details of this design:
   easily is powerful (e.g. see failures easily, dig into logs, e.g. like
   [the Jenkins test analyzer plugin](https://wiki.jenkins.io/display/JENKINS/Test+Results+Analyzer+Plugin))
 * [Tasks](#tasks) can depend on artifacts, output and parameters created by other tasks.
+* [Resources](#resources) are the artifacts used as inputs and outputs of TaskRuns.
 
 ## Task
 
@@ -53,9 +55,7 @@ with additional input types and clearly defined outputs.
 
 `Pipeline` describes a graph of [Tasks](#task) to execute. It defines the DAG
 and expresses how all inputs (including [PipelineParams](#pipelineparams) and outputs
-from previous `Tasks`) feed into each `Task`. It allows for fan in and fan out, and
-ordering can be expressed explicitly using `prev` and `next`, or it can be inferred
-from a `Task’s` inputs.
+from previous `Tasks`) feed into each `Task`.
 
 Dependencies between parameters or inputs/outputs are expressed as references to k8s objects.
 
@@ -66,9 +66,7 @@ can be invoked with many different instances of `PipelineParams`, which can allo
 for scenarios such as running against PRs and against a user’s personal setup.
 `PipelineParams` can control:
 
-* What **sources** the `Pipeline` runs against
 * Which **serviceAccount** to use (provided to all tasks)
-* What **artifact** stores are used (e.g. Docker registries)
 * Where **results** are stored (e.g. in GCS)
 
 ## TaskRun
@@ -125,3 +123,16 @@ completes (or fails).
 When the `PipelineRun` has completed, the `taskRuns` field will contain
 references to all `TaskRuns` which were executed and their next and
 previous `TaskRuns`.
+
+### Resources
+
+`Resources` in a pipelines are the set of objects that are going to be used 
+as inputs and outputs of a `TaskRun`. 
+
+* `Resources` should be created directly by a user in pipeline configuration 
+and bound to `TaskRun` as an input source. Also the output artifact is bound 
+to another output `Resource`.
+* The (optional) `passedConstraint` key on an `input source` defines a set of previous task names.
+* When the `passedConstraint` key is specified on an input source, only the version of 
+the resouces the passed through the defined list of tasks is used.
+* The `passedConstraint` allows for `Tasks` to fan in and fan out, and ordering can be expressed explicitly using this key.

examples/pipelineparams.yaml

@@ -5,31 +5,6 @@ metadata:
   namespace: default
 spec:
     serviceAccount: 'demoServiceAccount'
-    sources:
-        - name: 'guestbook'
-          type: 'github'
-          url: 'github.com/kubernetes/examples'
-          serviceAccount: 'githubServiceAccount'
-          branch: 'master'
-          commit: 'HEAD'
-        - name: 'redis-docker'
-          type: 'github'
-          url: 'github.com/GoogleCloudPlatform/redis-docker/blob/master/4/debian9/4.0/Dockerfile'
-          serviceAccount: 'githubServiceAccount'
-          branch: 'master'
-          commit: 'HEAD'
-        - name: 'kritis'
-          type: 'github'
-          url:  'github.com/grafeas/kritis'
-          branch: 'master'
-          commit: 'HEAD'
-    artifactStores:
-        - name: 'prodRegistry'
-          type: 'imageRegistry'
-          url: 'gcr.io/demo'
-        - name: 'stagingRegistry'
-          type: 'imageRegistry'
-          url: 'gcr.io/demo-staging'
     clusters:
         - name: 'testCluster'
           type: 'gke'

examples/pipelines/guestbook.yaml

@@ -8,10 +8,10 @@ spec:
         - name: build-guestbook                  # 1.a Build guestbook go sample code.
           taskRef:
               name: build-push
-          sourceBindings:
+          inputSourceBindings:
               - inputName: workspace
                 sourceKey: guestbook
-          artifactStoreBindings:
+          outputSourceBindings:
               - storeName: registry
                 storeKey: stagingRegistry
                 builtImage: gb-frontend          # TODO Add Commit SHA
@@ -21,63 +21,82 @@ spec:
         - name: build-redis                      # 1.b Build and push redis docker image.
           taskRef:
               name: build-push
-          sourceBindings:
+          inputSourceBindings:
               - inputName: workspace
                 sourceKey: redis-docker          # TODO Add Commit SHA
-          artifactStoreBindings:
+          outputSourceBindings:
               - storeName: registry
                 storeKey: stagingRegistry
-                builtImage: redis4
+                builtImage: redis
           params:
               - name: pathToDockerfile
                 value: 4/debian9/4.0/Dockerfile
         - name: deploy-bundle-test               # 2. Deploy GuestBook and Redis to test cluster
           taskRef:
               name: deploy-with-kubectl
-          sourceBindings:
-              - inputName: workspace
-                sourceKey: guestbook
+          inputSourceBindings:
+            - inputName: workspace
+               sourceKey: guestbook
+               passedConstraint:
+               - build-guestbook
+            - inputName: workspace
+              sourceKey: redis-docker   
+              passedConstraint:
+               - build-redis
           params:
               - name: pathToFiles
                 value: guestbook/all-in-one/guestbook-all-in-one.yaml
           clusterBindings:
               - clusterName: test
-          prevTasks:
-              -  build-redis4
-              -  build-guestbook
         - name: int-test-osx                    # 3.a Run Integration tests for osx
           taskRef:
               name: integrationTestInDocker
-          sourceBindings:
+          inputSourceBindings:
               - inputName: workspace
                 sourceKey: guestbook
+                passedConstraint:
+                - deploy-bundle-test
           params:
               - name: dockerBuildFile
                 value: guestbook-int/Dockerfile
-          prevTasks:
-              - deploy-test
         - name: int-test-linux                  # 3.b Run Integration tests for linux
           taskRef:
               name: integration-test-in-docker
-          sourceBindings:
+          inputSourceBindings:
               - inputName: workspace
                 sourceKey: guestbook
+                passedConstraint:
+                - deploy-bundle-test
           params:
               - name: dockerBuildFile
                 value: guestbook-int/Dockerfile
-          prevTasks:
-              - deploy-test
         - name: deploy-bundle-staging           # 4. Deploy GuestBook and Redis to staging cluster
           taskRef:
               name: deploy-with-kubectl
-          sourceBindings:
+          inputSourceBindings:
               - inputName: workspace
                 sourceKey: guestbook
+                passedConstraint:
+                - int-test-osx
+                - int-test-linux
           params:
               - name: pathToFiles
                 value: guestbook/all-in-one/guestbook-all-in-one.yaml
           clusterBindings:
               - clusterName: staging
-          prevTasks:
-              -  int-test-osx
-              -  int-test-linux
+    resources:
+    - name: guestbook
+      type: github
+      url: github.com/kubernetes/examples
+      serviceAccount: githubServiceAccount
+      branch: master
+      commit: HEAD
+    - name: redis-docker
+      type: github
+      url: github.com/GoogleCloudPlatform/redis-docker/blob/master/4/debian9/4.0/Dockerfile
+      serviceAccount: githubServiceAccount'
+      branch: master
+      commit: HEAD
+    - name: stagingRegistry
+      type: imageRegistry
+      url: gcr.io/demo-staging

examples/pipelines/kritis.yaml

@@ -10,7 +10,7 @@ spec:
               name: make
           inputSourceBindings:
               - inputName: workspace
-                sourceKey: kritis
+                sourceKey: kritis-app-github
           params:
               - name: makeTarget
                 value: test
@@ -19,20 +19,18 @@ spec:
               name: build-push
           inputSourceBindings:
               - name: workspace
-                sourceKey: kritis
+                sourceKey: kritis-app-github
+                passedConstraints: [unit-test-kritis]
           outputSourceBindings:
               - name: registry
                 sourceKey: stagingRegistry
           params:
               - name: pathToDockerfile
                 value: deploy/Dockerfile
-          prevTasks: ['unit-test-kritis']
         - name: deploy-test-env           # 3. Finally Deploy to Test environment
           taskRef:
               name: deploy-with-helm
           inputSourceBindings:
-              - name: workspace
-                sourceKey: kritis
               - name: registry
                 sourceKey: stagingRegistry
                 passedConstraints: [push-kritis]
@@ -41,24 +39,31 @@ spec:
                 value: kritis-charts
           clusterBindings:
               - clusterName: test
-          nextTasks: ['integration-test']
         - name: integration-test          # 4. Run Integration Tests in test cluster
           taskRef:
               name: integration-test-in-docker
           inputSourceBindings:
               - name: workspace
-                sourceKey: kritis
+                sourceKey: kritis-test-github
+                passedConstraint: [deploy-test-env]
           params:
               - name: testArgs
                 value: "-e REMOTE_INTEGRATION=true"
-    sources:
-      - name: kritis
+    resources:
+      - name: kritis-app-github
         type: git
         params:
           - name: url
             value: https://github.com/grafeas/kritis
           - name: branch
             value: master
+      - name: kritis-test-github
+        type: git
+        params:
+          - name: url
+            value: https://github.com/grafeas/kritis-test
+          - name: branch
+            value: master
       - name: stagingRegistry
         type: image
         params:

pkg/apis/pipeline/v1beta1/git_source.go

@@ -16,30 +16,29 @@ limitations under the License.
 
 package v1beta1
 
-// GitSource is an endpoint from which to get data which is required
+// GitResource is an endpoint from which to get data which is required
 // by a Build/Task for context (e.g. a repo from which to build an image).
-type GitSource struct {
-	Name           string `json:"name"`
-	Type           string `json:"type"`
-	URL            string `json:"url"`
-	Branch         string `json:"branch"`
-	Commit         string `json:"commit,omitempty"`
+type GitResource struct {
+	Name string `json:"name"`
+	Type string `json:"type"`
+	URL  string `json:"url"`
+	// Git revision (branch, tag, commit SHA or ref) to clone.  See
+	// https://git-scm.com/docs/gitrevisions#_specifying_revisions for more
+	// information.
+	Revision       string `json:"revision"`
 	ServiceAccount string `json:"serviceAccount,omitempty"`
 }
 
-func (s GitSource) getName() string {
+func (s GitResource) getName() string {
 	return s.Name
 }
 
-func (s GitSource) getType() string {
-	return "git"
+func (s GitResource) getType() ResourceType {
+	return ResourceTypeGit
 }
 
-func (s GitSource) getVersion() string {
-	return s.Commit
+func (s GitResource) getVersion() string {
+	return s.Revision
 }
 
-func (s GitSource) getParams() []Param {
-	var result []Param
-	return result
-}
+func (s GitResource) getParams() []Param { return []Param{} }

pkg/apis/pipeline/v1beta1/image_source.go

@@ -16,28 +16,25 @@ limitations under the License.
 
 package v1beta1
 
-// ImageSource defines an endpoint where artifacts can be stored, such as images.
-type ImageSource struct {
+// ImageResource defines an endpoint where artifacts can be stored, such as images.
+type ImageResource struct {
 	Name string `json:"name"`
 	// TODO: maybe an enum, with values like 'registry', GCS bucket
-	Type string `json:"type"`
-	URL  string `json:"url"`
-	Sha  string `json:"sha"`
+	Type   string `json:"type"`
+	URL    string `json:"url"`
+	Digest string `json:"digest"`
 }
 
-func (s ImageSource) getName() string {
+func (s ImageResource) getName() string {
 	return s.Name
 }
 
-func (s ImageSource) getType() string {
-	return "image"
+func (s ImageResource) getType() ResourceType {
+	return ResourceTypeImage
 }
 
-func (s ImageSource) getVersion() string {
-	return s.Sha
+func (s ImageResource) getVersion() string {
+	return s.Digest
 }
 
-func (s ImageSource) getParams() []Param {
-	var result []Param
-	return result
-}
+func (s ImageResource) getParams() []Param { return []Param{} }

pkg/apis/pipeline/v1beta1/pipeline_types.go

@@ -97,14 +97,14 @@ type TaskRef struct {
 
 // PipelineSource
 type PipelineSource struct {
-	Name      string    `json:"name"`
-	Type      string    `json:"type"`
-	SourceRef SourceRef `json:"sourceRef"`
-	Params    []Param   `json:"params,omitempty"`
+	Name        string      `json:"name"`
+	Type        string      `json:"type"`
+	ResourceRef ResourceRef `json:"resourceRef"`
+	Params      []Param     `json:"params,omitempty"`
 }
 
-// SourceRef can be used to refer to a specific instance of a Source.
-type SourceRef struct {
+// ResourceRef can be used to refer to a specific instance of a Source.
+type ResourceRef struct {
 	// Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds"
 	Kind string `json:"kind"`
 	// Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names