[Backport release-1.6] Feat: implement pipeline APIs (#4969)

* add context when run pipeline
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 72f3ad792ef6d0f7c05a89341e02a8bf2a33ee26)

* Feat: implement pipeline API
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit f560c346cc4424a1c782615a42da85cb81da5663)

* Extract get log logic and implement getPipelineRunLog API
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 060c6ab9e96a01d8e1230de5d25c1f5215cd96f8)

* Init and delete pipeline contexts
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 5e96bd31066960160977397decbbd12e08bbb1b1)

* fix panic
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 51072f7947f86594b4c32854c20e2c9580a70a21)

* Allow not specifying context
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 810ddcf0bdd98dfceee8aa6f86c283da7db9f3b0)

* change pipeline to path parameter
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 3d51c0cb2db9ca364884de539bcfc3fa27553e26)

* Add permission check filter
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 688376743040f9903bdce9af66949f6424946089)

* project -> projects in route
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 1f09f3996bf6953dc07f2b778314a238ec516836)

* fix route conflict
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 7eea69683048639ebd5d1efd362adf05d3c80bf8)

* Add project alias
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit b07dd72338f376c53522bed5d58c331449659f1a)

* Feat: change the list pipeline API
Signed-off-by: barnettZQG <barnett.zqg@gmail.com>
(cherry picked from commit bd804734b0bfa71ed59c26613c9ecb2094a0a3d5)

* Feat: filter the project
Signed-off-by: barnettZQG <barnett.zqg@gmail.com>
(cherry picked from commit 82eee2cc113862bcd6db589875dfdc99d7e2bf69)

* Fix: the error of the run APi
Signed-off-by: barnettZQG <barnett.zqg@gmail.com>
(cherry picked from commit ac87bd3f1a2eeaf5caafe6cfb8d7e7e3992d2a2b)

* fix log pipeline run API
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit acde8e981e9003a81ffa91f9e72776f29e275654)

* Fix lint, fix the error of log api
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit b8373e6cde596557976fc1d26feca4badd8fd264)

* fix error returning
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 2e9b4792b06cac89e6a42f6d64c80eb09a30431d)

* Fix: change the lable to annotation
Signed-off-by: barnettZQG <barnett.zqg@gmail.com>
(cherry picked from commit bf08275fde2f1aa7dd7c0a80ebb2ec74ced61617)

* remove log config not found error
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit cdd77dfd8fa5f32e3622c139fd51cead550742ed)

* fix pipeline list api return no context info
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit cdcfa165d16d355d0a6210357e4f13e678c620d9)

* Fix: create the namespace
Signed-off-by: barnettZQG <barnett.zqg@gmail.com>
(cherry picked from commit b6888dd87d7c2032c448ef77f570a5db57507409)

* get pipeline lastrun info
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit a943423d22c7bbbe3cfee1d98a7a4516258f6895)

* allow query single step output
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit e2310bbf34f20063917b6bba7999aa2328bc4a5c)

* organize code in api layer
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 6fd53ed078e6d3dd965e86e32c41abea50c50a44)

* fix project filter, add context value when get pp run, extend lastRun
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 861f69d555fc443a153c7e14c28897f40773b8b4)

* fix get output and implement get input api
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 61495ee70dbcd928f4749807adc635fad3b1f857)

* Fix: change the last run
Signed-off-by: barnettZQG <barnett.zqg@gmail.com>
(cherry picked from commit aeb842a45ea7c8a81aa75770a5e54ec1eb5cdaed)

* if query sub-step outout, return it directly
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 467ba257511662d3a688b2a27489dd7939e151ef)

* Fix: change the run stats
Signed-off-by: barnettZQG <barnett.zqg@gmail.com>
(cherry picked from commit 7a90e7e310181f5ec4ca60be1f7f4cb14b4e0a68)

* Fix: change the output
Signed-off-by: barnettZQG <barnett.zqg@gmail.com>
(cherry picked from commit 595a871b0d3aa39cf939cee1ffe6dbefa78dbf57)

* flatten the input/output api
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit efc9692354c21408c61f26e55d07a223778efa1e)

* more info for i/o vars
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 7fe0e1109cd0544d19cf432e885fab579490d9bf)

* fix nested i/o struct
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 201d1228bdc9e74ac85909f9fd05ce7cb5ed6441)

* add fromStep in input api
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 240001896289f7d9261bd590b17b9c6e75801cbb)

* add e2e test skeleton
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit f20f9a1ac67bbdc078a8a941809e207014d5c145)

* add more e2e test
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 98b27f886b0db09509381f12b76948f74567dcaa)

* use db to store pipeline
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 08962c4f2f61ea09e3cb9a40f8f92a723e5e7309)

* keep the last 5k lines of log
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 22b352da146b42465c790fcb927889f10f0cad0d)

* use stern param to keep last lines of logs
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 3eadbf91c818092d02948bbe4798360121e6079e)

* filter, nil labels, spec check
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit bad90b3f7a343647b39af2a8dd6dd8b101562dab)

* empty res, index, detail param
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit b12d889e978b4f0936a83c76010701c94e314323)

* Add e2e test
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit ae617a928ef0250708f478b944675dcce078c1ff)

* fix e2e test and unit test
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 0f4e030b84e9bb08697277cafb1116fcf499af72)

* add context e2e test
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 88879e6e43f986cb59d4430903305230d30f27d5)

* goimports
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 58429978b3c79983ec5baf3851c2bcac9595a4b2)

* add more test
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 75760521a460d881d206739fb1c33ccd5884f50c)

* review
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit cc4c7064667d9a7d5cbe3d36a0946cb0413edc77)

* remove optional tag in returned value, unify the imports name
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 658b184aef656cf4d611fddf09028b5a3e45d070)

* fix e2e test
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit a9e9c96856892b7394f77eeb6cb19fe7cb3552e3)

* add stop test
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 58aa2e5125b44fb26a3fe25d36d5a1f1859df480)

* more coverage
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 01ecb513233ac6015e09182abef00f92e2a491fe)

* single case selct
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit d9e8fd03426c811d741dff5567642f5d42f0ad3f)

* optimize log color
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit be9840c3cb0d4893708a40b778a500d90f941629)

* add default permission and role
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit cf074444ac51f7dd13131360f3f8dd695307927e)

* fix permission ut
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 1bdcad63a2128f6ace72825182d600451ef8cf99)

* change the log api implementation
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 587f745430d4f6d6655b7dc84f71a255a147ff49)

* add color, add container order
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 6e7f18760585bb5f9d07551ef0e00d8fc21c03b6)

* lint
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 64ba029031c6d8b1e13dea16544d561740e8f976)

* fix filter nil will cut all log
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 799dfe377ad8ba3b4868c93ea5b98427c2be5fed)

* longer timeout and lint
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
(cherry picked from commit 89873f1f66d167507da710c394b79f3f542b60ff)
Co-authored-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
Co-authored-by: barnettZQG <barnett.zqg@gmail.com>

.github/workflows/apiserver-test.yaml

@@ -176,10 +176,12 @@ jobs:
           make e2e-cleanup
           make e2e-setup-core
           bin/vela addon enable fluxcd
+          bin/vela addon enable vela-workflow
           timeout 600s bash -c -- 'while true; do kubectl get ns flux-system; if [ $? -eq 0 ] ; then break; else sleep 5; fi;done'
           kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=vela-core,app.kubernetes.io/instance=kubevela -n vela-system --timeout=600s
           kubectl wait --for=condition=Ready pod -l app=source-controller -n flux-system --timeout=600s
           kubectl wait --for=condition=Ready pod -l app=helm-controller -n flux-system --timeout=600s
+          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=vela-workflow -n vela-system --timeout=600s
 
       - name: Run api server e2e test
         run: |

e2e/addon/mock/testrepo/helm-repo/index.yaml

@@ -23,4 +23,15 @@ entries:
       annotations:
         system.vela: ">=1.5.0"
         system.kubernetes: ">=1.30.0"
+  vela-workflow:
+    - annotations:
+        system.vela: '>=v1.6.0-beta.1'
+      created: "2022-10-29T09:11:16.865230605Z"
+      description: vela-workflow provides the capability to run a standalone workflow
+      home: https://github.com/kubevela/workflow
+      icon: https://static.kubevela.net/images/logos/KubeVela%20-03.png
+      name: vela-workflow
+      urls:
+        - http://127.0.0.1:9098/helm/vela-workflow-v0.3.1.tgz
+      version: v0.3.1
 generated: "2022-06-15T13:17:04.733573+08:00"
\ No newline at end of file

e2e/addon/mock/vela_addon_mock_server.go

@@ -22,9 +22,9 @@ import (
 	"fmt"
 	"html/template"
 	"io/fs"
-	"io/ioutil"
 	"log"
 	"net/http"
+	"os"
 	"path"
 	"strings"
 
@@ -108,24 +108,31 @@ var ossHandler http.HandlerFunc = func(rw http.ResponseWriter, req *http.Request
 var helmHandler http.HandlerFunc = func(rw http.ResponseWriter, req *http.Request) {
 	switch {
 	case strings.Contains(req.URL.Path, "index.yaml"):
-		file, err := ioutil.ReadFile("./e2e/addon/mock/testrepo/helm-repo/index.yaml")
+		file, err := os.ReadFile("./e2e/addon/mock/testrepo/helm-repo/index.yaml")
 		if err != nil {
 			_, _ = rw.Write([]byte(err.Error()))
 		}
 		rw.Write(file)
 	case strings.Contains(req.URL.Path, "fluxcd-test-version-1.0.0.tgz"):
-		file, err := ioutil.ReadFile("./e2e/addon/mock/testrepo/helm-repo/fluxcd-test-version-1.0.0.tgz")
+		file, err := os.ReadFile("./e2e/addon/mock/testrepo/helm-repo/fluxcd-test-version-1.0.0.tgz")
 		if err != nil {
 			_, _ = rw.Write([]byte(err.Error()))
 		}
 		rw.Write(file)
 	case strings.Contains(req.URL.Path, "fluxcd-test-version-2.0.0.tgz"):
-		file, err := ioutil.ReadFile("./e2e/addon/mock/testrepo/helm-repo/fluxcd-test-version-2.0.0.tgz")
+		file, err := os.ReadFile("./e2e/addon/mock/testrepo/helm-repo/fluxcd-test-version-2.0.0.tgz")
+		if err != nil {
+			_, _ = rw.Write([]byte(err.Error()))
+		}
+		rw.Write(file)
+	case strings.Contains(req.URL.Path, "vela-workflow-v0.3.1.tgz"):
+		file, err := os.ReadFile("./e2e/addon/mock/testrepo/helm-repo/vela-workflow-v0.3.1.tgz")
 		if err != nil {
 			_, _ = rw.Write([]byte(err.Error()))
 		}
 		rw.Write(file)
 	}
+
 }
 
 func init() {

go.mod

@@ -60,6 +60,7 @@ require (
 	github.com/kubevela/workflow v0.3.1
 	github.com/kyokomi/emoji v2.2.4+incompatible
 	github.com/mitchellh/hashstructure/v2 v2.0.1
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd
 	github.com/oam-dev/cluster-gateway v1.4.0
 	github.com/oam-dev/cluster-register v1.0.4-0.20220928064144-5f76a9d7ca8c
 	github.com/oam-dev/terraform-config-inspect v0.0.0-20210418082552-fc72d929aa28
@@ -239,7 +240,6 @@ require (
 	github.com/moby/locker v1.0.1 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect

pkg/addon/addon.go

@@ -612,7 +612,7 @@ func unmarshalToContent(content []byte) (fileContent *github.RepositoryContent,
 }
 
 func genAddonAPISchema(addonRes *UIData) error {
-	cueScript := script.CUE([]byte(addonRes.Parameters))
+	cueScript := script.CUE(addonRes.Parameters)
 	schema, err := cueScript.ParsePropertiesToSchema()
 	if err != nil {
 		return err

pkg/apiserver/domain/model/context.go → pkg/apiserver/domain/model/pipeline.go

@@ -16,10 +16,52 @@ limitations under the License.
 
 package model
 
-import "fmt"
+import (
+	"fmt"
+
+	"github.com/kubevela/workflow/api/v1alpha1"
+)
 
 func init() {
 	RegisterModel(&PipelineContext{})
+	RegisterModel(&Pipeline{})
+}
+
+// Pipeline is the model of pipeline
+type Pipeline struct {
+	BaseModel
+	Spec        v1alpha1.WorkflowSpec
+	Name        string `json:"name"`
+	Project     string `json:"project"`
+	Alias       string `json:"alias"`
+	Description string `json:"description"`
+}
+
+// PrimaryKey return custom primary key
+func (p Pipeline) PrimaryKey() string {
+	return fmt.Sprintf("%s-%s", p.Project, p.Name)
+}
+
+// TableName return custom table name
+func (p Pipeline) TableName() string {
+	return tableNamePrefix + "pipeline"
+}
+
+// ShortTableName is the compressed version of table name for kubeapi storage and others
+func (p Pipeline) ShortTableName() string {
+	return "pipeline"
+}
+
+// Index return custom index
+func (p Pipeline) Index() map[string]string {
+	var index = make(map[string]string)
+	if p.Project != "" {
+		index["project"] = p.Project
+	}
+	if p.Name != "" {
+		index["name"] = p.Name
+	}
+	return index
 }
 
 // Value is a k-v pair

pkg/apiserver/domain/model/project.go

@@ -16,8 +16,6 @@ limitations under the License.
 
 package model
 
-import "fmt"
-
 func init() {
 	RegisterModel(&Project{})
 }
@@ -29,11 +27,15 @@ type Project struct {
 	Alias       string `json:"alias"`
 	Owner       string `json:"owner"`
 	Description string `json:"description,omitempty"`
+	Namespace   string `json:"namespace"`
 }
 
 // GetNamespace get the namespace name of this project.
 func (p *Project) GetNamespace() string {
-	return fmt.Sprintf("project-%s", p.Name)
+	if p.Namespace != "" {
+		return p.Namespace
+	}
+	return p.Name
 }
 
 // TableName return custom table name

pkg/apiserver/domain/service/cloudshell_test.go

@@ -66,7 +66,8 @@ var _ = Describe("Test cloudshell service function", func() {
 			ProjectService: projectService,
 		}
 		projectService = &projectServiceImpl{
-			Store: ds,
+			Store:     ds,
+			K8sClient: k8sClient,
 			RbacService: &rbacServiceImpl{
 				Store: ds,
 			},

pkg/apiserver/domain/service/helm_test.go

@@ -83,7 +83,7 @@ var _ = Describe("Test helm repo list", func() {
 		pSec = v1.Secret{}
 		gSec = v1.Secret{}
 		Expect(k8sClient.Create(ctx, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "vela-system"}})).Should(SatisfyAny(BeNil(), util.AlreadyExistMatcher{}))
-		Expect(k8sClient.Create(ctx, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "project-my-project"}})).Should(SatisfyAny(BeNil(), util.AlreadyExistMatcher{}))
+		Expect(k8sClient.Create(ctx, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "my-project"}})).Should(SatisfyAny(BeNil(), util.AlreadyExistMatcher{}))
 		Expect(yaml.Unmarshal([]byte(projectSecret), &pSec)).Should(BeNil())
 		Expect(yaml.Unmarshal([]byte(globalSecret), &gSec)).Should(BeNil())
 		Expect(k8sClient.Create(ctx, &pSec)).Should(BeNil())
@@ -390,7 +390,7 @@ apiVersion: v1
 kind: Secret
 metadata:
   name: project-helm-repo
-  namespace: project-my-project
+  namespace: my-project
   labels:
     config.oam.dev/type: helm-repository
     config.oam.dev/catalog: velacore-config

pkg/apiserver/domain/service/pipeline_test.go

+/*
+Copyright 2021 The KubeVela Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package service
+
+import (
+	"context"
+
+	"github.com/kubevela/workflow/api/v1alpha1"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/runtime"
+
+	"github.com/oam-dev/kubevela/pkg/apiserver/domain/model"
+	"github.com/oam-dev/kubevela/pkg/apiserver/infrastructure/datastore"
+	apisv1 "github.com/oam-dev/kubevela/pkg/apiserver/interfaces/api/dto/v1"
+	"github.com/oam-dev/kubevela/pkg/oam/util"
+)
+
+var (
+	// defaultNamespace   = "project-default-ns1-test"
+	pipelineService    *pipelineServiceImpl
+	pipelineRunService *pipelineRunServiceImpl
+	userService        *userServiceImpl
+	contextService     *contextServiceImpl
+	projectService     *projectServiceImpl
+	ctx                context.Context
+
+	pipelineName = "test-pipeline"
+	projectName  = "test-project"
+)
+var _ = Describe("Test pipeline service functions", func() {
+	It("Init services and project", func() {
+		ds, err := NewDatastore(datastore.Config{Type: "kubeapi", Database: "pipeline-test-kubevela"})
+		Expect(ds).ToNot(BeNil())
+		Expect(err).Should(BeNil())
+		Expect(err).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{}))
+		pipelineService = NewTestPipelineService(ds, k8sClient, cfg).(*pipelineServiceImpl)
+		pipelineRunService = pipelineService.PipelineRunService.(*pipelineRunServiceImpl)
+		contextService = pipelineService.ContextService.(*contextServiceImpl)
+		projectService = pipelineService.ProjectService.(*projectServiceImpl)
+		userService = &userServiceImpl{Store: ds, K8sClient: k8sClient}
+
+		ctx = context.WithValue(context.TODO(), &apisv1.CtxKeyUser, "admin")
+		err = userService.Init(context.TODO())
+		Expect(err).Should(BeNil())
+		_, err = projectService.CreateProject(ctx, apisv1.CreateProjectRequest{
+			Name:  projectName,
+			Owner: "admin",
+		})
+		Expect(err).Should(BeNil())
+		projModel, err := projectService.GetProject(context.TODO(), projectName)
+		Expect(err).Should(BeNil())
+		ctx = context.WithValue(ctx, &apisv1.CtxKeyProject, projModel)
+	})
+
+	It("Test create pipeline", func() {
+		rawProps := []byte(`{"url":"https://api.github.com/repos/kubevela/kubevela"}`)
+		testPipelineSteps := []v1alpha1.WorkflowStep{
+			{
+				SubSteps: []v1alpha1.WorkflowStepBase{
+					{
+						Name: "request",
+						Type: "request",
+						Outputs: v1alpha1.StepOutputs{
+							{
+								ValueFrom: "import \"strconv\"\n\"Current star count: \" + strconv.FormatInt(response[\"stargazers_count\"], 10)\n",
+								Name:      "stars",
+							},
+						},
+						Properties: &runtime.RawExtension{
+							Raw: rawProps,
+						},
+					},
+				},
+				WorkflowStepBase: v1alpha1.WorkflowStepBase{
+					Name: "step-group",
+					Type: "step-group",
+				},
+			},
+		}
+
+		By("create pipeline with sub-steps")
+		pipeline, err := pipelineService.CreatePipeline(ctx, apisv1.CreatePipelineRequest{
+			Name: pipelineName,
+			Spec: v1alpha1.WorkflowSpec{
+				Steps: testPipelineSteps,
+			},
+		})
+		Expect(err).Should(BeNil())
+		Expect(pipeline.Name).Should(Equal(pipelineName))
+		Expect(pipeline.Spec.Steps[0].Name).Should(Equal("step-group"))
+	})
+
+	It("list pipeline", func() {
+		pipelines, err := pipelineService.ListPipelines(ctx, apisv1.ListPipelineRequest{
+			Detailed: true,
+		})
+		Expect(err).Should(BeNil())
+		Expect(pipelines).ShouldNot(BeNil())
+		Expect(pipelines.Total).Should(Equal(1))
+		Expect(len(pipelines.Pipelines)).Should(Equal(1))
+		Expect(pipelines.Pipelines[0].Info).ShouldNot(BeNil())
+	})
+
+	It("get pipeline contexts", func() {
+		By("no context")
+		contexts, err := contextService.ListContexts(ctx, projectName, pipelineName)
+		Expect(err).Should(BeNil())
+		Expect(contexts.Total).Should(Equal(0))
+		Expect(len(contexts.Contexts)).Should(Equal(0))
+
+		By("create context")
+		contextName := "test-context"
+		contextKey := "test-key"
+		contextVal := "test-val"
+		ppCtx := apisv1.Context{
+			Name: contextName,
+			Values: []model.Value{
+				{
+					Key:   contextKey,
+					Value: contextVal,
+				},
+			},
+		}
+		context, err := contextService.CreateContext(ctx, projectName, pipelineName, ppCtx)
+		Expect(err).Should(BeNil())
+		Expect(len(context.Contexts)).Should(Equal(1))
+	})
+})

pkg/apiserver/domain/service/project.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 
 	terraformapi "github.com/oam-dev/terraform-controller/api/v1beta1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/oam-dev/kubevela/apis/types"
@@ -31,6 +32,7 @@ import (
 	"github.com/oam-dev/kubevela/pkg/apiserver/utils/bcode"
 	"github.com/oam-dev/kubevela/pkg/apiserver/utils/log"
 	"github.com/oam-dev/kubevela/pkg/multicluster"
+	"github.com/oam-dev/kubevela/pkg/utils"
 )
 
 // ProjectService project manage service.
@@ -159,6 +161,13 @@ func (p *projectServiceImpl) GetProject(ctx context.Context, projectName string)
 		}
 		return nil, err
 	}
+	if _, err := utils.GetNamespace(ctx, p.K8sClient, project.GetNamespace()); err != nil {
+		if apierrors.IsNotFound(err) {
+			if err := utils.CreateNamespace(ctx, p.K8sClient, projectName); err != nil && !apierrors.IsAlreadyExists(err) {
+				return nil, bcode.ErrProjectNamespaceFail
+			}
+		}
+	}
 	return project, nil
 }
 
@@ -319,11 +328,16 @@ func (p *projectServiceImpl) CreateProject(ctx context.Context, req apisv1.Creat
 		}
 	}
 
+	if err := utils.CreateNamespace(ctx, p.K8sClient, req.Name); err != nil && !apierrors.IsAlreadyExists(err) {
+		return nil, bcode.ErrProjectNamespaceFail
+	}
+
 	newProject := &model.Project{
 		Name:        req.Name,
 		Description: req.Description,
 		Alias:       req.Alias,
 		Owner:       owner,
+		Namespace:   req.Name,
 	}
 
 	if err := p.Store.Add(ctx, newProject); err != nil {
@@ -526,6 +540,7 @@ func ConvertProjectModel2Base(project *model.Project, owner *model.User) *apisv1
 		CreateTime:  project.CreateTime,
 		UpdateTime:  project.UpdateTime,
 		Owner:       apisv1.NameAlias{Name: project.Owner},
+		Namespace:   project.GetNamespace(),
 	}
 	if owner != nil && owner.Name == project.Owner {
 		base.Owner = apisv1.NameAlias{Name: owner.Name, Alias: owner.Alias}

pkg/apiserver/domain/service/rbac.go

@@ -59,6 +59,7 @@ var defaultProjectPermissionTemplate = []*model.PermissionTemplate{
 			"project:{projectName}/permission:*",
 			"project:{projectName}/environment:*",
 			"project:{projectName}/application:*/*",
+			"project:{projectName}/pipeline:*/*",
 		},
 		Actions: []string{"detail", "list"},
 		Effect:  "Allow",
@@ -96,6 +97,16 @@ var defaultProjectPermissionTemplate = []*model.PermissionTemplate{
 		Effect:    "Allow",
 		Scope:     "project",
 	},
+	{
+		Name:  "pipeline-management",
+		Alias: "Pipeline Management",
+		Resources: []string{
+			"project:{projectName}/pipeline:*",
+		},
+		Actions: []string{"*"},
+		Effect:  "Allow",
+		Scope:   "project",
+	},
 }
 
 var defaultPlatformPermission = []*model.PermissionTemplate{
@@ -234,6 +245,17 @@ var ResourceMaps = map[string]resourceMetadata{
 				pathName: "configName",
 			},
 			"provider": {},
+			"pipeline": {
+				pathName: "pipelineName",
+				subResources: map[string]resourceMetadata{
+					"context": {
+						pathName: "contextName",
+					},
+					"pipelineRun": {
+						pathName: "pipelineRunName",
+					},
+				},
+			},
 		},
 		pathName: "projectName",
 	},
@@ -866,7 +888,7 @@ func (p *rbacServiceImpl) InitDefaultRoleAndUsersForProject(ctx context.Context,
 	}, &model.Role{
 		Name:        "project-admin",
 		Alias:       "Project Admin",
-		Permissions: []string{"project-view", "app-management", "env-management", "role-management", "configuration-read"},
+		Permissions: []string{"project-view", "app-management", "env-management", "role-management", "pipeline-management", "configuration-read"},
 		Project:     project.Name,
 	}, &model.Role{
 		Name:        "project-viewer",

pkg/apiserver/domain/service/rbac_test.go

@@ -198,7 +198,7 @@ var _ = Describe("Test rbac service", func() {
 
 		policies, err := rbacService.ListPermissions(context.TODO(), "init-test")
 		Expect(err).Should(BeNil())
-		Expect(len(policies)).Should(BeEquivalentTo(int64(5)))
+		Expect(len(policies)).Should(BeEquivalentTo(int64(6)))
 	})
 
 	It("Test UpdatePermission", func() {

pkg/apiserver/domain/service/suite_test.go

@@ -25,6 +25,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/kubevela/workflow/api/v1alpha1"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
@@ -74,7 +75,10 @@ var _ = BeforeSuite(func(done Done) {
 
 	By("new kube client")
 	cfg.Timeout = time.Minute * 2
-	k8sClient, err = client.New(cfg, client.Options{Scheme: common.Scheme})
+	scheme := common.Scheme
+	err = v1alpha1.AddToScheme(scheme)
+	Expect(err).ShouldNot(HaveOccurred())
+	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme})
 	Expect(err).Should(BeNil())
 	Expect(k8sClient).ToNot(BeNil())
 	By("new kube client success")

pkg/apiserver/domain/service/workflow_test.go

@@ -58,7 +58,7 @@ var _ = Describe("Test workflow service functions", func() {
 		Expect(ds).ToNot(BeNil())
 		Expect(err).Should(BeNil())
 		rbacService := &rbacServiceImpl{Store: ds}
-		projectService = &projectServiceImpl{Store: ds, RbacService: rbacService}
+		projectService = &projectServiceImpl{Store: ds, RbacService: rbacService, K8sClient: k8sClient}
 		envService = &envServiceImpl{Store: ds, KubeClient: k8sClient, ProjectService: projectService}
 		envBinding = &envBindingServiceImpl{
 			Store:           ds,

pkg/apiserver/infrastructure/clients/kubeclient.go

@@ -20,13 +20,13 @@ import (
 	"fmt"
 
 	pkgmulticluster "github.com/kubevela/pkg/multicluster"
+	"github.com/kubevela/workflow/api/v1alpha1"
+	"github.com/kubevela/workflow/pkg/cue/packages"
 	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 
-	"github.com/kubevela/workflow/pkg/cue/packages"
-
 	apiConfig "github.com/oam-dev/kubevela/pkg/apiserver/config"
 	"github.com/oam-dev/kubevela/pkg/auth"
 	"github.com/oam-dev/kubevela/pkg/oam/discoverymapper"
@@ -73,11 +73,13 @@ func GetKubeClient() (client.Client, error) {
 	if kubeConfig == nil {
 		return nil, fmt.Errorf("please call SetKubeConfig first")
 	}
-	var err error
-	kubeClient, err = pkgmulticluster.NewClient(kubeConfig, pkgmulticluster.ClientOptions{
+	err := v1alpha1.AddToScheme(common.Scheme)
+	if err != nil {
+		return nil, err
+	}
+	return pkgmulticluster.NewClient(kubeConfig, pkgmulticluster.ClientOptions{
 		Options: client.Options{Scheme: common.Scheme},
 	})
-	return kubeClient, err
 }
 
 // GetKubeConfig create/get kube runtime config

pkg/apiserver/interfaces/api/dto/v1/types.go

@@ -50,12 +50,14 @@ var (
 	CtxKeyApplicationComponent = "component"
 	// CtxKeyUser request context key of user
 	CtxKeyUser = "user"
+	// CtxKeyProject request context key of project
+	CtxKeyProject = "project"
 	// CtxKeyToken request context key of request token
 	CtxKeyToken = "token"
 	// CtxKeyPipeline request context key of pipeline
 	CtxKeyPipeline = "pipeline"
-	// CtxKeyPipelineContex request context key of pipeline context
-	CtxKeyPipelineContex = "pipeline-context"
+	// CtxKeyPipelineContext request context key of pipeline context
+	CtxKeyPipelineContext = "pipeline-context"
 	// CtxKeyPipelineRun request context key of pipeline run
 	CtxKeyPipelineRun = "pipeline-run"
 )
@@ -791,6 +793,7 @@ type ProjectBase struct {
 	CreateTime  time.Time `json:"createTime"`
 	UpdateTime  time.Time `json:"updateTime"`
 	Owner       NameAlias `json:"owner,omitempty"`
+	Namespace   string    `json:"-"`
 }
 
 // CreateProjectRequest create project request body
@@ -1551,10 +1554,11 @@ type ListConfigDistributionResponse struct {
 
 // PipelineMeta is metadata of pipeline
 type PipelineMeta struct {
-	Name        string `json:"name" validate:"checkname"`
-	Alias       string `json:"alias" validate:"checkalias" optional:"true"`
-	Project     string `json:"project"`
-	Description string `json:"description" optional:"true"`
+	Name        string    `json:"name"`
+	Alias       string    `json:"alias"`
+	Project     NameAlias `json:"project"`
+	Description string    `json:"description"`
+	CreateTime  time.Time `json:"createTime"`
 }
 
 // PipelineBase is the base info of pipeline
@@ -1580,7 +1584,6 @@ type RunStat struct {
 // CreatePipelineRequest is the request body of creating pipeline
 type CreatePipelineRequest struct {
 	Name        string                        `json:"name" validate:"checkname"`
-	Project     string                        `json:"project"`
 	Alias       string                        `json:"alias" validate:"checkalias" optional:"true"`
 	Description string                        `json:"description" optional:"true"`
 	Spec        workflowv1alpha1.WorkflowSpec `json:"spec"`
@@ -1593,8 +1596,9 @@ type PipelineMetaResponse struct {
 
 // ListPipelineRequest is the request body of listing pipeline
 type ListPipelineRequest struct {
-	Projects []string `json:"projects"`
-	Query    string   `json:"query"`
+	Projects []string `json:"projects" optional:"true"`
+	Query    string   `json:"query" optional:"true"`
+	Detailed bool     `json:"detailed" optional:"true"`
 }
 
 // ListPipelineResponse is the response body of listing pipeline
@@ -1616,11 +1620,6 @@ type UpdatePipelineRequest struct {
 	Spec        workflowv1alpha1.WorkflowSpec `json:"spec" optional:"true"`
 }
 
-// GetPipelineRequest is the request body of getting pipeline
-type GetPipelineRequest struct {
-	Detailed bool `json:"detailed"`
-}
-
 // GetPipelineResponse is the response body of getting pipeline
 type GetPipelineResponse struct {
 	PipelineBase `json:",inline"`
@@ -1629,9 +1628,8 @@ type GetPipelineResponse struct {
 
 // PipelineInfo is the info of pipeline
 type PipelineInfo struct {
-	RelatedApps   []ApplicationBase                  `json:"relatedApps"`
-	LastRunStatus workflowv1alpha1.WorkflowRunStatus `json:"lastRunStatus"`
-	RunStat       RunStat                            `json:"runStat"`
+	LastRun *PipelineRun `json:"lastRun"`
+	RunStat RunStat      `json:"runStat"`
 }
 
 /***********************/
@@ -1652,9 +1650,9 @@ type PipelineRunBriefing struct {
 
 // PipelineRunMeta is the metadata of pipeline run
 type PipelineRunMeta struct {
-	PipelineName    string `json:"pipelineName"`
-	Project         string `json:"project"`
-	PipelineRunName string `json:"pipelineRunName"`
+	PipelineName    string    `json:"pipelineName"`
+	Project         NameAlias `json:"project"`
+	PipelineRunName string    `json:"pipelineRunName"`
 }
 
 // PipelineRun is the info of pipeline run
@@ -1667,14 +1665,16 @@ type PipelineRun struct {
 type PipelineRunBase struct {
 	PipelineRunMeta `json:",inline"`
 	// Record marks the run of the pipeline
-	Record      int64                            `json:"record"`
-	ContextName string                           `json:"contextName"`
-	Spec        workflowv1alpha1.WorkflowRunSpec `json:"spec"`
+	Record        int64                            `json:"record"`
+	ContextName   string                           `json:"contextName"`
+	ContextValues []model.Value                    `json:"contextValues"`
+	Spec          workflowv1alpha1.WorkflowRunSpec `json:"spec"`
 }
 
 // RunPipelineRequest is the request body of running pipeline
 type RunPipelineRequest struct {
 	// Mode is the mode of the pipeline run. Available values are: "StepByStep", "DAG" for both `step` and `subStep`
+	// default: "StepByStep" for `step`, "DAG" for `subStep`
 	Mode        workflowv1alpha1.WorkflowExecuteMode `json:"mode" optional:"true"`
 	ContextName string                               `json:"contextName"`
 }
@@ -1687,12 +1687,18 @@ type ListPipelineRunResponse struct {
 
 // GetPipelineRunLogResponse is the response body of getting pipeline run log
 type GetPipelineRunLogResponse struct {
-	Log []Log `json:"log"`
+	StepBase `json:",inline"`
+	Log      string `json:"log"`
 }
 
 // GetPipelineRunOutputResponse is the response body of getting pipeline run output
 type GetPipelineRunOutputResponse struct {
-	Output []Output `json:"output"`
+	StepOutputs []StepOutputBase `json:"outputs"`
+}
+
+// GetPipelineRunInputResponse is the response body of getting pipeline run input
+type GetPipelineRunInputResponse struct {
+	StepInputs []StepInputBase `json:"inputs"`
 }
 
 // StepBase is the base info of step
@@ -1703,16 +1709,31 @@ type StepBase struct {
 	Phase string `json:"phase"`
 }
 
-// Log is the log of step
-type Log struct {
+// StepOutputBase is the output of step
+type StepOutputBase struct {
 	StepBase `json:",inline"`
-	Log      string `json:"log"`
+	Values   []OutputVar `json:"values"`
 }
 
-// Output is the output of step
-type Output struct {
+// StepInputBase is the input of step
+type StepInputBase struct {
 	StepBase `json:",inline"`
-	Vars     map[string]string `json:"vars"`
+	Values   []InputVar `json:"values"`
+}
+
+// OutputVar is one output var
+type OutputVar struct {
+	Name      string `json:"name"`
+	Value     string `json:"value"`
+	ValueFrom string `json:"valueFrom"`
+}
+
+// InputVar is one input var
+type InputVar struct {
+	From         string `json:"from"`
+	FromStep     string `json:"fromStep"`
+	ParameterKey string `json:"parameterKey"`
+	Value        string `json:"value"`
 }
 
 /*******************/