Retention: Database: Add database persistence for retention policies and filters

Signed-off-by: Nathan Lowe <public@nlowe.me>

make/migrations/postgresql/0005_retention_policies.up.sql

+CREATE TABLE IF NOT EXISTS retention_policy (
+    id SERIAL PRIMARY KEY NOT NULL,
+    name VARCHAR(255) NOT NULL,
+    enabled BOOLEAN NOT NULL,
+
+    scope INT NOT NULL,
+    fall_through_action INT NOT NULL,
+
+    project_id INTEGER REFERENCES project(project_id) ON DELETE CASCADE,
+    repository_id INTEGER REFERENCES repository(repository_id) ON DELETE CASCADE,
+
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS retention_policy_unique ON retention_policy (
+    COALESCE(project_id, 0),
+    COALESCE(repository_id, 0)
+);
+
+CREATE TABLE IF NOT EXISTS retention_filter_metadata (
+    id SERIAL PRIMARY KEY NOT NULL,
+    type VARCHAR(128) NOT NULL,
+
+    options JSON,
+
+    policy INTEGER REFERENCES retention_policy(id) ON DELETE CASCADE
+);
\ No newline at end of file

src/retention/dao/dao_test.go

+package dao
+
+import (
+	"os"
+	"testing"
+
+	"github.com/goharbor/harbor/src/common/dao"
+)
+
+func TestMain(m *testing.M) {
+	dao.PrepareTestForPostgresSQL()
+
+	result := m.Run()
+	dao.PrepareTestData([]string{
+		`DELETE FROM "retention_filter_metadata"; ALTER SEQUENCE retention_filter_metadata_id_seq RESTART;`,
+		`DELETE FROM "retention_policy"; ALTER SEQUENCE retention_policy_id_seq RESTART;`,
+		`DELETE FROM repository; ALTER SEQUENCE repository_repository_id_seq RESTART;`,
+	}, nil)
+
+	os.Exit(result)
+}

src/retention/dao/models/FilterMetadata.go

+package models
+
+import (
+	"encoding/json"
+)
+
+type FilterMetadata struct {
+	ID   int64  `orm:"column(id);pk;auto" json:"id"`
+	Type string `orm:"column(type)" json:"type"`
+
+	RawOptions string                 `orm:"column(options);type(json)" json:"-"`
+	Options    map[string]interface{} `orm:"-" json:"options"`
+
+	Policy *Policy `orm:"column(policy);rel(fk)" json:"-"`
+}
+
+func (f *FilterMetadata) SyncJsonToORM() error {
+	if bytes, err := json.Marshal(&f.Options); err != nil {
+		return err
+	} else {
+		f.RawOptions = string(bytes)
+	}
+
+	return nil
+}
+
+func (f *FilterMetadata) SyncORMToJson() error {
+	if f.RawOptions == "" {
+		f.Options = map[string]interface{}{}
+		return nil
+	}
+
+	return json.Unmarshal([]byte(f.RawOptions), &f.Options)
+}
+
+func (f *FilterMetadata) TableName() string {
+	return "retention_filter_metadata"
+}

src/retention/dao/models/base.go

+package models
+
+import "github.com/astaxie/beego/orm"
+
+func init() {
+	orm.RegisterModel(
+		&Policy{},
+		&FilterMetadata{},
+	)
+}

src/retention/dao/models/policy.go

+package models
+
+import (
+	"time"
+
+	commonModels "github.com/goharbor/harbor/src/common/models"
+	"github.com/goharbor/harbor/src/common/retention"
+)
+
+type Policy struct {
+	ID      int64  `orm:"pk;auto;column(id)" json:"id"`
+	Name    string `orm:"column(name)" json:"name"`
+	Enabled bool   `orm:"column(enabled)" json:"enabled"`
+
+	Scope             retention.Scope             `orm:"column(scope)" json:"scope"`
+	FallThroughAction retention.FallThroughAction `orm:"column(fall_through_action)" json:"fall_through_action"`
+
+	// The project the policy belongs to. If nil, the policy is a server-wide policy
+	Project *commonModels.Project `orm:"column(project_id);null;rel(fk)" json:"project_id"`
+	// The repository the policy belongs to. If nil, the policy is a project-wide policy
+	Repository *commonModels.RepoRecord `orm:"column(repository_id);null;rel(fk)" json:"repository_id"`
+
+	Filters []*FilterMetadata `orm:"reverse(many)" json:"filters"`
+
+	CreatedAt time.Time `orm:"column(created_at);auto_now_add" json:"created_at"`
+	UpdatedAt time.Time `orm:"column(updated_at);auto_now" json:"updated_at"`
+}
+
+func (p *Policy) TableName() string {
+	return "retention_policy"
+}
+
+func (p *Policy) TableUnique() [][]string {
+	return [][]string{
+		{"Project", "Repository"},
+	}
+}

src/retention/dao/policy.go

+package dao
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/astaxie/beego/orm"
+	commonDao "github.com/goharbor/harbor/src/common/dao"
+	"github.com/goharbor/harbor/src/retention/dao/models"
+)
+
+var deleteFilterQuery = fmt.Sprintf(`DELETE FROM %s WHERE policy = ?`, (&models.FilterMetadata{}).TableName())
+
+func AddPolicy(p *models.Policy) (id int64, err error) {
+	o := commonDao.GetOrmer()
+	if err = o.Begin(); err != nil {
+		return
+	}
+
+	defer func() {
+		if err != nil {
+			_ = o.Rollback()
+		} else {
+			err = o.Commit()
+		}
+	}()
+
+	if id, err = o.Insert(p); err != nil {
+		return
+	}
+
+	for _, f := range p.Filters {
+		f.Policy = p
+
+		if err = f.SyncJsonToORM(); err != nil {
+			return
+		}
+
+		if _, err = o.Insert(f); err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func GetServerPolicy() (p *models.Policy, err error) {
+	p = &models.Policy{}
+	o := commonDao.GetOrmer()
+
+	if err = o.
+		QueryTable(p).
+		Filter("project_id__isnull", true).
+		Filter("repository_id__isnull", true).
+		RelatedSel().
+		One(p); err == orm.ErrNoRows {
+		p = nil
+		err = nil
+	} else if err == nil {
+		if _, err = o.LoadRelated(p, "Filters"); err != nil {
+			return
+		}
+
+		for _, f := range p.Filters {
+			if err = f.SyncORMToJson(); err != nil {
+				return
+			}
+		}
+	}
+
+	return
+}
+
+func GetProjectPolicy(projectID int64) (p *models.Policy, err error) {
+	p = &models.Policy{}
+	o := commonDao.GetOrmer()
+
+	if err = o.
+		QueryTable(&models.Policy{}).
+		Filter("project_id", projectID).
+		Filter("repository_id__isnull", true).
+		RelatedSel().
+		One(p); err == orm.ErrNoRows {
+		p = nil
+		err = nil
+	} else if err == nil {
+		if _, err = o.LoadRelated(p, "Filters"); err != nil {
+			return
+		}
+
+		for _, f := range p.Filters {
+			if err = f.SyncORMToJson(); err != nil {
+				return
+			}
+		}
+	}
+
+	return
+}
+
+func GetRepoPolicy(projectID, repoID int64) (p *models.Policy, err error) {
+	p = &models.Policy{}
+	o := commonDao.GetOrmer()
+
+	if err = o.
+		QueryTable(p).
+		Filter("project_id", projectID).
+		Filter("repository_id", repoID).
+		RelatedSel().
+		One(p); err == orm.ErrNoRows {
+		p = nil
+		err = nil
+	} else if err == nil {
+		if _, err = o.LoadRelated(p, "Filters"); err != nil {
+			return
+		}
+
+		for _, f := range p.Filters {
+			if err = f.SyncORMToJson(); err != nil {
+				return
+			}
+		}
+	}
+
+	return
+}
+
+func UpdatePolicy(p *models.Policy, props ...string) (err error) {
+	o := commonDao.GetOrmer()
+	if err = o.Begin(); err != nil {
+		return
+	}
+
+	defer func() {
+		if err != nil {
+			_ = o.Rollback()
+		} else {
+			err = o.Commit()
+		}
+	}()
+
+	if p == nil {
+		err = errors.New("policy: cannot be nil")
+		return
+	}
+
+	_, err = o.Update(p, props...)
+	if err != nil {
+		return
+	}
+
+	// Easiest way to update filter metadata is to just drop and re-add all of them
+	if _, err = o.Raw(deleteFilterQuery, p.ID).Exec(); err != nil {
+		return
+	}
+
+	for _, f := range p.Filters {
+		f.Policy = p
+
+		if err = f.SyncJsonToORM(); err != nil {
+			return
+		}
+
+		if _, err = o.Insert(f); err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func DeletePolicy(id int64) error {
+	_, err := commonDao.GetOrmer().Delete(&models.Policy{ID: id})
+	return err
+}

src/retention/dao/policy_test.go

+package dao
+
+import (
+	"testing"
+
+	"github.com/magiconair/properties/assert"
+
+	common_dao "github.com/goharbor/harbor/src/common/dao"
+	common_models "github.com/goharbor/harbor/src/common/models"
+
+	"github.com/goharbor/harbor/src/retention/filter"
+
+	"github.com/goharbor/harbor/src/common/retention"
+	"github.com/goharbor/harbor/src/retention/dao/models"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	exampleServerPolicy = &models.Policy{
+		Name:    "Example Server Policy",
+		Enabled: true,
+
+		Scope:             retention.ScopeServer,
+		FallThroughAction: retention.KeepExtraTags,
+
+		Filters: []*models.FilterMetadata{
+			{Type: filter.TypeKeepMostRecentN, Options: map[string]interface{}{filter.MetaDataKeyN: 3}},
+		},
+	}
+
+	exampleProjectPolicy = &models.Policy{
+		Name:    "Example Project Policy",
+		Enabled: true,
+
+		Scope:   retention.ScopeProject,
+		Project: &common_models.Project{ProjectID: 1},
+
+		Filters: []*models.FilterMetadata{
+			{Type: filter.TypeKeepRegex, Options: map[string]interface{}{filter.MetaDataKeyMatch: `^latest$`}},
+			{Type: filter.TypeKeepMostRecentN, Options: map[string]interface{}{filter.MetaDataKeyN: 3}},
+			{Type: filter.TypeDeleteEverything},
+		},
+	}
+
+	exampleRepoPolicy = &models.Policy{
+		Name:    "Example Repo Policy",
+		Enabled: true,
+
+		Scope:      retention.ScopeRepository,
+		Project:    &common_models.Project{ProjectID: 1},
+		Repository: &common_models.RepoRecord{RepositoryID: 1},
+
+		Filters: []*models.FilterMetadata{
+			{Type: filter.TypeKeepEverything},
+		},
+	}
+)
+
+func TestAddPolicy(t *testing.T) {
+	require.NoError(t, common_dao.AddRepository(common_models.RepoRecord{
+		ProjectID: 1,
+		Name:      "RetentionTest",
+	}))
+
+	tests := []struct {
+		Name       string
+		Policy     *models.Policy
+		ExpectedID int64
+	}{
+		{Name: "Server", ExpectedID: 1, Policy: exampleServerPolicy},
+		{Name: "Project", ExpectedID: 2, Policy: exampleProjectPolicy},
+		{Name: "Repo", ExpectedID: 3, Policy: exampleRepoPolicy},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.Name, func(t *testing.T) {
+			id, err := AddPolicy(tt.Policy)
+
+			require.NoError(t, err)
+			require.Equal(t, tt.ExpectedID, id)
+		})
+	}
+}
+
+func TestAddPolicy_SinglePolicyPerScope(t *testing.T) {
+	tests := []struct {
+		Name   string
+		Policy *models.Policy
+	}{
+		{Name: "Server", Policy: exampleServerPolicy},
+		{Name: "Project", Policy: exampleProjectPolicy},
+		{Name: "Repo", Policy: exampleRepoPolicy},
+	}
+
+	for idx, tt := range tests {
+		t.Run(tt.Name, func(t *testing.T) {
+			tt.Policy.ID = 0
+			_, err := AddPolicy(tt.Policy)
+
+			require.Error(t, err)
+			tt.Policy.ID = int64(idx + 1)
+		})
+	}
+}
+
+func TestGetServerPolicy(t *testing.T) {
+	p, err := GetServerPolicy()
+
+	require.NoError(t, err)
+	assertPolicyEffectivelyEqual(t, exampleServerPolicy, p)
+}
+
+func TestGetProjectPolicy(t *testing.T) {
+	p, err := GetProjectPolicy(1)
+
+	require.NoError(t, err)
+	assertPolicyEffectivelyEqual(t, exampleProjectPolicy, p)
+}
+
+func TestGetRepoPolicy(t *testing.T) {
+	p, err := GetRepoPolicy(1, 1)
+
+	require.NoError(t, err)
+	assertPolicyEffectivelyEqual(t, exampleRepoPolicy, p)
+}
+
+func TestUpdatePolicy(t *testing.T) {
+	tests := []struct {
+		Name       string
+		Policy     *models.Policy
+		UpdateFunc func(p *models.Policy)
+	}{
+		{Name: "Server Policy", Policy: exampleServerPolicy, UpdateFunc: func(p *models.Policy) {
+			p.FallThroughAction = retention.DeleteExtraTags
+
+			p.Filters[0].Options[filter.MetaDataKeyN] = 5
+			p.Filters = append(p.Filters, &models.FilterMetadata{
+				Type:    filter.TypeKeepRegex,
+				Options: map[string]interface{}{filter.MetaDataKeyMatch: `^foo$`},
+			})
+		}},
+		{Name: "Project Policy", Policy: exampleProjectPolicy, UpdateFunc: func(p *models.Policy) {
+			p.FallThroughAction = retention.DeleteExtraTags
+			p.Filters[0].Options[filter.MetaDataKeyMatch] = `^bar$`
+			p.Filters = append(p.Filters[:1], p.Filters[2:]...)
+		}},
+		{Name: "Repo Policy", Policy: exampleRepoPolicy, UpdateFunc: func(p *models.Policy) {
+			p.FallThroughAction = retention.DeleteExtraTags
+		}},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.Name, func(t *testing.T) {
+			tt.UpdateFunc(tt.Policy)
+			require.NoError(t, UpdatePolicy(tt.Policy))
+
+			var updatedPolicy *models.Policy
+			var err error
+			switch tt.Policy.Scope {
+			case retention.ScopeServer:
+				updatedPolicy, err = GetServerPolicy()
+			case retention.ScopeProject:
+				updatedPolicy, err = GetProjectPolicy(1)
+			case retention.ScopeRepository:
+				updatedPolicy, err = GetRepoPolicy(1, 1)
+			}
+
+			require.NoError(t, err)
+			require.NotNil(t, updatedPolicy)
+
+			assertPolicyEffectivelyEqual(t, tt.Policy, updatedPolicy)
+		})
+	}
+}
+
+func TestDeletePolicy(t *testing.T) {
+	tests := []struct {
+		Name   string
+		Policy *models.Policy
+	}{
+		{Name: "Server Policy", Policy: exampleServerPolicy},
+		{Name: "Project Policy", Policy: exampleProjectPolicy},
+		{Name: "Repo Policy", Policy: exampleRepoPolicy},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.Name, func(t *testing.T) {
+			require.NoError(t, DeletePolicy(tt.Policy.ID))
+		})
+	}
+
+	sp, err := GetServerPolicy()
+	require.NoError(t, err)
+	require.Nil(t, sp)
+
+	pp, err := GetProjectPolicy(1)
+	require.NoError(t, err)
+	require.Nil(t, pp)
+
+	rp, err := GetRepoPolicy(1, 1)
+	require.NoError(t, err)
+	require.Nil(t, rp)
+}
+
+// assertPolicyEffectivelyEqual compares most properties of two policies for equality.
+//
+// This is an easy way to check that policy representations are mostly equal
+// while accounting for differences in encoding (like ints being float64 after
+// deserialization or dates being in the wrong timezone).
+func assertPolicyEffectivelyEqual(t *testing.T, expected, actual *models.Policy) {
+	require.NotNil(t, expected)
+	require.NotNil(t, actual)
+
+	assert.Equal(t, actual.ID, expected.ID, "ID Not Equal")
+	assert.Equal(t, actual.Name, expected.Name, "Name Not Equal")
+	assert.Equal(t, actual.Enabled, expected.Enabled, "Enabled flag not equal")
+
+	assert.Equal(t, actual.Scope, expected.Scope, "Scope not equal")
+	assert.Equal(t, actual.FallThroughAction, expected.FallThroughAction, "FallThroughAction not equal")
+
+	if expected.Project != nil {
+		assert.Equal(t, actual.Project.ProjectID, expected.Project.ProjectID, "Project ID Not Equal")
+	} else {
+		require.Nil(t, actual.Project)
+	}
+
+	if expected.Repository != nil {
+		assert.Equal(t, actual.Repository.RepositoryID, expected.Repository.RepositoryID, "Repository ID Not Equal")
+	} else {
+		require.Nil(t, actual.Repository)
+	}
+
+	if len(expected.Filters) == 0 {
+		require.Empty(t, actual.Filters, "Filter list not empty")
+		return
+	}
+
+	require.Equal(t, len(expected.Filters), len(actual.Filters), "Filter list different length")
+	for i, ef := range expected.Filters {
+		af := actual.Filters[i]
+
+		assert.Equal(t, af.ID, ef.ID, "Filter ID not equal")
+		assert.Equal(t, af.Type, ef.Type, "Filter type not equal")
+
+		assert.Equal(t, af.RawOptions, ef.RawOptions, "Filter Options not equal")
+	}
+}