Unverified Commit 288a9fa2 authored by Nathan Lowe's avatar Nathan Lowe
Browse files

Retention: Database: Add database persistence for retention policies and filters

Signed-off-by: default avatarNathan Lowe <public@nlowe.me>
parent 4b1edd04
Showing with 556 additions and 0 deletions
+556 -0
CREATE TABLE IF NOT EXISTS retention_policy (
id SERIAL PRIMARY KEY NOT NULL,
name VARCHAR(255) NOT NULL,
enabled BOOLEAN NOT NULL,
scope INT NOT NULL,
fall_through_action INT NOT NULL,
project_id INTEGER REFERENCES project(project_id) ON DELETE CASCADE,
repository_id INTEGER REFERENCES repository(repository_id) ON DELETE CASCADE,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
CREATE UNIQUE INDEX IF NOT EXISTS retention_policy_unique ON retention_policy (
COALESCE(project_id, 0),
COALESCE(repository_id, 0)
);
CREATE TABLE IF NOT EXISTS retention_filter_metadata (
id SERIAL PRIMARY KEY NOT NULL,
type VARCHAR(128) NOT NULL,
options JSON,
policy INTEGER REFERENCES retention_policy(id) ON DELETE CASCADE
);
\ No newline at end of file
package dao
import (
"os"
"testing"
"github.com/goharbor/harbor/src/common/dao"
)
func TestMain(m *testing.M) {
dao.PrepareTestForPostgresSQL()
result := m.Run()
dao.PrepareTestData([]string{
`DELETE FROM "retention_filter_metadata"; ALTER SEQUENCE retention_filter_metadata_id_seq RESTART;`,
`DELETE FROM "retention_policy"; ALTER SEQUENCE retention_policy_id_seq RESTART;`,
`DELETE FROM repository; ALTER SEQUENCE repository_repository_id_seq RESTART;`,
}, nil)
os.Exit(result)
}
package models
import (
"encoding/json"
)
type FilterMetadata struct {
ID int64 `orm:"column(id);pk;auto" json:"id"`
Type string `orm:"column(type)" json:"type"`
RawOptions string `orm:"column(options);type(json)" json:"-"`
Options map[string]interface{} `orm:"-" json:"options"`
Policy *Policy `orm:"column(policy);rel(fk)" json:"-"`
}
func (f *FilterMetadata) SyncJsonToORM() error {
if bytes, err := json.Marshal(&f.Options); err != nil {
return err
} else {
f.RawOptions = string(bytes)
}
return nil
}
func (f *FilterMetadata) SyncORMToJson() error {
if f.RawOptions == "" {
f.Options = map[string]interface{}{}
return nil
}
return json.Unmarshal([]byte(f.RawOptions), &f.Options)
}
func (f *FilterMetadata) TableName() string {
return "retention_filter_metadata"
}
package models
import "github.com/astaxie/beego/orm"
func init() {
orm.RegisterModel(
&Policy{},
&FilterMetadata{},
)
}
package models
import (
"time"
commonModels "github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/retention"
)
type Policy struct {
ID int64 `orm:"pk;auto;column(id)" json:"id"`
Name string `orm:"column(name)" json:"name"`
Enabled bool `orm:"column(enabled)" json:"enabled"`
Scope retention.Scope `orm:"column(scope)" json:"scope"`
FallThroughAction retention.FallThroughAction `orm:"column(fall_through_action)" json:"fall_through_action"`
// The project the policy belongs to. If nil, the policy is a server-wide policy
Project *commonModels.Project `orm:"column(project_id);null;rel(fk)" json:"project_id"`
// The repository the policy belongs to. If nil, the policy is a project-wide policy
Repository *commonModels.RepoRecord `orm:"column(repository_id);null;rel(fk)" json:"repository_id"`
Filters []*FilterMetadata `orm:"reverse(many)" json:"filters"`
CreatedAt time.Time `orm:"column(created_at);auto_now_add" json:"created_at"`
UpdatedAt time.Time `orm:"column(updated_at);auto_now" json:"updated_at"`
}
func (p *Policy) TableName() string {
return "retention_policy"
}
func (p *Policy) TableUnique() [][]string {
return [][]string{
{"Project", "Repository"},
}
}
package dao
import (
"errors"
"fmt"
"github.com/astaxie/beego/orm"
commonDao "github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/retention/dao/models"
)
var deleteFilterQuery = fmt.Sprintf(`DELETE FROM %s WHERE policy = ?`, (&models.FilterMetadata{}).TableName())
func AddPolicy(p *models.Policy) (id int64, err error) {
o := commonDao.GetOrmer()
if err = o.Begin(); err != nil {
return
}
defer func() {
if err != nil {
_ = o.Rollback()
} else {
err = o.Commit()
}
}()
if id, err = o.Insert(p); err != nil {
return
}
for _, f := range p.Filters {
f.Policy = p
if err = f.SyncJsonToORM(); err != nil {
return
}
if _, err = o.Insert(f); err != nil {
return
}
}
return
}
func GetServerPolicy() (p *models.Policy, err error) {
p = &models.Policy{}
o := commonDao.GetOrmer()
if err = o.
QueryTable(p).
Filter("project_id__isnull", true).
Filter("repository_id__isnull", true).
RelatedSel().
One(p); err == orm.ErrNoRows {
p = nil
err = nil
} else if err == nil {
if _, err = o.LoadRelated(p, "Filters"); err != nil {
return
}
for _, f := range p.Filters {
if err = f.SyncORMToJson(); err != nil {
return
}
}
}
return
}
func GetProjectPolicy(projectID int64) (p *models.Policy, err error) {
p = &models.Policy{}
o := commonDao.GetOrmer()
if err = o.
QueryTable(&models.Policy{}).
Filter("project_id", projectID).
Filter("repository_id__isnull", true).
RelatedSel().
One(p); err == orm.ErrNoRows {
p = nil
err = nil
} else if err == nil {
if _, err = o.LoadRelated(p, "Filters"); err != nil {
return
}
for _, f := range p.Filters {
if err = f.SyncORMToJson(); err != nil {
return
}
}
}
return
}
func GetRepoPolicy(projectID, repoID int64) (p *models.Policy, err error) {
p = &models.Policy{}
o := commonDao.GetOrmer()
if err = o.
QueryTable(p).
Filter("project_id", projectID).
Filter("repository_id", repoID).
RelatedSel().
One(p); err == orm.ErrNoRows {
p = nil
err = nil
} else if err == nil {
if _, err = o.LoadRelated(p, "Filters"); err != nil {
return
}
for _, f := range p.Filters {
if err = f.SyncORMToJson(); err != nil {
return
}
}
}
return
}
func UpdatePolicy(p *models.Policy, props ...string) (err error) {
o := commonDao.GetOrmer()
if err = o.Begin(); err != nil {
return
}
defer func() {
if err != nil {
_ = o.Rollback()
} else {
err = o.Commit()
}
}()
if p == nil {
err = errors.New("policy: cannot be nil")
return
}
_, err = o.Update(p, props...)
if err != nil {
return
}
// Easiest way to update filter metadata is to just drop and re-add all of them
if _, err = o.Raw(deleteFilterQuery, p.ID).Exec(); err != nil {
return
}
for _, f := range p.Filters {
f.Policy = p
if err = f.SyncJsonToORM(); err != nil {
return
}
if _, err = o.Insert(f); err != nil {
return
}
}
return
}
func DeletePolicy(id int64) error {
_, err := commonDao.GetOrmer().Delete(&models.Policy{ID: id})
return err
}
package dao
import (
"testing"
"github.com/magiconair/properties/assert"
common_dao "github.com/goharbor/harbor/src/common/dao"
common_models "github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/retention/filter"
"github.com/goharbor/harbor/src/common/retention"
"github.com/goharbor/harbor/src/retention/dao/models"
"github.com/stretchr/testify/require"
)
var (
exampleServerPolicy = &models.Policy{
Name: "Example Server Policy",
Enabled: true,
Scope: retention.ScopeServer,
FallThroughAction: retention.KeepExtraTags,
Filters: []*models.FilterMetadata{
{Type: filter.TypeKeepMostRecentN, Options: map[string]interface{}{filter.MetaDataKeyN: 3}},
},
}
exampleProjectPolicy = &models.Policy{
Name: "Example Project Policy",
Enabled: true,
Scope: retention.ScopeProject,
Project: &common_models.Project{ProjectID: 1},
Filters: []*models.FilterMetadata{
{Type: filter.TypeKeepRegex, Options: map[string]interface{}{filter.MetaDataKeyMatch: `^latest$`}},
{Type: filter.TypeKeepMostRecentN, Options: map[string]interface{}{filter.MetaDataKeyN: 3}},
{Type: filter.TypeDeleteEverything},
},
}
exampleRepoPolicy = &models.Policy{
Name: "Example Repo Policy",
Enabled: true,
Scope: retention.ScopeRepository,
Project: &common_models.Project{ProjectID: 1},
Repository: &common_models.RepoRecord{RepositoryID: 1},
Filters: []*models.FilterMetadata{
{Type: filter.TypeKeepEverything},
},
}
)
func TestAddPolicy(t *testing.T) {
require.NoError(t, common_dao.AddRepository(common_models.RepoRecord{
ProjectID: 1,
Name: "RetentionTest",
}))
tests := []struct {
Name string
Policy *models.Policy
ExpectedID int64
}{
{Name: "Server", ExpectedID: 1, Policy: exampleServerPolicy},
{Name: "Project", ExpectedID: 2, Policy: exampleProjectPolicy},
{Name: "Repo", ExpectedID: 3, Policy: exampleRepoPolicy},
}
for _, tt := range tests {
t.Run(tt.Name, func(t *testing.T) {
id, err := AddPolicy(tt.Policy)
require.NoError(t, err)
require.Equal(t, tt.ExpectedID, id)
})
}
}
func TestAddPolicy_SinglePolicyPerScope(t *testing.T) {
tests := []struct {
Name string
Policy *models.Policy
}{
{Name: "Server", Policy: exampleServerPolicy},
{Name: "Project", Policy: exampleProjectPolicy},
{Name: "Repo", Policy: exampleRepoPolicy},
}
for idx, tt := range tests {
t.Run(tt.Name, func(t *testing.T) {
tt.Policy.ID = 0
_, err := AddPolicy(tt.Policy)
require.Error(t, err)
tt.Policy.ID = int64(idx + 1)
})
}
}
func TestGetServerPolicy(t *testing.T) {
p, err := GetServerPolicy()
require.NoError(t, err)
assertPolicyEffectivelyEqual(t, exampleServerPolicy, p)
}
func TestGetProjectPolicy(t *testing.T) {
p, err := GetProjectPolicy(1)
require.NoError(t, err)
assertPolicyEffectivelyEqual(t, exampleProjectPolicy, p)
}
func TestGetRepoPolicy(t *testing.T) {
p, err := GetRepoPolicy(1, 1)
require.NoError(t, err)
assertPolicyEffectivelyEqual(t, exampleRepoPolicy, p)
}
func TestUpdatePolicy(t *testing.T) {
tests := []struct {
Name string
Policy *models.Policy
UpdateFunc func(p *models.Policy)
}{
{Name: "Server Policy", Policy: exampleServerPolicy, UpdateFunc: func(p *models.Policy) {
p.FallThroughAction = retention.DeleteExtraTags
p.Filters[0].Options[filter.MetaDataKeyN] = 5
p.Filters = append(p.Filters, &models.FilterMetadata{
Type: filter.TypeKeepRegex,
Options: map[string]interface{}{filter.MetaDataKeyMatch: `^foo$`},
})
}},
{Name: "Project Policy", Policy: exampleProjectPolicy, UpdateFunc: func(p *models.Policy) {
p.FallThroughAction = retention.DeleteExtraTags
p.Filters[0].Options[filter.MetaDataKeyMatch] = `^bar$`
p.Filters = append(p.Filters[:1], p.Filters[2:]...)
}},
{Name: "Repo Policy", Policy: exampleRepoPolicy, UpdateFunc: func(p *models.Policy) {
p.FallThroughAction = retention.DeleteExtraTags
}},
}
for _, tt := range tests {
t.Run(tt.Name, func(t *testing.T) {
tt.UpdateFunc(tt.Policy)
require.NoError(t, UpdatePolicy(tt.Policy))
var updatedPolicy *models.Policy
var err error
switch tt.Policy.Scope {
case retention.ScopeServer:
updatedPolicy, err = GetServerPolicy()
case retention.ScopeProject:
updatedPolicy, err = GetProjectPolicy(1)
case retention.ScopeRepository:
updatedPolicy, err = GetRepoPolicy(1, 1)
}
require.NoError(t, err)
require.NotNil(t, updatedPolicy)
assertPolicyEffectivelyEqual(t, tt.Policy, updatedPolicy)
})
}
}
func TestDeletePolicy(t *testing.T) {
tests := []struct {
Name string
Policy *models.Policy
}{
{Name: "Server Policy", Policy: exampleServerPolicy},
{Name: "Project Policy", Policy: exampleProjectPolicy},
{Name: "Repo Policy", Policy: exampleRepoPolicy},
}
for _, tt := range tests {
t.Run(tt.Name, func(t *testing.T) {
require.NoError(t, DeletePolicy(tt.Policy.ID))
})
}
sp, err := GetServerPolicy()
require.NoError(t, err)
require.Nil(t, sp)
pp, err := GetProjectPolicy(1)
require.NoError(t, err)
require.Nil(t, pp)
rp, err := GetRepoPolicy(1, 1)
require.NoError(t, err)
require.Nil(t, rp)
}
// assertPolicyEffectivelyEqual compares most properties of two policies for equality.
//
// This is an easy way to check that policy representations are mostly equal
// while accounting for differences in encoding (like ints being float64 after
// deserialization or dates being in the wrong timezone).
func assertPolicyEffectivelyEqual(t *testing.T, expected, actual *models.Policy) {
require.NotNil(t, expected)
require.NotNil(t, actual)
assert.Equal(t, actual.ID, expected.ID, "ID Not Equal")
assert.Equal(t, actual.Name, expected.Name, "Name Not Equal")
assert.Equal(t, actual.Enabled, expected.Enabled, "Enabled flag not equal")
assert.Equal(t, actual.Scope, expected.Scope, "Scope not equal")
assert.Equal(t, actual.FallThroughAction, expected.FallThroughAction, "FallThroughAction not equal")
if expected.Project != nil {
assert.Equal(t, actual.Project.ProjectID, expected.Project.ProjectID, "Project ID Not Equal")
} else {
require.Nil(t, actual.Project)
}
if expected.Repository != nil {
assert.Equal(t, actual.Repository.RepositoryID, expected.Repository.RepositoryID, "Repository ID Not Equal")
} else {
require.Nil(t, actual.Repository)
}
if len(expected.Filters) == 0 {
require.Empty(t, actual.Filters, "Filter list not empty")
return
}
require.Equal(t, len(expected.Filters), len(actual.Filters), "Filter list different length")
for i, ef := range expected.Filters {
af := actual.Filters[i]
assert.Equal(t, af.ID, ef.ID, "Filter ID not equal")
assert.Equal(t, af.Type, ef.Type, "Filter type not equal")
assert.Equal(t, af.RawOptions, ef.RawOptions, "Filter Options not equal")
}
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment