add sqlite support

.travis.yml

@@ -38,6 +38,7 @@ before_install:
 
 install: 
   - sudo apt-get update && sudo apt-get install -y libldap2-dev 
+  - sudo apt-get install -y sqlite3
 #  - sudo apt-get remove -y mysql-common mysql-server-5.5 mysql-server-core-5.5 mysql-client-5.5 mysql-client-core-5.5
 #  - sudo apt-get autoremove -y
 #  - sudo apt-get install -y libaio1
@@ -71,6 +72,7 @@ install:
 before_script:
   # create tables and load data
   # - mysql < ./Deploy/db/registry.sql -uroot --verbose
+  - sudo sqlite3 /registry.db < ./Deploy/db/registry_sqlite.sql
 
 script:
   - sudo ./tests/testprepare.sh 

Deploy/db/registry_sqlite.sql

+create table access (
+ access_id INTEGER PRIMARY KEY,
+ access_code char(1),
+ comment varchar (30)
+);
+
+insert into access (access_code, comment) values 
+('M', 'Management access for project'),
+('R', 'Read access for project'),
+('W', 'Write access for project'),
+('D', 'Delete access for project'),
+('S', 'Search access for project');
+
+
+create table role (
+ role_id INTEGER PRIMARY KEY,
+ role_mask int DEFAULT 0 NOT NULL,
+ role_code varchar(20),
+ name varchar (20)
+);
+/*
+role mask is used for future enhancement when a project member can have multi-roles
+currently set to 0
+*/
+
+insert into role (role_code, name) values 
+('MDRWS', 'projectAdmin'),
+('RWS', 'developer'),
+('RS', 'guest');
+
+
+create table user (
+ user_id INTEGER PRIMARY KEY,
+/*
+ The max length of username controlled by API is 20, 
+ and 11 is reserved for marking the deleted users.
+ The mark of deleted user is "#user_id".
+ The 11 consist of 10 for the max value of user_id(4294967295)  
+ in MySQL and 1 of '#'.
+*/
+ username varchar(32),
+/*
+ 11 bytes is reserved for marking the deleted users.
+*/
+ email varchar(255),
+ password varchar(40) NOT NULL,
+ realname varchar (20) NOT NULL,
+ comment varchar (30),
+ deleted tinyint (1) DEFAULT 0 NOT NULL,
+ reset_uuid varchar(40) DEFAULT NULL,
+ salt varchar(40) DEFAULT NULL,
+ sysadmin_flag tinyint (1),
+ creation_time timestamp,
+ update_time timestamp,
+ UNIQUE (username),
+ UNIQUE (email)
+);
+
+insert into user (username, email, password, realname, comment, deleted, sysadmin_flag, creation_time, update_time) values 
+('admin', 'admin@example.com', '', 'system admin', 'admin user',0, 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
+('anonymous', 'anonymous@example.com', '', 'anonymous user', 'anonymous user', 1, 0, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP);
+                                                                          
+create table project (
+ project_id INTEGER PRIMARY KEY,
+ owner_id int NOT NULL,
+/*
+ The max length of name controlled by API is 30, 
+ and 11 is reserved for marking the deleted project.
+*/
+ name varchar (41) NOT NULL,
+ creation_time timestamp,
+ update_time timestamp,
+ deleted tinyint (1) DEFAULT 0 NOT NULL,
+ public tinyint (1) DEFAULT 0 NOT NULL,
+ FOREIGN KEY (owner_id) REFERENCES user(user_id),
+ UNIQUE (name)
+);
+
+insert into project (owner_id, name, creation_time, update_time, public) values 
+(1, 'library', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 1);
+
+create table project_member (
+ project_id int NOT NULL,
+ user_id int NOT NULL,
+ role int NOT NULL,
+ creation_time timestamp,
+ update_time timestamp,
+ PRIMARY KEY (project_id, user_id),
+ FOREIGN KEY (role) REFERENCES role(role_id),
+ FOREIGN KEY (project_id) REFERENCES project(project_id),
+ FOREIGN KEY (user_id) REFERENCES user(user_id)
+ );
+
+insert into project_member (project_id, user_id, role, creation_time, update_time) values
+(1, 1, 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP);
+
+create table access_log (
+ log_id INTEGER PRIMARY KEY,
+ user_id int NOT NULL,
+ project_id int NOT NULL,
+ repo_name varchar (256), 
+ repo_tag varchar (128),
+ GUID varchar(64), 
+ operation varchar(20) NOT NULL,
+ op_time timestamp,
+ FOREIGN KEY (user_id) REFERENCES user(user_id),
+ FOREIGN KEY (project_id) REFERENCES project (project_id)
+);
+
+CREATE INDEX pid_optime ON access_log (project_id, op_time);
+
+create table repository (
+ repository_id INTEGER PRIMARY KEY,
+ name varchar(255) NOT NULL,
+ project_id int NOT NULL,
+ owner_id int NOT NULL,
+ description text,
+ pull_count int DEFAULT 0 NOT NULL,
+ star_count int DEFAULT 0 NOT NULL,
+ creation_time timestamp default CURRENT_TIMESTAMP,
+ update_time timestamp default CURRENT_TIMESTAMP,
+ FOREIGN KEY (owner_id) REFERENCES user(user_id),
+ FOREIGN KEY (project_id) REFERENCES project(project_id),
+ UNIQUE (name)
+);
+
+create table replication_policy (
+ id INTEGER PRIMARY KEY,
+ name varchar(256),
+ project_id int NOT NULL,
+ target_id int NOT NULL,
+ enabled tinyint(1) NOT NULL DEFAULT 1,
+ description text,
+ deleted tinyint (1) DEFAULT 0 NOT NULL,
+ cron_str varchar(256),
+ start_time timestamp NULL,
+ creation_time timestamp default CURRENT_TIMESTAMP,
+ update_time timestamp default CURRENT_TIMESTAMP
+ );
+
+create table replication_target (
+ id INTEGER PRIMARY KEY,
+ name varchar(64),
+ url varchar(64),
+ username varchar(40),
+ password varchar(128),
+ /*
+ target_type indicates the type of target registry,
+ 0 means it's a harbor instance,
+ 1 means it's a regulart registry
+ */
+ target_type tinyint(1) NOT NULL DEFAULT 0,
+ creation_time timestamp default CURRENT_TIMESTAMP,
+ update_time timestamp default CURRENT_TIMESTAMP
+ );
+
+create table replication_job (
+ id INTEGER PRIMARY KEY,
+ status varchar(64) NOT NULL,
+ policy_id int NOT NULL,
+ repository varchar(256) NOT NULL,
+ operation  varchar(64) NOT NULL,
+ tags   varchar(16384),
+ creation_time timestamp default CURRENT_TIMESTAMP,
+ update_time timestamp default CURRENT_TIMESTAMP
+ );
+
+CREATE INDEX policy ON replication_job (policy_id);
+CREATE INDEX poid_uptime ON replication_job (policy_id, update_time);
+ 
+create table properties (
+ k varchar(64) NOT NULL,
+ v varchar(128) NOT NULL,
+ primary key (k)
+ );
+
+create table alembic_version (
+    version_num varchar(32) NOT NULL
+);
+
+insert into alembic_version values ('0.3.0');

api/harborapi_test.go

@@ -5,14 +5,18 @@ package api
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/vmware/harbor/dao"
-	"github.com/vmware/harbor/models"
-	"github.com/vmware/harbor/tests/apitests/apilib"
 	"io/ioutil"
 	"net/http/httptest"
 	"path/filepath"
 	"runtime"
 
+	"github.com/vmware/harbor/dao"
+	"github.com/vmware/harbor/models"
+	"github.com/vmware/harbor/tests/apitests/apilib"
+	"github.com/vmware/harbor/utils"
+	//	"strconv"
+	//	"strings"
+
 	"github.com/astaxie/beego"
 	"github.com/dghubble/sling"
 
@@ -52,7 +56,7 @@ type usrInfo struct {
 }
 
 func init() {
-	dao.InitDB()
+	dao.InitDatabase()
 	_, file, _, _ := runtime.Caller(1)
 	apppath, _ := filepath.Abs(filepath.Dir(filepath.Join(file, ".."+string(filepath.Separator))))
 	beego.BConfig.WebConfig.Session.SessionOn = true
@@ -835,12 +839,7 @@ func updateInitPassword(userID int, password string) error {
 		return fmt.Errorf("User id: %d does not exist.", userID)
 	}
 	if user.Salt == "" {
-		salt, err := dao.GenerateRandomString()
-		if err != nil {
-			return fmt.Errorf("Failed to generate salt for encrypting password, %v", err)
-		}
-
-		user.Salt = salt
+		user.Salt = utils.GenerateRandomString()
 		user.Password = password
 		err = dao.ChangeUserPassword(*user)
 		if err != nil {

controllers/password.go

@@ -53,11 +53,7 @@ func (cc *CommonController) SendEmail() {
 		if harborURL == "" {
 			harborURL = "localhost"
 		}
-		uuid, err := dao.GenerateRandomString()
-		if err != nil {
-			log.Errorf("Error occurred in GenerateRandomString: %v", err)
-			cc.CustomAbort(http.StatusInternalServerError, "Internal error.")
-		}
+		uuid := utils.GenerateRandomString()
 		err = messageTemplate.Execute(message, messageDetail{
 			Hint: cc.Tr("reset_email_hint"),
 			URL:  harborURL,

dao/accesslog.go

@@ -17,6 +17,7 @@ package dao
 
 import (
 	"strings"
+	"time"
 
 	"github.com/vmware/harbor/models"
 	"github.com/vmware/harbor/utils/log"
@@ -27,13 +28,14 @@ func AddAccessLog(accessLog models.AccessLog) error {
 	o := GetOrmer()
 	p, err := o.Raw(`insert into access_log
 		 (user_id, project_id, repo_name, repo_tag, guid, operation, op_time)
-		 values (?, ?, ?, ?, ?, ?, now())`).Prepare()
+		 values (?, ?, ?, ?, ?, ?, ?)`).Prepare()
 	if err != nil {
 		return err
 	}
 	defer p.Close()
 
-	_, err = p.Exec(accessLog.UserID, accessLog.ProjectID, accessLog.RepoName, accessLog.RepoTag, accessLog.GUID, accessLog.Operation)
+	_, err = p.Exec(accessLog.UserID, accessLog.ProjectID, accessLog.RepoName, accessLog.RepoTag,
+		accessLog.GUID, accessLog.Operation, time.Now())
 
 	return err
 }
@@ -145,8 +147,8 @@ func AccessLog(username, projectName, repoName, repoTag, action string) error {
 	o := GetOrmer()
 	sql := "insert into  access_log (user_id, project_id, repo_name, repo_tag, operation, op_time) " +
 		"select (select user_id as user_id from user where username=?), " +
-		"(select project_id as project_id from project where name=?), ?, ?, ?, now() "
-	_, err := o.Raw(sql, username, projectName, repoName, repoTag, action).Exec()
+		"(select project_id as project_id from project where name=?), ?, ?, ?, ? "
+	_, err := o.Raw(sql, username, projectName, repoName, repoTag, action, time.Now()).Exec()
 
 	if err != nil {
 		log.Errorf("error in AccessLog: %v ", err)

dao/base.go

@@ -17,67 +17,75 @@ package dao
 
 import (
 	"fmt"
-	"net"
-
 	"os"
+	"strings"
 	"sync"
-	"time"
 
 	"github.com/astaxie/beego/orm"
-	_ "github.com/go-sql-driver/mysql" //register mysql driver
 	"github.com/vmware/harbor/utils/log"
 )
 
 // NonExistUserID : if a user does not exist, the ID of the user will be 0.
 const NonExistUserID = 0
 
-// GenerateRandomString generates a random string
-func GenerateRandomString() (string, error) {
-	o := orm.NewOrm()
-	var uuid string
-	err := o.Raw(`select uuid() as uuid`).QueryRow(&uuid)
+// Database is an interface of different databases
+type Database interface {
+	// Name returns the name of database
+	Name() string
+	// String returns the details of database
+	String() string
+	// Register registers the database which will be used
+	Register(alias ...string) error
+}
+
+// InitDatabase initializes the database
+func InitDatabase() {
+	database, err := getDatabase()
 	if err != nil {
-		return "", err
+		panic(err)
 	}
-	return uuid, nil
 
+	log.Infof("initializing database: %s", database.String())
+	if err := database.Register(); err != nil {
+		panic(err)
+	}
 }
 
-//InitDB initializes the database
-func InitDB() {
-	// orm.Debug = true
-	orm.RegisterDriver("mysql", orm.DRMySQL)
-	addr := os.Getenv("MYSQL_HOST")
-	port := os.Getenv("MYSQL_PORT")
-	username := os.Getenv("MYSQL_USR")
-	password := os.Getenv("MYSQL_PWD")
-
-	log.Debugf("db url: %s:%s, db user: %s", addr, port, username)
-	dbStr := username + ":" + password + "@tcp(" + addr + ":" + port + ")/registry"
-	ch := make(chan int, 1)
-	go func() {
-		var err error
-		var c net.Conn
-		for {
-			c, err = net.DialTimeout("tcp", addr+":"+port, 20*time.Second)
-			if err == nil {
-				c.Close()
-				ch <- 1
-			} else {
-				log.Errorf("failed to connect to db, retry after 2 seconds :%v", err)
-				time.Sleep(2 * time.Second)
-			}
-		}
-	}()
-	select {
-	case <-ch:
-	case <-time.After(60 * time.Second):
-		panic("Failed to connect to DB after 60 seconds")
+func getDatabase() (db Database, err error) {
+	switch strings.ToLower(os.Getenv("DATABASE")) {
+	case "", "mysql":
+		host, port, usr, pwd, database := getMySQLConnInfo()
+		db = NewMySQL(host, port, usr, pwd, database)
+	case "sqlite":
+		file := getSQLiteConnInfo()
+		db = NewSQLite(file)
+	default:
+		err = fmt.Errorf("invalid database: %s", os.Getenv("DATABASE"))
 	}
-	err := orm.RegisterDataBase("default", "mysql", dbStr)
-	if err != nil {
-		panic(err)
+
+	return
+}
+
+// TODO read from config
+func getMySQLConnInfo() (host, port, username, password, database string) {
+	host = os.Getenv("MYSQL_HOST")
+	port = os.Getenv("MYSQL_PORT")
+	username = os.Getenv("MYSQL_USR")
+	password = os.Getenv("MYSQL_PWD")
+	database = os.Getenv("MYSQL_DATABASE")
+	if len(database) == 0 {
+		database = "registry"
+	}
+	return
+}
+
+// TODO read from config
+func getSQLiteConnInfo() string {
+	file := os.Getenv("SQLITE_FILE")
+	if len(file) == 0 {
+		file = "registry.db"
 	}
+	return file
 }
 
 var globalOrm orm.Ormer

dao/dao_test.go

@@ -16,13 +16,13 @@
 package dao
 
 import (
-	"fmt"
 	"os"
 	"testing"
 	"time"
 
 	"github.com/astaxie/beego/orm"
 	"github.com/vmware/harbor/models"
+	"github.com/vmware/harbor/utils"
 	"github.com/vmware/harbor/utils/log"
 )
 
@@ -45,41 +45,49 @@ func clearUp(username string) {
 	o := orm.NewOrm()
 	o.Begin()
 
-	err = execUpdate(o, `delete pm 
-		from project_member pm 
-		join user u 
-		on pm.user_id = u.user_id 
-		where u.username = ?`, username)
+	err = execUpdate(o, `delete 
+		from project_member 
+		where user_id = (
+			select user_id
+			from user
+			where username = ?
+		) `, username)
 	if err != nil {
 		o.Rollback()
 		log.Error(err)
 	}
 
-	err = execUpdate(o, `delete pm 
-		from project_member pm
-		join project p 
-		on pm.project_id = p.project_id 
-		where p.name = ?`, projectName)
+	err = execUpdate(o, `delete  
+		from project_member
+		where project_id = (
+			select project_id
+			from project
+			where name = ?
+		)`, projectName)
 	if err != nil {
 		o.Rollback()
 		log.Error(err)
 	}
 
-	err = execUpdate(o, `delete al 
-		from access_log al
-		join user u 
-		on al.user_id = u.user_id 
-		where u.username = ?`, username)
+	err = execUpdate(o, `delete 
+		from access_log 
+		where user_id = (
+			select user_id
+			from user
+			where username = ?
+		)`, username)
 	if err != nil {
 		o.Rollback()
 		log.Error(err)
 	}
 
-	err = execUpdate(o, `delete al 
-		from access_log al
-		join project p 
-		on al.project_id = p.project_id 
-		where p.name = ?`, projectName)
+	err = execUpdate(o, `delete 
+		from access_log
+		where project_id = (
+			select project_id
+			from project
+			where name = ?
+		)`, projectName)
 	if err != nil {
 		o.Rollback()
 		log.Error(err)
@@ -127,6 +135,31 @@ const publicityOn = 1
 const publicityOff = 0
 
 func TestMain(m *testing.M) {
+	databases := []string{"mysql", "sqlite"}
+	for _, database := range databases {
+		log.Infof("run test cases for database: %s", database)
+
+		result := 1
+		switch database {
+		case "mysql":
+			result = testForMySQL(m)
+		case "sqlite":
+			result = testForSQLite(m)
+		default:
+			log.Fatalf("invalid database: %s", database)
+		}
+
+		if result != 0 {
+			os.Exit(result)
+		}
+	}
+}
+
+func testForMySQL(m *testing.M) int {
+	db := os.Getenv("DATABASE")
+	defer os.Setenv("DATABASE", db)
+
+	os.Setenv("DATABASE", "mysql")
 
 	dbHost := os.Getenv("DB_HOST")
 	if len(dbHost) == 0 {
@@ -148,11 +181,51 @@ func TestMain(m *testing.M) {
 	os.Setenv("MYSQL_PORT", dbPort)
 	os.Setenv("MYSQL_USR", dbUser)
 	os.Setenv("MYSQL_PWD", dbPassword)
+
+	return testForAll(m)
+}
+
+func testForSQLite(m *testing.M) int {
+	db := os.Getenv("DATABASE")
+	defer os.Setenv("DATABASE", db)
+
+	os.Setenv("DATABASE", "sqlite")
+
+	file := os.Getenv("SQLITE_FILE")
+	if len(file) == 0 {
+		os.Setenv("SQLITE_FILE", "/registry.db")
+		defer os.Setenv("SQLITE_FILE", "")
+	}
+
+	return testForAll(m)
+}
+
+func testForAll(m *testing.M) int {
 	os.Setenv("AUTH_MODE", "db_auth")
-	InitDB()
+	initDatabaseForTest()
 	clearUp(username)
-	os.Exit(m.Run())
 
+	return m.Run()
+}
+
+var defaultRegistered = false
+
+func initDatabaseForTest() {
+	database, err := getDatabase()
+	if err != nil {
+		panic(err)
+	}
+
+	log.Infof("initializing database: %s", database.String())
+
+	alias := database.Name()
+	if !defaultRegistered {
+		defaultRegistered = true
+		alias = "default"
+	}
+	if err := database.Register(alias); err != nil {
+		panic(err)
+	}
 }
 
 func TestRegister(t *testing.T) {
@@ -332,12 +405,9 @@ func TestListUsers(t *testing.T) {
 }
 
 func TestResetUserPassword(t *testing.T) {
-	uuid, err := GenerateRandomString()
-	if err != nil {
-		t.Errorf("Error occurred in GenerateRandomString: %v", err)
-	}
+	uuid := utils.GenerateRandomString()
 
-	err = UpdateUserResetUUID(models.User{ResetUUID: uuid, Email: currentUser.Email})
+	err := UpdateUserResetUUID(models.User{ResetUUID: uuid, Email: currentUser.Email})
 	if err != nil {
 		t.Errorf("Error occurred in UpdateUserResetUuid: %v", err)
 	}
@@ -1494,39 +1564,6 @@ func TestGetOrmer(t *testing.T) {
 	}
 }
 
-func TestDeleteProject(t *testing.T) {
-	name := "project_for_test"
-	project := models.Project{
-		OwnerID: currentUser.UserID,
-		Name:    name,
-	}
-
-	id, err := AddProject(project)
-	if err != nil {
-		t.Fatalf("failed to add project: %v", err)
-	}
-
-	if err = DeleteProject(id); err != nil {
-		t.Fatalf("failed to delete project: %v", err)
-	}
-
-	p := &models.Project{}
-	if err = GetOrmer().Raw(`select * from project where project_id = ?`, id).
-		QueryRow(p); err != nil {
-		t.Fatalf("failed to get project: %v", err)
-	}
-
-	if p.Deleted != 1 {
-		t.Errorf("unexpeced deleted column: %d != %d", p.Deleted, 1)
-	}
-
-	deletedName := fmt.Sprintf("%s#%d", name, id)
-	if p.Name != deletedName {
-		t.Errorf("unexpected name: %s != %s", p.Name, deletedName)
-	}
-
-}
-
 func TestAddRepository(t *testing.T) {
 	repoRecord := models.RepoRecord{
 		Name:        currentProject.Name + "/" + repositoryName,

dao/mysql.go

+/*
+   Copyright (c) 2016 VMware, Inc. All Rights Reserved.
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package dao
+
+import (
+	"errors"
+	"fmt"
+	"net"
+
+	"time"
+
+	"github.com/astaxie/beego/orm"
+	_ "github.com/go-sql-driver/mysql" //register mysql driver
+	"github.com/vmware/harbor/utils/log"
+)
+
+type mysql struct {
+	host     string
+	port     string
+	usr      string
+	pwd      string
+	database string
+}
+
+// NewMySQL returns an instance of mysql
+func NewMySQL(host, port, usr, pwd, database string) Database {
+	return &mysql{
+		host:     host,
+		port:     port,
+		usr:      usr,
+		pwd:      pwd,
+		database: database,
+	}
+}
+
+// Register registers MySQL as the underlying database used
+func (m *mysql) Register(alias ...string) error {
+	if err := m.testConn(m.host, m.port); err != nil {
+		return err
+	}
+
+	if err := orm.RegisterDriver("mysql", orm.DRMySQL); err != nil {
+		return err
+	}
+
+	an := "default"
+	if len(alias) != 0 {
+		an = alias[0]
+	}
+	conn := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s", m.usr,
+		m.pwd, m.host, m.port, m.database)
+	return orm.RegisterDataBase(an, "mysql", conn)
+}
+
+func (m *mysql) testConn(host, port string) error {
+	ch := make(chan int, 1)
+	go func() {
+		var err error
+		var c net.Conn
+		for {
+			c, err = net.DialTimeout("tcp", host+":"+port, 20*time.Second)
+			if err == nil {
+				c.Close()
+				ch <- 1
+			} else {
+				log.Errorf("failed to connect to db, retry after 2 seconds :%v", err)
+				time.Sleep(2 * time.Second)
+			}
+		}
+	}()
+	select {
+	case <-ch:
+		return nil
+	case <-time.After(60 * time.Second):
+		return errors.New("failed to connect to database after 60 seconds")
+	}
+}
+
+// Name returns the name of MySQL
+func (m *mysql) Name() string {
+	return "MySQL"
+}
+
+// String returns the details of database
+func (m *mysql) String() string {
+	return fmt.Sprintf("type-%s host-%s port-%s user-%s database-%s",
+		m.Name(), m.host, m.port, m.usr, m.database)
+}

dao/project.go

@@ -279,9 +279,16 @@ func getProjects(userID int, name string, args ...int64) ([]models.Project, erro
 
 // DeleteProject ...
 func DeleteProject(id int64) error {
+	project, err := GetProjectByID(id)
+	if err != nil {
+		return err
+	}
+
+	name := fmt.Sprintf("%s#%d", project.Name, project.ProjectID)
+
 	sql := `update project 
-		set deleted = 1, name = concat(name,"#",project_id) 
+		set deleted = 1, name = ? 
 		where project_id = ?`
-	_, err := GetOrmer().Raw(sql, id).Exec()
+	_, err = GetOrmer().Raw(sql, name, id).Exec()
 	return err
 }

dao/project_test.go

+/*
+   Copyright (c) 2016 VMware, Inc. All Rights Reserved.
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package dao
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/vmware/harbor/models"
+)
+
+func TestDeleteProject(t *testing.T) {
+	name := "project_for_test"
+	project := models.Project{
+		OwnerID: currentUser.UserID,
+		Name:    name,
+	}
+
+	id, err := AddProject(project)
+	if err != nil {
+		t.Fatalf("failed to add project: %v", err)
+	}
+	defer func() {
+		if err := delProjPermanent(id); err != nil {
+			t.Errorf("failed to clear up project %d: %v", id, err)
+		}
+	}()
+
+	if err = DeleteProject(id); err != nil {
+		t.Fatalf("failed to delete project: %v", err)
+	}
+
+	p := &models.Project{}
+	if err = GetOrmer().Raw(`select * from project where project_id = ?`, id).
+		QueryRow(p); err != nil {
+		t.Fatalf("failed to get project: %v", err)
+	}
+
+	if p.Deleted != 1 {
+		t.Errorf("unexpeced deleted column: %d != %d", p.Deleted, 1)
+	}
+
+	deletedName := fmt.Sprintf("%s#%d", name, id)
+	if p.Name != deletedName {
+		t.Errorf("unexpected name: %s != %s", p.Name, deletedName)
+	}
+
+}
+
+func delProjPermanent(id int64) error {
+	_, err := GetOrmer().QueryTable("access_log").
+		Filter("ProjectID", id).
+		Delete()
+	if err != nil {
+		return err
+	}
+
+	_, err = GetOrmer().Raw(`delete from project_member 
+		where project_id = ?`, id).Exec()
+	if err != nil {
+		return err
+	}
+
+	_, err = GetOrmer().QueryTable("project").
+		Filter("ProjectID", id).
+		Delete()
+	return err
+}

dao/register.go

@@ -32,10 +32,7 @@ func Register(user models.User) (int64, error) {
 	}
 	defer p.Close()
 
-	salt, err := GenerateRandomString()
-	if err != nil {
-		return 0, err
-	}
+	salt := utils.GenerateRandomString()
 
 	now := time.Now()
 	r, err := p.Exec(user.Username, utils.Encrypt(user.Password, salt), user.Realname, user.Email, user.Comment, salt, user.HasAdminRole, now, now)

dao/replication_job.go

@@ -76,7 +76,8 @@ func DeleteRepTarget(id int64) error {
 // UpdateRepTarget ...
 func UpdateRepTarget(target models.RepTarget) error {
 	o := GetOrmer()
-	_, err := o.Update(&target, "URL", "Name", "Username", "Password")
+	target.UpdateTime = time.Now()
+	_, err := o.Update(&target, "URL", "Name", "Username", "Password", "UpdateTime")
 	return err
 }
 
@@ -105,18 +106,23 @@ func FilterRepTargets(name string) ([]*models.RepTarget, error) {
 // AddRepPolicy ...
 func AddRepPolicy(policy models.RepPolicy) (int64, error) {
 	o := GetOrmer()
-	sqlTpl := `insert into replication_policy (name, project_id, target_id, enabled, description, cron_str, start_time, creation_time, update_time ) values (?, ?, ?, ?, ?, ?, %s, NOW(), NOW())`
-	var sql string
-	if policy.Enabled == 1 {
-		sql = fmt.Sprintf(sqlTpl, "NOW()")
-	} else {
-		sql = fmt.Sprintf(sqlTpl, "NULL")
-	}
+	sql := `insert into replication_policy (name, project_id, target_id, enabled, description, cron_str, start_time, creation_time, update_time ) values (?, ?, ?, ?, ?, ?, ?, ?, ?)`
 	p, err := o.Raw(sql).Prepare()
 	if err != nil {
 		return 0, err
 	}
-	r, err := p.Exec(policy.Name, policy.ProjectID, policy.TargetID, policy.Enabled, policy.Description, policy.CronStr)
+
+	params := []interface{}{}
+	params = append(params, policy.Name, policy.ProjectID, policy.TargetID, policy.Enabled, policy.Description, policy.CronStr)
+	now := time.Now()
+	if policy.Enabled == 1 {
+		params = append(params, now)
+	} else {
+		params = append(params, nil)
+	}
+	params = append(params, now, now)
+
+	r, err := p.Exec(params...)
 	if err != nil {
 		return 0, err
 	}
@@ -241,7 +247,8 @@ func GetRepPolicyByProjectAndTarget(projectID, targetID int64) ([]*models.RepPol
 // UpdateRepPolicy ...
 func UpdateRepPolicy(policy *models.RepPolicy) error {
 	o := GetOrmer()
-	_, err := o.Update(policy, "TargetID", "Name", "Enabled", "Description", "CronStr")
+	policy.UpdateTime = time.Now()
+	_, err := o.Update(policy, "TargetID", "Name", "Enabled", "Description", "CronStr", "UpdateTime")
 	return err
 }
 
@@ -249,8 +256,9 @@ func UpdateRepPolicy(policy *models.RepPolicy) error {
 func DeleteRepPolicy(id int64) error {
 	o := GetOrmer()
 	policy := &models.RepPolicy{
-		ID:      id,
-		Deleted: 1,
+		ID:         id,
+		Deleted:    1,
+		UpdateTime: time.Now(),
 	}
 	_, err := o.Update(policy, "Deleted")
 	return err
@@ -260,8 +268,9 @@ func DeleteRepPolicy(id int64) error {
 func UpdateRepPolicyEnablement(id int64, enabled int) error {
 	o := GetOrmer()
 	p := models.RepPolicy{
-		ID:      id,
-		Enabled: enabled,
+		ID:         id,
+		Enabled:    enabled,
+		UpdateTime: time.Now(),
 	}
 
 	var err error
@@ -386,10 +395,11 @@ func DeleteRepJob(id int64) error {
 func UpdateRepJobStatus(id int64, status string) error {
 	o := GetOrmer()
 	j := models.RepJob{
-		ID:     id,
-		Status: status,
+		ID:         id,
+		Status:     status,
+		UpdateTime: time.Now(),
 	}
-	num, err := o.Update(&j, "Status")
+	num, err := o.Update(&j, "Status", "UpdateTime")
 	if num == 0 {
 		err = fmt.Errorf("Failed to update replication job with id: %d %s", id, err.Error())
 	}
@@ -399,8 +409,8 @@ func UpdateRepJobStatus(id int64, status string) error {
 // ResetRunningJobs update all running jobs status to pending
 func ResetRunningJobs() error {
 	o := GetOrmer()
-	sql := fmt.Sprintf("update replication_job set status = '%s' where status = '%s'", models.JobPending, models.JobRunning)
-	_, err := o.Raw(sql).Exec()
+	sql := fmt.Sprintf("update replication_job set status = '%s', update_time = ? where status = '%s'", models.JobPending, models.JobRunning)
+	_, err := o.Raw(sql, time.Now()).Exec()
 	return err
 }
 

dao/repository.go

@@ -17,6 +17,7 @@ package dao
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/astaxie/beego/orm"
 	"github.com/vmware/harbor/models"
@@ -27,9 +28,10 @@ func AddRepository(repo models.RepoRecord) error {
 	o := GetOrmer()
 	sql := "insert into repository (owner_id, project_id, name, description, pull_count, star_count, creation_time, update_time) " +
 		"select (select user_id as owner_id from user where username=?), " +
-		"(select project_id as project_id from project where name=?), ?, ?, ?, ?, NOW(), NULL "
+		"(select project_id as project_id from project where name=?), ?, ?, ?, ?, ?, NULL "
 
-	_, err := o.Raw(sql, repo.OwnerName, repo.ProjectName, repo.Name, repo.Description, repo.PullCount, repo.StarCount).Exec()
+	_, err := o.Raw(sql, repo.OwnerName, repo.ProjectName, repo.Name, repo.Description,
+		repo.PullCount, repo.StarCount, time.Now()).Exec()
 	return err
 }
 
@@ -62,6 +64,7 @@ func DeleteRepository(name string) error {
 // UpdateRepository ...
 func UpdateRepository(repo models.RepoRecord) error {
 	o := GetOrmer()
+	repo.UpdateTime = time.Now()
 	_, err := o.Update(&repo)
 	return err
 }
@@ -71,7 +74,8 @@ func IncreasePullCount(name string) (err error) {
 	o := GetOrmer()
 	num, err := o.QueryTable("repository").Filter("name", name).Update(
 		orm.Params{
-			"pull_count": orm.ColValue(orm.ColAdd, 1),
+			"pull_count":  orm.ColValue(orm.ColAdd, 1),
+			"update_time": time.Now(),
 		})
 	if num == 0 {
 		err = fmt.Errorf("Failed to increase repository pull count with name: %s %s", name, err.Error())

dao/sqlite.go

+/*
+   Copyright (c) 2016 VMware, Inc. All Rights Reserved.
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package dao
+
+import (
+	"fmt"
+
+	"github.com/astaxie/beego/orm"
+	_ "github.com/mattn/go-sqlite3" //register sqlite driver
+)
+
+type sqlite struct {
+	file string
+}
+
+// NewSQLite returns an instance of sqlite
+func NewSQLite(file string) Database {
+	return &sqlite{
+		file: file,
+	}
+}
+
+// Register registers SQLite as the underlying database used
+func (s *sqlite) Register(alias ...string) error {
+	if err := orm.RegisterDriver("sqlite3", orm.DRSqlite); err != nil {
+		return err
+	}
+
+	an := "default"
+	if len(alias) != 0 {
+		an = alias[0]
+	}
+	if err := orm.RegisterDataBase(an, "sqlite3", s.file); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Name returns the name of SQLite
+func (s *sqlite) Name() string {
+	return "SQLite"
+}
+
+// String returns the details of database
+func (s *sqlite) String() string {
+	return fmt.Sprintf("type-%s file:%s", s.Name(), s.file)
+}

dao/user.go

@@ -18,6 +18,7 @@ package dao
 import (
 	"database/sql"
 	"errors"
+	"fmt"
 
 	"github.com/vmware/harbor/models"
 	"github.com/vmware/harbor/utils"
@@ -214,10 +215,20 @@ func CheckUserPassword(query models.User) (*models.User, error) {
 // DeleteUser ...
 func DeleteUser(userID int) error {
 	o := GetOrmer()
-	_, err := o.Raw(`update user 
-		set deleted = 1, username = concat(username, "#", user_id),
-			email = concat(email, "#", user_id)
-		where user_id = ?`, userID).Exec()
+
+	user, err := GetUser(models.User{
+		UserID: userID,
+	})
+	if err != nil {
+		return err
+	}
+
+	name := fmt.Sprintf("%s#%d", user.Username, user.UserID)
+	email := fmt.Sprintf("%s#%d", user.Email, user.UserID)
+
+	_, err = o.Raw(`update user 
+		set deleted = 1, username = ?, email = ?
+		where user_id = ?`, name, email, userID).Exec()
 	return err
 }
 

jobservice/main.go

@@ -24,7 +24,7 @@ import (
 )
 
 func main() {
-	dao.InitDB()
+	dao.InitDatabase()
 	initRouters()
 	job.InitWorkerPool()
 	go job.Dispatch()

ui/main.go

@@ -19,16 +19,17 @@ import (
 	"fmt"
 	"os"
 
-	log "github.com/vmware/harbor/utils/log"
+	"github.com/vmware/harbor/utils"
+	"github.com/vmware/harbor/utils/log"
+
+	"github.com/astaxie/beego"
+	_ "github.com/astaxie/beego/session/redis"
 
 	"github.com/vmware/harbor/api"
 	_ "github.com/vmware/harbor/auth/db"
 	_ "github.com/vmware/harbor/auth/ldap"
 	"github.com/vmware/harbor/dao"
 	"github.com/vmware/harbor/models"
-
-	"github.com/astaxie/beego"
-	_ "github.com/astaxie/beego/session/redis"
 )
 
 const (
@@ -45,10 +46,7 @@ func updateInitPassword(userID int, password string) error {
 		return fmt.Errorf("User id: %d does not exist.", userID)
 	}
 	if user.Salt == "" {
-		salt, err := dao.GenerateRandomString()
-		if err != nil {
-			return fmt.Errorf("Failed to generate salt for encrypting password, %v", err)
-		}
+		salt := utils.GenerateRandomString()
 
 		user.Salt = salt
 		user.Password = password
@@ -75,7 +73,9 @@ func main() {
 	}
 	//
 	beego.AddTemplateExt("htm")
-	dao.InitDB()
+
+	dao.InitDatabase()
+
 	if err := updateInitPassword(adminUserID, os.Getenv("HARBOR_ADMIN_PASSWORD")); err != nil {
 		log.Error(err)
 	}

utils/utils.go

@@ -16,8 +16,10 @@
 package utils
 
 import (
+	"math/rand"
 	"net/url"
 	"strings"
+	"time"
 )
 
 // FormatEndpoint formats endpoint
@@ -56,3 +58,15 @@ func ParseRepository(repository string) (project, rest string) {
 	rest = repository[index+1:]
 	return
 }
+
+// GenerateRandomString generates a random string
+func GenerateRandomString() string {
+	length := 32
+	rand.Seed(time.Now().UTC().UnixNano())
+	const chars = "abcdefghijklmnopqrstuvwxyz0123456789"
+	result := make([]byte, length)
+	for i := 0; i < length; i++ {
+		result[i] = chars[rand.Intn(len(chars))]
+	}
+	return string(result)
+}

utils/utils_test.go

@@ -134,3 +134,10 @@ func TestReversibleEncrypt(t *testing.T) {
 		t.Errorf("decrypted password: %s, is not identical to original", decrypted)
 	}
 }
+
+func TestGenerateRandomString(t *testing.T) {
+	str := GenerateRandomString()
+	if len(str) != 32 {
+		t.Errorf("unexpected length: %d != %d", len(str), 32)
+	}
+}

vendor/github.com/mattn/go-sqlite3/LICENSE

+The MIT License (MIT)
+
+Copyright (c) 2014 Yasuhiro Matsumoto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.