Commit 94c56688 authored by zauberstuhl's avatar zauberstuhl

Merge branch 'move_to_gitlab' into 'master'

Move to gitlab

See merge request feneas/federation-testsuite-server!1
parents 19114edc a8bfd562
server.db
config.conf
......@@ -20,4 +20,4 @@ WORKDIR /home/user
EXPOSE 8181
CMD ["sh", "-c", "/usr/local/bin/github-integration --github-id ${GITHUB_ID} --github-secret ${GITHUB_SECRET} --server-domain ${DOMAIN} --travis-token ${TRAVIS_SECRET}"]
ENTRYPOINT ["/usr/local/bin/github-integration"]
//
// TheFederation Github Integration Server
// Copyright (C) 2018 Lukas Matt <lukas@zauberstuhl.de>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
package main
import "time"
func BuildAgent() {
logger.Println("Started build agent")
// if we still have some unfinished jobs start watching them
runAllJobsBelow(STATUS_SUCCESS)
for {
// sleep for a bit before continuing
time.Sleep(10 * time.Second)
runAllJobsBelow(STATUS_PENDING)
}
}
func runAllJobsBelow(status BuildStatus) {
db, err := OpenDatabase()
if err != nil {
panic(err.Error())
}
defer db.Close()
var builds Builds
err = db.Where("status < ?", status).Find(&builds).Error
if err != nil {
//logger.Printf("Cannot fetch new builds: %+v\n", err)
return
}
for _, build := range builds {
logger.Printf("#%d: starting new build\n", build.ID)
go build.Run()
}
}
This diff is collapsed.
......@@ -47,7 +47,9 @@ func (d DataSet) Less(i, j int) bool {
}
func buildsPNG(w http.ResponseWriter, r *http.Request) {
db, err := gorm.Open(databaseDriver, databaseDSN)
db, err := gorm.Open(
confd.StringDefault("database.driver", "sqlite3"),
confd.StringDefault("database.dsn", "./server.db"))
if err != nil {
logger.Println(err)
return
......@@ -67,9 +69,9 @@ func buildsPNG(w http.ResponseWriter, r *http.Request) {
for _, build := range builds {
year, month, day := build.CreatedAt.Date()
date := time.Date(year, month, day,0, 0, 0, 0, time.Local)
if build.Status == BUILD_FINISHED {
if build.Status == STATUS_SUCCESS {
passed[date] += 1
} else if build.Status == BUILD_FINISHED_ERROR {
} else if build.Status > STATUS_SUCCESS {
failed[date] += 1
}
}
......
[DEFAULT]
github.id = string
github.secret = string
gitlab.token.repository = string
gitlab.token.trigger = string
gitlab.server = https://git.feneas.org
gitlab.api = %(gitlab.server)s/api/v4
gitlab.project.id = 102
server.domain = string
status.name = Federation Suite
status.description = Continuous integration tests for the federation network
development = false
database.driver = sqlite3
database.dsn = ./server.db
......@@ -22,7 +22,6 @@ import (
"html/template"
"fmt"
"golang.org/x/oauth2"
"github.com/google/go-github/github"
"context"
"strings"
)
......@@ -31,9 +30,14 @@ func add(a, b int) int {
return a + b
}
func length(a []interface{}) int {
return len(a)
}
func render(w http.ResponseWriter, name string, s interface{}) {
rootTmpl := template.New("").Funcs(template.FuncMap{
"add": add,
"len": length,
})
tmpl, err := rootTmpl.ParseFiles(
......@@ -55,113 +59,92 @@ func render(w http.ResponseWriter, name string, s interface{}) {
}
func indexPage(w http.ResponseWriter, r *http.Request) {
var repos Repos
err := repos.FindAll()
var projects Projects
err := projects.FindAll()
if err != nil {
logger.Println(err)
render(w, "error.html", "No repositories found")
render(w, "error.html", "No projects found")
return
}
render(w, "index.html", repos)
render(w, "index.html", projects)
}
func resultPage(w http.ResponseWriter, r *http.Request) {
accessToken := r.URL.Query().Get("access_token")
accessToken := r.URL.Query().Get("accessToken")
server := r.URL.Query().Get("server")
repo := r.URL.Query().Get("repo")
project := r.URL.Query().Get("project")
if accessToken != "" && repo != "" && project != "" {
ctx := context.Background()
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: accessToken},
)
tc := oauth2.NewClient(ctx, ts)
client := github.NewClient(tc)
repoSlice := strings.Split(repo, "/")
if len(repoSlice) < 2 {
logger.Println("invalid repository string")
render(w, "error.html", "Invalid repository string")
return
}
projectName := r.URL.Query().Get("project")
secret := Secret(16)
repo := Repo{
Project: project,
Slug: repo,
Token: accessToken,
Secret: secret,
}
secret := Secret(16)
isGithub := accessToken != "" && repo != "" && projectName != ""
isGitlab := server != "" && isGithub
// set the repo to opt-in
if strings.ToUpper(r.URL.Query().Get("optin")) == "ON" {
repo.OptIn = true
}
// define custom opt-in flag
if r.URL.Query().Get("optinFlag") != "" {
repo.OptInFlag = r.URL.Query().Get("optinFlag")
}
// define custom opt-out flag
if r.URL.Query().Get("optoutFlag") != "" {
repo.OptOutFlag = r.URL.Query().Get("optoutFlag")
}
if !isGitlab && !isGithub {
render(w, "error.html", "Missing parameters: accessToken, server, repo or project")
return
}
name := "web"
newHook := github.Hook{
Name: &name, Events: []string{"pull_request"},
Config: map[string]interface{}{
"content_type": "json",
"url": serverDomain + "/hook",
"secret": secret,
},
}
if isGithub {
server = "https://github.com"
}
project := Project{
Name: projectName,
FQDN: server,
Slug: repo,
Token: accessToken,
Secret: secret,
OptIn: strings.ToUpper(r.URL.Query().Get("optin")) == "ON",
OptInFlag: r.URL.Query().Get("optinFlag"),
OptOutFlag: r.URL.Query().Get("optoutFlag"),
}
if isGitlab {
project.Type = GITLAB
if !devMode {
hooks, _, err := client.Repositories.ListHooks(ctx,
repoSlice[0], repoSlice[1], &github.ListOptions{})
err := createGitlabHook(project)
if err != nil {
logger.Println(err)
render(w, "error.html", "Cannot list repository hooks")
render(w, "error.html", err.Error())
return
}
}
} else if isGithub {
project.Type = GITHUB
var hookExists = false
hookURL, _ := newHook.Config["url"].(string)
for _, hook := range hooks {
curHookURL, _ := hook.Config["url"].(string)
if hookURL == curHookURL {
hookExists = true
}
}
if !hookExists {
_, _, err := client.Repositories.CreateHook(ctx,
repoSlice[0], repoSlice[1], &newHook)
if err != nil {
logger.Println(err)
render(w, "error.html", "Cannot create repository hooks")
return
}
if !devMode {
err := createGithubHook(project)
if err != nil {
logger.Println(err)
render(w, "error.html", err.Error())
return
}
}
err := repo.CreateOrUpdate()
if err != nil {
logger.Println(err)
render(w, "error.html", "Cannot insert/update the database record")
return
}
}
render(w, "result.html", repo.Slug)
} else {
render(w, "error.html", "Missing parameters: access_token, repo or project")
err := project.CreateOrUpdate()
if err != nil {
logger.Println(err)
render(w, "error.html", "Cannot insert/update the database record")
return
}
render(w, "result.html", struct{
Gitlab bool
Project Project
}{Gitlab: isGitlab, Project: project})
}
func authGitlabPage(w http.ResponseWriter, r *http.Request) {
render(w, "auth.html", struct{Gitlab bool}{Gitlab: true})
}
func authenticationPage(w http.ResponseWriter, r *http.Request) {
func authGithubPage(w http.ResponseWriter, r *http.Request) {
code := r.URL.Query().Get("code")
if code != "" {
tok, err := conf.Exchange(context.Background(), code)
tok, err := oauth2GithubConfig.Exchange(context.Background(), code)
if !devMode && err != nil {
render(w, "error.html", "Invalid token")
} else {
......@@ -169,10 +152,13 @@ func authenticationPage(w http.ResponseWriter, r *http.Request) {
if !devMode {
token = tok.AccessToken
}
render(w, "auth.html", token)
render(w, "auth.html", struct{
Gitlab bool
Token string
}{Gitlab: false, Token: token})
}
} else {
url := conf.AuthCodeURL("state", oauth2.AccessTypeOffline)
url := oauth2GithubConfig.AuthCodeURL("state", oauth2.AccessTypeOffline)
http.Redirect(w, r, url, http.StatusMovedPermanently)
}
}
// TheFederation Github Integration Server
// Copyright (C) 2018 Lukas Matt <lukas@zauberstuhl.de>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
package main
import (
"log"
"golang.org/x/oauth2"
oauth2Github "golang.org/x/oauth2/github"
"github.com/revel/config"
"math/rand"
"time"
"os"
"github.com/jinzhu/gorm"
_ "github.com/jinzhu/gorm/dialects/sqlite"
)
var (
configDir = "./"
confd *config.Context
devMode = false
logger = log.New(os.Stdout, "", log.Ldate | log.Ltime | log.Lmicroseconds | log.Lshortfile)
runes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890")
oauth2GithubConfig = &oauth2.Config{
Scopes: []string{"admin:repo_hook", "repo:status"},
Endpoint: oauth2Github.Endpoint,
}
)
type ServerType uint
const (
GITLAB ServerType = iota
GITHUB
)
type BuildStatus uint
const (
// The state of the status. Can be one of the following:
// pending, running, success, failed, canceled
STATUS_PENDING BuildStatus = 10 + iota
STATUS_RUNNING
STATUS_SUCCESS
STATUS_FAILED
STATUS_CANCELED
)
func init() {
rand.Seed(time.Now().UnixNano())
}
func LoadConfig() {
var err error
confd, err = config.LoadContext("config.conf", []string{configDir})
if err != nil {
panic(err.Error())
}
devMode = confd.BoolDefault("development", false)
oauth2GithubConfig.ClientID = confd.StringDefault("github.id", "")
oauth2GithubConfig.ClientSecret = confd.StringDefault("github.secret", "")
}
func OpenDatabase() (*gorm.DB, error) {
return gorm.Open(
confd.StringDefault("database.driver", "sqlite3"),
confd.StringDefault("database.dsn", "./server.db"))
}
func Secret(n int) string {
b := make([]rune, n)
for i := range b {
b[i] = runes[rand.Intn(len(runes))]
}
return string(b)
}
//
// TheFederation Github Integration Server
// Copyright (C) 2018 Lukas Matt <lukas@zauberstuhl.de>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
package main
import (
"net/url"
"net/http"
"strings"
"bytes"
"fmt"
)
func (project *Project) ApiRequest(method, path string, values url.Values) (*http.Response, error) {
endpoint := fmt.Sprintf("%s/api/v4%s", project.FQDN, path)
logger.Printf("http method=%s url=%s values=%+v\n", method, endpoint, values)
req, err := http.NewRequest(method, endpoint,
bytes.NewBufferString(values.Encode()))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("Accept", "application/json")
req.Header.Set("PRIVATE-TOKEN", project.Token)
client := &http.Client{}
return client.Do(req)
}
func test_server_request(method, path string, values url.Values) (*http.Response, error) {
endpoint := fmt.Sprintf("%s%s", confd.StringDefault("gitlab.api",
"https://git.feneas.org/api/v4"), path)
logger.Printf("http method=%s url=%s values=%+v\n", method, endpoint, values)
req, err := http.NewRequest(method, endpoint,
bytes.NewBufferString(values.Encode()))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("Accept", "application/json")
req.Header.Set("PRIVATE-TOKEN",
confd.StringDefault("gitlab.token.repository", ""))
client := &http.Client{}
return client.Do(req)
}
func test_server_pipeline(ref string, values url.Values) (*http.Response, error) {
newValues := url.Values{}
for key, value := range values {
newValues.Set(fmt.Sprintf("variables[%s]", key), strings.Join(value, ","))
}
newValues.Set("token", confd.StringDefault("gitlab.token.trigger", ""))
newValues.Set("ref", ref)
return http.PostForm(fmt.Sprintf("%s/projects/%d/trigger/pipeline",
confd.StringDefault("gitlab.api", "https://git.feneas.org/api/v4"),
confd.IntDefault("gitlab.project.id", 102)), newValues)
}
......@@ -22,42 +22,77 @@ import (
_ "github.com/jinzhu/gorm/dialects/sqlite"
)
type Repo struct {
func LoadSchema() {
db, err := OpenDatabase()
if err != nil {
panic(err.Error())
}
defer db.Close()
build := &Build{}
db.Model(build).AddUniqueIndex("index_builds_on_ids",
"project_id", "merge_request_id", "pipeline_id")
db.AutoMigrate(build)
project := &Project{}
db.Model(project).AddUniqueIndex(
"index_projects_on_fqdn_and_slug", "fqdn", "slug")
db.AutoMigrate(project)
mergeRequest := &MergeRequest{}
db.AutoMigrate(mergeRequest)
}
// merge request table
type MergeRequest struct {
gorm.Model
Project string
Slug string
Token string
Secret string
MergeRequestID uint
SourceBranch string
Sha, URL string
}
// project table
type Project struct {
gorm.Model
Name string
FQDN, Slug, Secret, Token string
Type ServerType
Active bool
OptIn bool
OptInFlag string `gorm:"default:'ci'"`
OptOutFlag string `gorm:"default:'ci skip'"`
}
type Repos []Repo
type Projects []Project
func (repos *Repos) FindAll() error {
db, err := gorm.Open(databaseDriver, databaseDSN)
func (projects *Projects) FindAll() error {
db, err := OpenDatabase()
if err != nil {
return err
}
defer db.Close()
return db.Find(repos).Error
return db.Find(projects).Error
}
func (repo *Repo) CreateOrUpdate() error {
db, err := gorm.Open(databaseDriver, databaseDSN)
func (project *Project) CreateOrUpdate() error {
db, err := OpenDatabase()
if err != nil {
return err
}
defer db.Close()
var oldRecord Repo
err = db.Where(
"project = ? and slug = ?", repo.Project, repo.Slug,
).Find(&oldRecord).Error
var oldRecord Project
err = db.Where("name = ? and fqdn = ? and slug = ?",
project.Name, project.FQDN, project.Slug).Find(&oldRecord).Error
if err == gorm.ErrRecordNotFound {
err = db.Create(repo).Error
err = db.Create(project).Error
if err != nil {
return err
}
......@@ -65,12 +100,45 @@ func (repo *Repo) CreateOrUpdate() error {
// NOTE you have to specify opt_in as extra update field
// since gorm will not update if bool is false
// see http://gorm.io/docs/update.html
repo.Secret = "" // set to default then it won't update
err = db.Model(repo).Where("id = ?", oldRecord.ID).
Update(repo).Update("opt_in", repo.OptIn).Error
project.Secret = "" // set to default then it won't update
err = db.Model(project).Where("id = ?", oldRecord.ID).
Update(project).Update("opt_in", project.OptIn).Error
if err != nil {
return err
}
}
return err
}
// build table
type Build struct {
gorm.Model
MergeRequestID, ProjectID uint
PipelineID uint
Status BuildStatus
Project Project `gorm:"ForeignKey:ProjectID"`
MergeRequest MergeRequest `gorm:"ForeignKey:MergeRequestID"`
}
type Builds []Build
func (build *Build) AfterFind(db *gorm.DB) error {
err := db.Model(build).Related(&build.Project).Error
if err != nil {
return err
}
return db.Model(build).Related(&build.MergeRequest).Error
}
func (build *Build) Save() error {
db, err := OpenDatabase()
if err != nil {
return err
}
defer db.Close()
return db.Save(build).Error
}
......@@ -18,87 +18,32 @@
package main
import (
"log"
"net/http"
"golang.org/x/oauth2"
oauth2Github "golang.org/x/oauth2/github"
"time"
"math/rand"
"github.com/jinzhu/gorm"
_ "github.com/jinzhu/gorm/dialects/sqlite"
"flag"
"os"
)
var (