Move to gitlab

.gitignore

 server.db
+config.conf

Dockerfile

@@ -20,4 +20,4 @@ WORKDIR /home/user
 
 EXPOSE 8181
 
-CMD ["sh", "-c", "/usr/local/bin/github-integration --github-id ${GITHUB_ID} --github-secret ${GITHUB_SECRET} --server-domain ${DOMAIN} --travis-token ${TRAVIS_SECRET}"]
+ENTRYPOINT ["/usr/local/bin/github-integration"]

agent.go

+//
+// TheFederation Github Integration Server
+// Copyright (C) 2018 Lukas Matt <lukas@zauberstuhl.de>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+//
+package main
+
+import "time"
+
+func BuildAgent() {
+  logger.Println("Started build agent")
+
+  // if we still have some unfinished jobs start watching them
+  runAllJobsBelow(STATUS_SUCCESS)
+
+  for {
+    // sleep for a bit before continuing
+    time.Sleep(10 * time.Second)
+
+    runAllJobsBelow(STATUS_PENDING)
+  }
+}
+
+func runAllJobsBelow(status BuildStatus) {
+  db, err := OpenDatabase()
+  if err != nil {
+    panic(err.Error())
+  }
+  defer db.Close()
+
+  var builds Builds
+  err = db.Where("status < ?", status).Find(&builds).Error
+  if err != nil {
+    //logger.Printf("Cannot fetch new builds: %+v\n", err)
+    return
+  }
+
+  for _, build := range builds {
+    logger.Printf("#%d: starting new build\n", build.ID)
+    go build.Run()
+  }
+}

charts.go

@@ -47,7 +47,9 @@ func (d DataSet) Less(i, j int) bool {
 }
 
 func buildsPNG(w http.ResponseWriter, r *http.Request) {
-  db, err := gorm.Open(databaseDriver, databaseDSN)
+  db, err := gorm.Open(
+    confd.StringDefault("database.driver", "sqlite3"),
+    confd.StringDefault("database.dsn", "./server.db"))
   if err != nil {
     logger.Println(err)
     return
@@ -67,9 +69,9 @@ func buildsPNG(w http.ResponseWriter, r *http.Request) {
   for _, build := range builds {
     year, month, day := build.CreatedAt.Date()
     date := time.Date(year, month, day,0, 0, 0, 0, time.Local)
-    if build.Status == BUILD_FINISHED {
+    if build.Status == STATUS_SUCCESS {
       passed[date] += 1
-    } else if build.Status == BUILD_FINISHED_ERROR {
+    } else if build.Status > STATUS_SUCCESS {
       failed[date] += 1
     }
   }

config.conf.example

+[DEFAULT]
+
+github.id = string
+github.secret = string
+
+gitlab.token.repository = string
+gitlab.token.trigger = string
+gitlab.server = https://git.feneas.org
+gitlab.api = %(gitlab.server)s/api/v4
+gitlab.project.id = 102
+
+server.domain = string
+status.name = Federation Suite
+status.description = Continuous integration tests for the federation network
+development = false
+database.driver = sqlite3
+database.dsn = ./server.db

frontend.go

@@ -22,7 +22,6 @@ import (
   "html/template"
   "fmt"
   "golang.org/x/oauth2"
-  "github.com/google/go-github/github"
   "context"
   "strings"
 )
@@ -31,9 +30,14 @@ func add(a, b int) int {
   return a + b
 }
 
+func length(a []interface{}) int {
+  return len(a)
+}
+
 func render(w http.ResponseWriter, name string, s interface{}) {
   rootTmpl := template.New("").Funcs(template.FuncMap{
     "add": add,
+    "len": length,
   })
 
   tmpl, err := rootTmpl.ParseFiles(
@@ -55,113 +59,92 @@ func render(w http.ResponseWriter, name string, s interface{}) {
 }
 
 func indexPage(w http.ResponseWriter, r *http.Request) {
-  var repos Repos
-  err := repos.FindAll()
+  var projects Projects
+  err := projects.FindAll()
   if err != nil {
     logger.Println(err)
-    render(w, "error.html", "No repositories found")
+    render(w, "error.html", "No projects found")
     return
   }
 
-  render(w, "index.html", repos)
+  render(w, "index.html", projects)
 }
 
 func resultPage(w http.ResponseWriter, r *http.Request) {
-  accessToken := r.URL.Query().Get("access_token")
+  accessToken := r.URL.Query().Get("accessToken")
+  server := r.URL.Query().Get("server")
   repo := r.URL.Query().Get("repo")
-  project := r.URL.Query().Get("project")
-
-  if accessToken != "" && repo != "" && project != "" {
-    ctx := context.Background()
-    ts := oauth2.StaticTokenSource(
-      &oauth2.Token{AccessToken: accessToken},
-    )
-    tc := oauth2.NewClient(ctx, ts)
-    client := github.NewClient(tc)
-
-    repoSlice := strings.Split(repo, "/")
-    if len(repoSlice) < 2 {
-      logger.Println("invalid repository string")
-      render(w, "error.html", "Invalid repository string")
-      return
-    }
+  projectName := r.URL.Query().Get("project")
 
-    secret := Secret(16)
-    repo := Repo{
-      Project: project,
-      Slug: repo,
-      Token: accessToken,
-      Secret: secret,
-    }
+  secret := Secret(16)
+  isGithub := accessToken != "" && repo != "" && projectName != ""
+  isGitlab := server != "" && isGithub
 
-    // set the repo to opt-in
-    if strings.ToUpper(r.URL.Query().Get("optin")) == "ON" {
-      repo.OptIn = true
-    }
-    // define custom opt-in flag
-    if r.URL.Query().Get("optinFlag") != "" {
-      repo.OptInFlag = r.URL.Query().Get("optinFlag")
-    }
-    // define custom opt-out flag
-    if r.URL.Query().Get("optoutFlag") != "" {
-      repo.OptOutFlag = r.URL.Query().Get("optoutFlag")
-    }
+  if !isGitlab && !isGithub {
+    render(w, "error.html", "Missing parameters: accessToken, server, repo or project")
+    return
+  }
 
-    name := "web"
-    newHook := github.Hook{
-      Name: &name, Events: []string{"pull_request"},
-      Config: map[string]interface{}{
-        "content_type": "json",
-        "url": serverDomain + "/hook",
-        "secret": secret,
-      },
-    }
+  if isGithub {
+    server = "https://github.com"
+  }
+
+  project := Project{
+    Name: projectName,
+    FQDN: server,
+    Slug: repo,
+    Token: accessToken,
+    Secret: secret,
+    OptIn: strings.ToUpper(r.URL.Query().Get("optin")) == "ON",
+    OptInFlag: r.URL.Query().Get("optinFlag"),
+    OptOutFlag: r.URL.Query().Get("optoutFlag"),
+  }
+
+  if isGitlab {
+    project.Type = GITLAB
 
     if !devMode {
-      hooks, _, err := client.Repositories.ListHooks(ctx,
-        repoSlice[0], repoSlice[1], &github.ListOptions{})
+      err := createGitlabHook(project)
       if err != nil {
         logger.Println(err)
-        render(w, "error.html", "Cannot list repository hooks")
+        render(w, "error.html", err.Error())
         return
       }
+    }
+  } else if isGithub {
+    project.Type = GITHUB
 
-      var hookExists = false
-      hookURL, _ := newHook.Config["url"].(string)
-      for _, hook := range hooks {
-        curHookURL, _ := hook.Config["url"].(string)
-        if hookURL == curHookURL {
-          hookExists = true
-        }
-      }
-
-      if !hookExists {
-        _, _, err := client.Repositories.CreateHook(ctx,
-          repoSlice[0], repoSlice[1], &newHook)
-        if err != nil {
-          logger.Println(err)
-          render(w, "error.html", "Cannot create repository hooks")
-          return
-        }
+    if !devMode {
+      err := createGithubHook(project)
+      if err != nil {
+        logger.Println(err)
+        render(w, "error.html", err.Error())
+        return
       }
     }
-    err := repo.CreateOrUpdate()
-    if err != nil {
-      logger.Println(err)
-      render(w, "error.html", "Cannot insert/update the database record")
-      return
-    }
+  }
 
-    render(w, "result.html", repo.Slug)
-  } else {
-    render(w, "error.html", "Missing parameters: access_token, repo or project")
+  err := project.CreateOrUpdate()
+  if err != nil {
+    logger.Println(err)
+    render(w, "error.html", "Cannot insert/update the database record")
+    return
   }
+
+  render(w, "result.html", struct{
+    Gitlab bool
+    Project Project
+  }{Gitlab: isGitlab, Project: project})
+}
+
+func authGitlabPage(w http.ResponseWriter, r *http.Request) {
+  render(w, "auth.html", struct{Gitlab bool}{Gitlab: true})
 }
 
-func authenticationPage(w http.ResponseWriter, r *http.Request) {
+func authGithubPage(w http.ResponseWriter, r *http.Request) {
   code := r.URL.Query().Get("code")
   if code != "" {
-    tok, err := conf.Exchange(context.Background(), code)
+    tok, err := oauth2GithubConfig.Exchange(context.Background(), code)
     if !devMode && err != nil {
       render(w, "error.html", "Invalid token")
     } else {
@@ -169,10 +152,13 @@ func authenticationPage(w http.ResponseWriter, r *http.Request) {
       if !devMode {
         token = tok.AccessToken
       }
-      render(w, "auth.html", token)
+      render(w, "auth.html", struct{
+        Gitlab bool
+        Token string
+      }{Gitlab: false, Token: token})
     }
   } else {
-    url := conf.AuthCodeURL("state", oauth2.AccessTypeOffline)
+    url := oauth2GithubConfig.AuthCodeURL("state", oauth2.AccessTypeOffline)
     http.Redirect(w, r, url, http.StatusMovedPermanently)
   }
 }

global.go

+// TheFederation Github Integration Server
+// Copyright (C) 2018 Lukas Matt <lukas@zauberstuhl.de>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+//
+package main
+
+import (
+  "log"
+  "golang.org/x/oauth2"
+  oauth2Github "golang.org/x/oauth2/github"
+  "github.com/revel/config"
+  "math/rand"
+  "time"
+  "os"
+  "github.com/jinzhu/gorm"
+  _ "github.com/jinzhu/gorm/dialects/sqlite"
+)
+
+var (
+  configDir = "./"
+  confd *config.Context
+
+  devMode = false
+  logger = log.New(os.Stdout, "", log.Ldate | log.Ltime | log.Lmicroseconds | log.Lshortfile)
+  runes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890")
+
+  oauth2GithubConfig = &oauth2.Config{
+    Scopes: []string{"admin:repo_hook", "repo:status"},
+    Endpoint: oauth2Github.Endpoint,
+  }
+)
+
+type ServerType uint
+
+const (
+  GITLAB ServerType = iota
+  GITHUB
+)
+
+type BuildStatus uint
+
+const (
+  // The state of the status. Can be one of the following:
+  // pending, running, success, failed, canceled
+  STATUS_PENDING BuildStatus = 10 + iota
+  STATUS_RUNNING
+  STATUS_SUCCESS
+  STATUS_FAILED
+  STATUS_CANCELED
+)
+
+func init() {
+  rand.Seed(time.Now().UnixNano())
+}
+
+func LoadConfig() {
+  var err error
+  confd, err = config.LoadContext("config.conf", []string{configDir})
+  if err != nil {
+    panic(err.Error())
+  }
+
+  devMode = confd.BoolDefault("development", false)
+  oauth2GithubConfig.ClientID = confd.StringDefault("github.id", "")
+  oauth2GithubConfig.ClientSecret = confd.StringDefault("github.secret", "")
+}
+
+func OpenDatabase() (*gorm.DB, error) {
+  return gorm.Open(
+    confd.StringDefault("database.driver", "sqlite3"),
+    confd.StringDefault("database.dsn", "./server.db"))
+}
+
+func Secret(n int) string {
+  b := make([]rune, n)
+  for i := range b {
+    b[i] = runes[rand.Intn(len(runes))]
+  }
+  return string(b)
+}

http.go

+//
+// TheFederation Github Integration Server
+// Copyright (C) 2018 Lukas Matt <lukas@zauberstuhl.de>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+//
+package main
+
+import (
+  "net/url"
+  "net/http"
+  "strings"
+  "bytes"
+  "fmt"
+)
+
+func (project *Project) ApiRequest(method, path string, values url.Values) (*http.Response, error) {
+  endpoint := fmt.Sprintf("%s/api/v4%s", project.FQDN, path)
+
+  logger.Printf("http method=%s url=%s values=%+v\n", method, endpoint, values)
+
+  req, err := http.NewRequest(method, endpoint,
+    bytes.NewBufferString(values.Encode()))
+  if err != nil {
+    return nil, err
+  }
+
+  req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+  req.Header.Set("Accept", "application/json")
+  req.Header.Set("PRIVATE-TOKEN", project.Token)
+
+  client := &http.Client{}
+  return client.Do(req)
+}
+
+func test_server_request(method, path string, values url.Values) (*http.Response, error) {
+  endpoint := fmt.Sprintf("%s%s", confd.StringDefault("gitlab.api",
+    "https://git.feneas.org/api/v4"), path)
+
+  logger.Printf("http method=%s url=%s values=%+v\n", method, endpoint, values)
+
+  req, err := http.NewRequest(method, endpoint,
+    bytes.NewBufferString(values.Encode()))
+  if err != nil {
+    return nil, err
+  }
+  req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+  req.Header.Set("Accept", "application/json")
+  req.Header.Set("PRIVATE-TOKEN",
+    confd.StringDefault("gitlab.token.repository", ""))
+
+  client := &http.Client{}
+  return client.Do(req)
+}
+
+func test_server_pipeline(ref string, values url.Values) (*http.Response, error) {
+  newValues := url.Values{}
+  for key, value := range values {
+    newValues.Set(fmt.Sprintf("variables[%s]", key), strings.Join(value, ","))
+  }
+  newValues.Set("token", confd.StringDefault("gitlab.token.trigger", ""))
+  newValues.Set("ref", ref)
+
+  return http.PostForm(fmt.Sprintf("%s/projects/%d/trigger/pipeline",
+    confd.StringDefault("gitlab.api", "https://git.feneas.org/api/v4"),
+    confd.IntDefault("gitlab.project.id", 102)), newValues)
+}

repo.go → schema.go

@@ -22,42 +22,77 @@ import (
   _ "github.com/jinzhu/gorm/dialects/sqlite"
 )
 
-type Repo struct {
+func LoadSchema() {
+  db, err := OpenDatabase()
+  if err != nil {
+    panic(err.Error())
+  }
+  defer db.Close()
+
+  build := &Build{}
+  db.Model(build).AddUniqueIndex("index_builds_on_ids",
+    "project_id", "merge_request_id", "pipeline_id")
+  db.AutoMigrate(build)
+
+  project := &Project{}
+  db.Model(project).AddUniqueIndex(
+    "index_projects_on_fqdn_and_slug", "fqdn", "slug")
+  db.AutoMigrate(project)
+
+  mergeRequest := &MergeRequest{}
+  db.AutoMigrate(mergeRequest)
+}
+
+// merge request table
+
+type MergeRequest struct {
   gorm.Model
-  Project string
-  Slug string
-  Token string
-  Secret string
+  MergeRequestID uint
+  SourceBranch string
+  Sha, URL string
+}
+
+// project table
+
+type Project struct {
+  gorm.Model
+
+  Name string
+
+  FQDN, Slug, Secret, Token string
+  Type ServerType
+
+  Active bool
+
   OptIn bool
   OptInFlag string `gorm:"default:'ci'"`
   OptOutFlag string `gorm:"default:'ci skip'"`
 }
 
-type Repos []Repo
+type Projects []Project
 
-func (repos *Repos) FindAll() error {
-  db, err := gorm.Open(databaseDriver, databaseDSN)
+func (projects *Projects) FindAll() error {
+  db, err := OpenDatabase()
   if err != nil {
     return err
   }
   defer db.Close()
 
-  return db.Find(repos).Error
+  return db.Find(projects).Error
 }
 
-func (repo *Repo) CreateOrUpdate() error {
-  db, err := gorm.Open(databaseDriver, databaseDSN)
+func (project *Project) CreateOrUpdate() error {
+  db, err := OpenDatabase()
   if err != nil {
     return err
   }
   defer db.Close()
 
-  var oldRecord Repo
-  err = db.Where(
-    "project = ? and slug = ?", repo.Project, repo.Slug,
-  ).Find(&oldRecord).Error
+  var oldRecord Project
+  err = db.Where("name = ? and fqdn = ? and slug = ?",
+    project.Name, project.FQDN, project.Slug).Find(&oldRecord).Error
   if err == gorm.ErrRecordNotFound {
-    err = db.Create(repo).Error
+    err = db.Create(project).Error
     if err != nil {
       return err
     }
@@ -65,12 +100,45 @@ func (repo *Repo) CreateOrUpdate() error {
     // NOTE you have to specify opt_in as extra update field
     // since gorm will not update if bool is false
     // see http://gorm.io/docs/update.html
-    repo.Secret = "" // set to default then it won't update
-    err = db.Model(repo).Where("id = ?", oldRecord.ID).
-      Update(repo).Update("opt_in", repo.OptIn).Error
+    project.Secret = "" // set to default then it won't update
+    err = db.Model(project).Where("id = ?", oldRecord.ID).
+      Update(project).Update("opt_in", project.OptIn).Error
     if err != nil {
       return err
     }
   }
   return err
 }
+
+// build table
+
+type Build struct {
+  gorm.Model
+  MergeRequestID, ProjectID uint
+
+  PipelineID uint
+  Status BuildStatus
+
+  Project Project `gorm:"ForeignKey:ProjectID"`
+  MergeRequest MergeRequest `gorm:"ForeignKey:MergeRequestID"`
+}
+
+type Builds []Build
+
+func (build *Build) AfterFind(db *gorm.DB) error {
+  err := db.Model(build).Related(&build.Project).Error
+  if err != nil {
+    return err
+  }
+  return db.Model(build).Related(&build.MergeRequest).Error
+}
+
+func (build *Build) Save() error {
+  db, err := OpenDatabase()
+  if err != nil {
+    return err
+  }
+  defer db.Close()
+
+  return db.Save(build).Error
+}

server.go

@@ -18,87 +18,32 @@
 package main
 
 import (
-  "log"
-  "net/http"
-  "golang.org/x/oauth2"
-  oauth2Github "golang.org/x/oauth2/github"
-  "time"
-  "math/rand"
-  "github.com/jinzhu/gorm"
-  _ "github.com/jinzhu/gorm/dialects/sqlite"
   "flag"
-  "os"
-)
-
-var (