Commit 73d31d69 authored by Lukas Matt's avatar Lukas Matt

Merge branch 'add_testproject' into 'master'

Add testproject

See merge request !7
parents e44464d9 9c7ea258
Pipeline #881 failed with stage
in 1 minute and 9 seconds
image: debian
stages:
- build
- test
- deploy
.job_template: &job_definition
image: debian
stage: test
artifacts:
reports:
......@@ -20,6 +21,16 @@ stages:
variables:
- $CI_PIPELINE_TRIGGERED
build docker images:
stage: build
before_script:
- apt-get update && apt-get install -y curl
- curl -L https://get.docker.com | bash -
script:
- bash scripts/build_docker_image.sh
only:
- tags
single project:
<<: *job_definition
only:
......@@ -46,6 +57,12 @@ test socialhome:
variables:
PROJECT: socialhome
test testproject:
<<: *job_definition
<<: *no_pipeline
variables:
PROJECT: testproject
pages:
<<: *no_pipeline
image: ruby:2.3
......@@ -61,3 +78,5 @@ pages:
artifacts:
paths:
- public
only:
- master
# Federation Tests
You'll find the documentation here: [feneas.git.feneas.org/federation-testsuite](https://feneas.git.feneas.org/federation-testsuite/)
You'll find the documentation here: [feneas.git.feneas.org/federation/testsuite](https://feneas.git.feneas.org/federation/testsuite/)
......@@ -13,7 +13,7 @@ load ganggo_helper
}
@test "$btf start diaspora#1 server" {
start_app "d1" "3000" "testing_diaspora"$(latest_tag "diaspora")
start_app "d1" "3000" "diaspora"$(latest_tag "diaspora")
[ "$?" -eq 0 ]
code=$(wait_for "docker logs $(container_id "d1")" "Starting Diaspora in production")
echo "expected 0, got $code"
......
......@@ -2,21 +2,35 @@ nav:
- page: Federation Tests
permalink: "#federation-tests"
subnav:
- page: Projects
permalink: "#projects"
- page: Add your project
permalink: "#add-your-project"
- page: Development
permalink: "#development"
- page: Vocabulary
permalink: "#vocabulary"
- page: Workflow
permalink: "#workflow"
- page: Testsuite
permalink: "#testsuite"
subnav:
- page: Dependencies
permalink: "#dependencies"
- page: Run tests
permalink: "#run-tests"
- page: Run images
permalink: "#run-images"
- page: Docker Image
permalink: "#docker-image"
subnav:
- page: Environment
permalink: "#environment"
- page: Dockerfile
permalink: "#dockerfile"
- page: "entrypoint.sh"
permalink: "#entrypointsh"
- page: Building
permalink: "#building"
- page: Testing
permalink: "#testing"
subnav:
- page: Run BATS
permalink: "#run-bats"
- page: Testsuite Server
permalink: "#testsuite-server"
baseurl: "/federation-testsuite"
baseurl: "/federation/testsuite"
url: "https://feneas.git.feneas.org"
markdown: kramdown
......
<div class="row alert alert-info">
<div class="eleven columns">
<p markdown="1">{{ include.text }}</p>
</div>
<div class="one columns">
<div data-icon="ei-exclamation" data-size="s"></div>
</div>
</div>
......@@ -12,6 +12,13 @@
<ol>
{% for s in n.subnav %}
<li><a href="{{ s.permalink }}">{{ s.page }}</a></li>
{% if s.subnav.size > 0 %}
<ol>
{% for ss in s.subnav %}
<li><a href="{{ ss.permalink }}">{{ ss.page }}</a></li>
{% endfor %}
</ol>
{% endif %}
{% endfor %}
</ol>
{% endif %}
......
......@@ -60,6 +60,32 @@ $on-laptop: 780px;
}
.alert-info {
color: #0c5460;
background-color: #d1ecf1;
border-color: #bee5eb;
}
.alert p {
margin: 0;
}
.alert {
position: relative;
padding: .75rem 1.25rem;
margin-bottom: 1rem;
border: 1px solid transparent;
border-top-color: transparent;
border-right-color: transparent;
border-bottom-color: transparent;
border-left-color: transparent;
border-radius: .25rem;
}
*, ::after, ::before {
box-sizing: border-box;
}
@import
"normalize",
"base",
......
......@@ -5,128 +5,322 @@ title: Home
# Federation Tests
[![](https://testsuite.feneas.org/images/stats/builds.png)](https://testsuite.feneas.org)
With the [testsuite](https://testsuite.feneas.org/) you can write simple unit tests ensuring that your server is actually capable of sending/receiving federated payloads.
This repository is about automating federation tests in the network itself!
Maybe you want to checkout the [introduction post](https://feneas.org/federation-testsuite/) before continuing!
* [the-federation.info](https://the-federation.info/)
* [testsuite.feneas.org](https://testsuite.feneas.org/)
## Vocabulary
## Projects
* Testsuite: TS
* Testsuite Server: TSS
* Merge-Request: MR
* Pull-Request: PR
* Bash Automated Testing System: BATS
Following projects are using the testsuite:
## Workflow
* [GangGo](https://github.com/ganggo)
* [Socialhome](https://github.com/jaywink/socialhome)
<div class="row">
<div class="six columns" markdown="1">
## Add your project
[![testsuite-and-server.png]({{"/assets/img/testsuite-and-server.png" | prepend:site.baseurl }})]({{"/assets/img/testsuite-and-server.png" | prepend:site.baseurl }})
Clone this repository and create a new directory:
</div>
<div class="six columns" markdown="1">
git clone https://git.feneas.org/feneas/federation-testsuite.git
cd federation-testsuite && mkdir <project-name>
In this folder you have to put everything which is required for building an automated docker image.
Starting with a `Dockerfile` and a start-up script e.g. `start.sh`
1. MR was created
* Webhook is triggered (to testsuite.feneas.org/hook)
2. Testsuite Server
* Parsing and validating Webhook
* Triggering new TS build
3. Testsuite
* Running tests for specified project
* Reporting results to TSS
4. Testsuite Server
* Evaluate and store test results
* Reporting results to MR
The start-up script is required so that we can do some configuration magic at boot time.
</div>
</div>
# Testsuite
Say we have a project called `testproject` and it can communicate with other `testproject` instances.
If we have two servers running on `172.17.0.4:3000` and `172.17.0.5:3000` we can send `172.17.0.4`
a simple message by executing following command:
curl http://172.17.0.5:3000/add@172.17.0.4:3000
If we ask the server directly whether it received a message or not:
curl http://172.17.0.4:3000
it should return one, otherwise it will return zero!
**Getting started**
You can answer following questions with the testsuite now:
1. can we send a message via 172.17.0.5
2. was the message received on 172.17.0.4
## Dependencies
In case you want to test specific parts locally you need `docker`.
Make sure it is up and running:
sudo systemctl start docker.service
Then you can install the testsuite dependencies:
* github.com/stedolan/jq
* github.com/sstephenson/bats
Simply run the install script or do it manually:
bash scripts/install.sh
## Docker Image
Before we can start testing we need a docker image and a custom entrypoint.
You can use your own `Dockerfile` but we need a special entrypoint in case
a user creates a new PR via GitHub or MR via GitLab! Then the image will
start running with special variables.
In our case it will set `PRREPO` to the repository URL of the user and `PRSHA` to the commit SHA to be tested with the suite.
Start by cloning/forking the testsuite first:
git clone https://git.feneas.org/feneas/federation/testsuite.git
Switch into the repository and create a new folder called `testproject`:
cd testsuite
mkdir testproject
cd testproject
### Environment
Following environment variables will be available while tests are running:
$DATABASE (on every run)
$PORT (on every run)
$PROJECT (only on pull_requests from a user)
$PRREPO (only on pull_requests from a user)
$PRSHA (only on pull_requests from a user)
$(hostname -i) (if you need the container IP address)
This information we can use in our start-up script mentioned above.
For example if `PRREPO` and `PRSHA` is set we probably want to checkout the source code of the pull request first.
### Dockerfile
See for reference the start-up script of the ganggo image:
For our `testproject` that could look something like this:
```
FROM debian:jessie
RUN apt-get update
RUN apt-get install -y \
git-core postgresql-client netcat curl
RUN git clone https://git.feneas.org/feneas/federation/testsuite-testproject.git
WORKDIR testsuite-testproject
ADD entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]
```
### entrypoint.sh
Now for our special entrypoint you can see it will change the source branch if a custom build is used:
```
#!/bin/bash
repo=$GOPATH/src/github.com/ganggo/ganggo
if [ ! -z ${PRSHA} ]; then
if [ "$(basename $PRREPO)" == "ganggo.git" ]; then
cd $repo && git stash \
&& git remote add custom $PRREPO \
&& git fetch custom \
&& git checkout $PRSHA \
&& git log -1 || {
echo "Cannot find $PRREPO $PRSHA"
exit 1
}
fi
if [ "$(basename $PRREPO)" == "federation.git" ]; then
wd=$repo/vendor/github.com/ganggo/federation
rm -r $wd && git clone $PRREPO $wd \
&& cd $wd \
&& git checkout $PRSHA \
&& git log -1 || {
echo "Cannot checkout $PRREPO $PRSHA"
exit 1
}
fi
if [ "$PROJECT" == "testproject" ]; then
git checkout . \
&& git remote add custom $PRREPO \
&& git fetch custom \
&& git merge -m merge $PRSHA \
&& git log -1 || {
echo "Cannot pull from repo=$PRREPO sha=$PRSHA"
exit 1
}
fi
sed -i "s/NAME/$DATABASE/g" $repo/conf/app.conf \
&& sed -i "s/PORT/$PORT/g" $repo/conf/app.conf \
&& revel run github.com/ganggo/ganggo
./testproject.sh $DATABASE postgreshost $PORT
```
In the above example support for `github.com/ganggo/federation` and `github.com/ganggo/ganggo` was added.
If a user triggers a build with the following parameters:
### Building
PRREPO=https://github.com/someuser/federation.git
PRSHA=1234567890
{% include info.html text="Since you do not have permission to upload images to the feneas namespace, create a MR with your changes [here](https://git.feneas.org/feneas/federation/testsuite/merge_requests/new)! If you want to test locally proceed.." %}
The testsuite will replace the offical with the user repository and checkout the mentioned commit.
Last final step before we have a fully functional docker image is to build the actual image:
If you did all that `\m/` Create a pull-request with your changes in this repository and as soon as we merged it.
You can add your repository [here](https://testsuite.feneas.org/)!
docker build --no-cache -t feneas/testsuite_testproject:v1.0.3-testproject .
Now the testing can begin :)
The testsuite requires a special naming for every image: `feneas/testsuite_<project>:<version>-<project>`
# Development
If we succesfully build the docker image we have to commit our changes
and create a project tag and we are done with the image part:
## Dependencies
cd <workspace>/testsuite/testproject
git add .
git commit -m "Add testproject docker image"
git tag v1.0.3-testproject
In case you want to test specific parts locally you need `docker`.
Make sure it is up and running:
## Testing
sudo systemctl start docker.service
After we created a docker image we can start testing to ensure `testproject`
is actually capable of sending/receiving mentioned messages.
Then you can install the testsuite dependencies:
* github.com/stedolan/jq
* github.com/sstephenson/bats
For this we switch in our testsuite again and create a test file called `testproject.bats`:
Simply run the install script or do it manually:
cd <workspace>/testsuite
vi testproject.bats
bash scripts/install.sh
I suggest we start with loading the test helper first since it holds many helper functions which can come in handy:
```
# vim:ft=sh
load test_helper
```
Since we need two instances of our `testproject` we have to create two postgresql databases first:
```
@test "$btf create databases" {
for tp in tp1 tp2; do
create_database $tp
[ "$?" -eq 0 ]
done
}
```
and start-up the mentioned servers:
```
@test "$btf start testproject#1 server" {
start_app tp1 3000 testproject$(latest_tag testproject)
[ "$?" -eq 0 ]
}
@test "$btf start testproject#2 server" {
start_app tp2 3000 testproject$(latest_tag testproject)
[ "$?" -eq 0 ]
}
```
Now the interesting part starts.. We want to test a request to tp2!
If we execute the curl command (mentioned at the beginning of this
document) tp2 should federated/relay the message to tp1.
To prove this we send two requests:
1. to tp2 requesting the message to be forwarded
2. to tp1 ensuring the message was received
```
@test "$btf increase count on tp1 via tp2" {
get "http://$(container_ip tp2):3000/add@$(container_ip tp1):3000"
[ "$HTTP_STATUS_CODE" == "200" ]
}
@test "$btf test count on tp1, it should be one" {
get "http://$(container_ip tp1):3000"
echo "expected 200, got $HTTP_STATUS_CODE"
[ "$HTTP_STATUS_CODE" == "200" ]
echo "expected 1, got $HTTP_BODY"
[ "$HTTP_BODY" == "1" ]
}
```
Last but not least we do some cleaning and we are done with our first unit test for the federated web `\m/`
```
@test "$btf stop and delete the containers" {
for tp in tp1 tp2; do
stop_app $tp
[ "$?" -eq 0 ]
remove_app $tp
[ "$?" -eq 0 ]
done
}
@test "$btf drop databases" {
for tp in tp1 tp2; do
drop_database $tp
[ "$?" -eq 0 ]
done
}
```
{% include info.html text="You will find the full version of testproject.bats [here](https://git.feneas.org/feneas/federation/testsuite/raw/master/testproject.bats)!" %}
## Run tests
That should be it! You can add and commit your changes and request a MR upstream:
After installing all required dependencies you can start tests via:
git add testproject.bats
git commit -m "Add testproject.bats"
# bats <test-file> e.g.:
bats ganggo-ganggo.tx-rx.bats
### Run BATS
Or run all tests with a single command:
If you installed the dependencies from `scripts/install.sh`
locally you can run the testsuite by executing:
bats .
```
cd <workspace>/testsuite
bats testproject.bats
✓ testproject.bats create databases
✓ testproject.bats start testproject#1 server
✓ testproject.bats start testproject#2 server
✓ testproject.bats test initial count, it should be zero
✓ testproject.bats increase count on tp1 via tp2
✓ testproject.bats test count on tp1, it should be one
✓ testproject.bats check the database on tp1 too
✓ testproject.bats stop and delete the containers
✓ testproject.bats drop databases
9 tests, 0 failures
```
Thats it ! :)
## Run images
---
Sometimes doing tests manually helps developing a new feature. You can run single docker images by using the helper script.
Sometimes doing tests manually helps developing a new feature.
You can run single docker images by using the helper script.
Setup the environment by executing following command once:
BATS_TEST_FILENAME=local . ./test_helper.bash
Then starting can be done by executing:
# start_app <database-name> <port> "testing_<project>"$(latest_tag <project>)
# start_app <database-name> <port> "<project>"$(latest_tag <project>)
# for starting a ganggo image that could look like following:
start_app g1 9000 "testing_ganggo"$(latest_tag "ganggo")
start_app tp1 3000 testproject$(latest_tag testproject)
# Testsuite Server
**Why and how do I integrate my project with the server?**
Assuming we have some unit tests for your project in the `testsuite` repository you can authenticate your personal project against our integration service. Then every time someone creates a merge request an automatic build is triggered and the testsuite will verify that the unit tests still succeed.
In GitLab that would look something like this:
![gitlab-pipeline]({{"/assets/img/gitlab-pipeline.png" | prepend:site.baseurl }})
An external pipeline will be added and it will update the status whether tests are failing or not.
The same applies for GitHub:
![github-checks]({{"/assets/img/github-checks.png" | prepend:site.baseurl }})
**So how can you add your project?**
Simply visit [testsuite.feneas.org](https://testsuite.feneas.org) and click on
![gitlab-or-github]({{"/assets/img/gitlab-or-github.png" | prepend:site.baseurl }})
The GitLab integration also supports self-hosted instances `\m/`
If you clicked on the buttons follow the instructions and as soon as you hit `Submit` your project should be listed on the index page of [testsuite.feneas.org](https://testsuite.feneas.org).
Now the testing can begin :)
......@@ -16,7 +16,7 @@ load ganggo_helper
}
@test "$btf start diaspora#1 server" {
start_app "d1" "3000" "testing_diaspora"$(latest_tag "diaspora")
start_app "d1" "3000" "diaspora"$(latest_tag "diaspora")
[ "$?" -eq 0 ]
code=$(wait_for "docker logs $(container_id "d1")" "Starting Diaspora in production")
echo "expected 0, got $code"
......
# ganggo_start_server "g1" "9000"
function ganggo_start_server() {
start_app "$1" "$2" "testing_ganggo"$(latest_tag "ganggo")
start_app "$1" "$2" "ganggo"$(latest_tag "ganggo")
[ "$?" -eq 0 ]
code=$(wait_for "docker logs $(container_id "$1")" "Listening on")
echo "expected 0, got $code"
......
......@@ -3,7 +3,7 @@
load test_helper
@test "$btf start mastodon#1 server" {
start_app "m1" "3000" "testing_mastodon"$(latest_tag "mastodon")
start_app "m1" "3000" "mastodon"$(latest_tag "mastodon")
[ "$?" -eq 0 ]
code=$(wait_for_mastodon "m1")
echo "expected 0, got $code"
......@@ -11,7 +11,7 @@ load test_helper
}
@test "$btf start mastodon#2 server" {
start_app "m2" "3001" "testing_mastodon"$(latest_tag "mastodon")
start_app "m2" "3001" "mastodon"$(latest_tag "mastodon")
[ "$?" -eq 0 ]
code=$(wait_for_mastodon "m2")
echo "expected 0, got $code"
......
#!/bin/bash
project=$(echo $CI_COMMIT_TAG |cut -d- -f2)
# change into docker project
cd $CI_PROJECT_DIR/$project || exit 1
# authenticate with docker registry first
docker login -u $DOCKER_USER -p $DOCKER_PASSWORD || exit 1
# build the image locally
docker build -t \
feneas/testsuite_${project}:$CI_COMMIT_TAG . || exit 1
# overwrite latest-tag with new image
docker tag feneas/testsuite_${project}:$CI_COMMIT_TAG \
feneas/testsuite_${project}:latest || exit 1
# upload the image and both tags
docker push feneas/testsuite_${project}:$CI_COMMIT_TAG || exit 1
docker push feneas/testsuite_${project}:latest || exit 1
# socialhome_start_server "s1" "9001"
function socialhome_start_server() {
start_app "$1" "$2" "testing_socialhome"$(latest_tag "socialhome")
start_app "$1" "$2" "socialhome"$(latest_tag "socialhome")
[ "$?" -eq 0 ]
code=$(wait_for "docker logs $(container_id "$1")" "Listening on endpoint tcp")
echo "expected 0, got $code"
......
......@@ -72,7 +72,7 @@ function get() {
fetch "GET" "" "url" "$1"
}
# start_app "g1" "3000" "testing_diaspora:v1.0.1"
# start_app "g1" "3000" "diaspora:v1.0.1"
function start_app() {
params=""
if [ ! -z ${PRSHA} ]; then
......@@ -82,11 +82,11 @@ function start_app() {
params="-e PROJECT=$PROJECT -e PRSHA=$PRSHA -e PRREPO=$PRREPO"
fi
cid=$(container_id "$1")
echo "Starting docker container $cid (thefederation/$3) on port $2"
echo "Starting docker container $cid (feneas/testsuite_$3) on port $2"
docker run --name=$cid $params \
--link $(container_id "postgres"):postgreshost \
--link $(container_id "redis"):redishost \
-e DATABASE=$1 -e PORT=$2 -d thefederation/$3
-e DATABASE=$1 -e PORT=$2 -d feneas/testsuite_$3
}
# stop_app "g1"
......
# vim:ft=sh
#
# this test file is related to https://feneas.git.feneas.org/federation/testsuite
load test_helper
@test "$btf create databases" {
for tp in tp1 tp2; do
create_database $tp
[ "$?" -eq 0 ]
done
}
@test "$btf start testproject#1 server" {
start_app tp1 3000 testproject$(latest_tag testproject)
[ "$?" -eq 0 ]
# NOTE You can also wait for certain log output!
# Incase your server is only operational after
# a special message then you can use the wait_for
# function from the test_helper file:
#
# wait_for <command> <search-text>
#
# for this project that could look something like this:
#
# cmd="docker logs $(container_id tp1)"
# text="CREATE TABLE"
# code=$(wait_for "$cmd" "$text")
# [ "$code" -eq "0" ]
}
@test "$btf start testproject#2 server" {
start_app tp2 3000 testproject$(latest_tag testproject)
[ "$?" -eq 0 ]
}
@test "$btf test initial count, it should be zero" {
for tp in tp1 tp2; do
get "http://$(container_ip $tp):3000"
echo "expected 200, got $HTTP_STATUS_CODE"
[ "$HTTP_STATUS_CODE" == "200" ]
echo "expected 0, got $HTTP_BODY"
[ "$HTTP_BODY" == "0" ]
done
}
@test "$btf increase count on tp1 via tp2" {
get "http://$(container_ip tp2):3000/add@$(container_ip tp1):3000"
[ "$HTTP_STATUS_CODE" == "200" ]
}
@test "$btf test count on tp1, it should be one" {
get "http://$(container_ip tp1):3000"
echo "expected 200, got $HTTP_STATUS_CODE"
[ "$HTTP_STATUS_CODE" == "200" ]
echo "expected 1, got $HTTP_BODY"
[ "$HTTP_BODY" == "1" ]
}
@test "$btf check the database on tp1 too" {
result=$(query tp1 "select count(*) from testtable;")
echo "expected 1, got $result"
[[ $result =~ "1" ]]
}
@test "$btf stop and delete the containers" {
for tp in tp1 tp2; do
stop_app $tp
[ "$?" -eq 0 ]
remove_app $tp
[ "$?" -eq 0 ]
done
}
@test "$btf drop databases" {
for tp in tp1 tp2; do
drop_database $tp
[ "$?" -eq 0 ]
done
}
FROM debian:jessie
RUN apt-get update
RUN apt-get install -y \
git-core postgresql-client netcat curl
RUN git clone https://git.feneas.org/feneas/federation/testsuite-testproject.git
WORKDIR testsuite-testproject
ADD entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]