Commit 5acb3095 authored by dmorley's avatar dmorley
Browse files

Merge branch 'develop'

parents 25aa7016 2b775a14
Pipeline #5003 passed with stage
in 44 seconds
......@@ -9,12 +9,7 @@ before_script:
- curl -sS https://getcomposer.org/installer | php
- php composer.phar install
test:7.2:
image: php:7.2-cli-alpine
script:
- php composer.phar check-code
test:7.3:
image: php:7.3-cli-alpine
test:7.4:
image: php:7.4-cli-alpine
script:
- php composer.phar check-code
......@@ -9,6 +9,56 @@ The format is based on [Keep a Changelog] and this project adheres to [Semantic
### Fixed
### Security
## [2.7.0]
:exclamation: DB migrations required! (see [SQL migration script][2.7.0-sql-migration])
:exclamation: `config.php` changed - requires updates
### Added
- Donation link to bottom of pages if set in config.php
- Server location in config.php
- Use dispatcher to do things in parallel not one by one, see config.php
- Support subdomain for software name highlight
- basic idn support, you need to install php7.x-intl
- podmin last few connection errors on edit page
- config when to stop checking a pod
### Changed
- Update PHP and packages
- Check git code less as it is buggy
- Make diaspora more a highlight again in the UI/UX
- Small changes to make things easier on the eyes on mobile and desktop
- Resort the menus to be more clear how to move around
- Less data points on simple view
### Deprecated
### Removed
### Fixed
- Pod checking on the last date code was updated to pod - SQL migration
- Remove ? from ajax query to allow better cache
- Strip off the extra - in versioning for friendica and all
### Security
## [2.6.4]
### Added
- maxmind license key in config.php - now required by maxmind
### Changed
- mapbox api url structure
### Deprecated
### Removed
### Fixed
- nodeinfo that does not follow the spec will lower score
### Security
## [2.6.3]
### Added
### Changed
- js for table no longer filters pods that are signup only, this is a 20 second improvment on page load
- update remote data less often to save update time
- Color scheme and default table colums to be more simple
### Deprecated
### Removed
### Fixed
- update remote pods fixed
- try and handle pleroma or others making up own nodeinfo spec. dirty fix for now
### Security
## [2.6.2] - 2019-11-24
### Added
- Total rows count o loading screen as that is growing
......@@ -202,6 +252,7 @@ The format is based on [Keep a Changelog] and this project adheres to [Semantic
### Fixed
- Fix ipv6
[2.7.0-sql-migration]: https://git.feneas.org/diasporg/Poduptime/blob/master/db/migrations/2.6.0-2.7.0.sql
[2.6.0-sql-migration]: https://git.feneas.org/diasporg/Poduptime/blob/master/db/migrations/2.5.2-2.6.0.sql
[2.5.2-sql-migration]: https://git.feneas.org/diasporg/Poduptime/blob/master/db/migrations/2.5.0-2.5.2.sql
[2.5.0-sql-migration]: https://git.feneas.org/diasporg/Poduptime/blob/master/db/migrations/2.4.1-2.5.0.sql
......@@ -211,6 +262,7 @@ The format is based on [Keep a Changelog] and this project adheres to [Semantic
[2.2.0-sql-migration]: https://git.feneas.org/diasporg/Poduptime/blob/master/db/migrations/2.1.4-2.2.0.sql
[Unreleased]: https://git.feneas.org/diasporg/Poduptime/compare/master...develop
[2.7.0]: https://git.feneas.org/diasporg/Poduptime/compare/2.6.4...2.7.0
[2.6.1]: https://git.feneas.org/diasporg/Poduptime/compare/2.6.0...2.6.1
[2.6.0]: https://git.feneas.org/diasporg/Poduptime/compare/2.5.2...2.6.0
[2.5.2]: https://git.feneas.org/diasporg/Poduptime/compare/2.5.1...2.5.2
......
......@@ -8,7 +8,7 @@ Environmental items you need (debian based system assumed):
OS Dependencies:
```
php7.2 php7.2-curl php7.2-pgsql php7.2-bcmath php-geoip php7.2-cli php7.2-common php7.2-fpm php7.2-bcmath php7.2-json php7.2-readline php7.2-mbstring php7.2-xml php-cgi git curl postgresql postgresql-contrib dnsutils bind9 npm nodejs composer
php7.4 php7.4-curl php7.4-pgsql php7.4-bcmath php-geoip php7.4-cli php7.4-common php7.4-fpm php7.4-bcmath php7.4-json php7.4-readline php7.4-mbstring php7.4-xml php7.4-intl php-cgi git curl postgresql postgresql-contrib dnsutils bind9 npm nodejs composer
```
Yarn is a separate install: https://yarnpkg.com
......@@ -52,7 +52,7 @@ see file http-server-config.example
# To Use:
run `php db/update.php` to update your data
run `php-cgi db/update-all.php` to update your data for all pods
run `php db/update.php debug` to debug output
run `php db/update.php sqldebug` to debug sql
run `php db/update.php develop` to run without email alerts to end users
......
......@@ -4,18 +4,18 @@
"license": "AGPL-3.0-or-later",
"config": {
"platform": {
"php": "7.2"
"php": "7.4"
}
},
"require": {
"php": "^7.2",
"php": "^7.4",
"ext-curl": "*",
"ext-dom": "*",
"ext-json": "*",
"ext-libxml": "*",
"ext-bcmath": "*",
"gabordemooij/redbean": "^5.1",
"nesbot/carbon": "^1.33",
"nesbot/carbon": "^2.0.0",
"commerceguys/enum": "^1.0",
"noplanman/xec": "0.1.0",
"jaybizzle/crawler-detect": "^1.2",
......@@ -24,18 +24,23 @@
"matriphe/iso-639": "^1.2",
"detectlanguage/detectlanguage": "^2.2",
"longman/ip-tools": "^1.2",
"rinvex/countries": "^5.0"
"rinvex/countries": "^5.0",
"fastbill/parallel-process-dispatcher": "^1.2"
},
"require-dev": {
"squizlabs/php_codesniffer": "^3.3"
},
"autoload": {
"classmap": ["lib"],
"files": ["lib/functions.php"]
"classmap": [
"lib"
],
"files": [
"lib/functions.php"
]
},
"scripts": {
"check-code": [
"vendor/bin/phpcs --standard=PSR2 -snp --encoding=utf-8 --report-width=150 db lib *.php"
"vendor/bin/phpcs --standard=ruleset.xml -snp --encoding=utf-8 --report-width=150 db lib *.php"
]
}
}
This diff is collapsed.
......@@ -5,16 +5,25 @@
*/
return [
//backup directory - full dir path
//backup directory - full dir path (don't forget to create this directory if new)
'backup_dir' => __DIR__ . '/backup',
//log directory - full dir path
'log_dir' => __DIR__ . '/log',
//location of pg dump - full dir path
//location of pg dump binary - full dir path
'pg_dump_dir' => '/usr/bin',
//db host
//location of php bin - full location
'phpbin' => '/usr/bin/php',
//location of php-cgi bin - full location
'phpcgibin' => '/usr/bin/php-cgi',
//location of nice bin - full location
'nicebin' => '/usr/bin/nice',
//db hostname
'pghost' => 'localhost',
//db port
......@@ -32,9 +41,27 @@ return [
//admin email for forms
'adminemail' => '',
//number to stop checking a pod. usually a negative.
'minscore' => '-99',
//location checks are run from to show in UI
'serverlocation' => '',
//text for a link below the stats link
'serverad' => '',
//URL for a link for the text above to go somewhere
'serveradurl' => '',
//DNS server for dnssec testing. 1.1.1.1 tests the best
'dnsserver' => '',
//server parallel processes max - greater than 10 can eat a lot of CPU - also make sure you have db connections/speed to support this number
'serverprocesses' => '',
//Number of minutes to report on status.php that the run is still green
'status_green' => '',
//Number of minutes to report on status.php that the run is still green
'status_green' => '',
......@@ -47,6 +74,9 @@ return [
//Geolite2-city database file in mmdb format - full file path
'geoip2db' => '',
//Maxmind License key
'maxmindkey' => '',
//sitemap file - full dir path
'sitemap' => __DIR__ . '/sitemap.xml',
......@@ -74,8 +104,11 @@ return [
'peertube' => ['repo' => 'Chocobozzz/PeerTube', 'gitsite' => 'api.github.com', 'gittype' => 'github', 'devbranch' => 'develop'],
'plume' => ['repo' => 'Plume-org/Plume', 'gitsite' => 'api.github.com', 'gittype' => 'github', 'devbranch' => ''],
'rustodon' => ['repo' => 'rustodon/rustodon', 'gitsite' => 'api.github.com', 'gittype' => 'github', 'devbranch' => ''],
'microblogpub' => ['repo' => 'tsileo/microblog.pub', 'gitsite' => 'api.github.com', 'gittype' => 'github', 'devbranch' => ''],
'mobilizon' => ['repo' => 'framasoft/mobilizon', 'gitsite' => 'api.github.com', 'gittype' => 'github', 'devbranch' => ''],
'lemmy' => ['repo' => 'fLemmyNet/lemmy', 'gitsite' => 'api.github.com', 'gittype' => 'github', 'devbranch' => ''],
],
//hidden software word strings from table view - %(one)% OR %(one|two|three)%
'hidden-softwares' => '%(relay)%',
'hidden-softwares' => '%(relay|n/q)%',
];
@import url("/node_modules/typeface-roboto/index.css");
body {
font-family: 'Roboto', sans-serif;
font-family: 'Helvetica', sans-serif;
font-weight: 400;
--blue: #303030;
--grey: #4e89a4;
--green: #6B8E23;
--blue: #002145;
--grey: #A5ACAF;
--green: #207A00;
}
.bg-blue {
......@@ -36,10 +34,6 @@ a {
color: var(--blue);
}
.tablesorter-header {
background-color: var(--grey) !important;
}
.main {
padding: 5px;
}
......@@ -63,7 +57,7 @@ a {
}
.tfont {
font-size: 12px;
font-size: 16px;
}
.smlogo {
......@@ -179,6 +173,7 @@ a {
.tablesorter-header {
background-size: 10px 15px !important;
background-color: rgba(188, 202, 255, 0.9) !important;
}
.columnSelectorWrapper {
......@@ -225,7 +220,7 @@ a {
.loadingtable {
width: 100%;
height: 100%;
background-color: rgba(78,137,164,.8);
background-color: var(--blue);
position: fixed;
margin: 0;
padding: 0;
......@@ -236,13 +231,13 @@ a {
border: 4px solid #ffffff;
border-radius: 50px;
height: 40px;
left: 520px;
left: 40px;
margin: -20px 0 0 -20px;
opacity: 0;
position: absolute;
top: 197px;
top: 40px;
width: 40px;
animation: pulsate 5s ease-out;
animation: pulsate 1s ease-out;
animation-iteration-count: infinite;
}
......@@ -296,3 +291,11 @@ a {
.navbar-custom {
background-color: rgba(78,137,164,.4);
}
.url {
max-width: 99%;
text-overflow: ellipsis;
overflow: hidden;
display: inline-block;
white-space: nowrap;
}
......@@ -112,6 +112,7 @@ if (!$stop) {
$p['podmin_notify'] = $_podmin_notify;
$p['podmin_notify_level'] = $_podmin_notify_level;
$p['publickey'] = $publickey;
$p['date_updated'] = date('Y-m-d H:i:s');
R::store($p);
} catch (\RedBeanPHP\RedException $e) {
......@@ -135,6 +136,5 @@ if (!$stop) {
echo 'Data successfully inserted! Your pod will be checked and live on the list in a few hours!';
} else {
echo 'Could not validate your pod, check your setup!<br>Take a look at your <a href="' . $nodeinfo_url . '">nodeinfo</a>';
updateMeta('add_attempt', $_domain);
}
}
......@@ -91,7 +91,7 @@ if ('save' === $_action) {
$message = 'Data for ' . $_domain . ' updated. If it was not you reply and let me know!';
@mail($to, $subject, $message, implode("\r\n", $headers));
die('Data saved. Will go into effect on next hourly change');
die('Data saved. Will go into effect on next update');
}
// Forms.
......@@ -154,3 +154,38 @@ Authorized to edit <b><?php echo $_domain; ?></b> for <?php echo (new Carbon($po
<small id="debug" class="form-text text-muted">
Do a debug pull of your pod. Won't update the database just show what it would look like on a pass.
</small>
<br>
Last 20 connection errors from your pod:
<?php
try {
$errors = R::getAll('
SELECT error, date_checked
FROM checks
WHERE domain = ?
AND ONLINE = false
ORDER BY date_checked DESC
LIMIT 20
', [$_domain]);
} catch (\RedBeanPHP\RedException $e) {
debug('Error in SQL query: ' . sprintf($e->getMessage()));
}
?>
<table class="table col-lg-8">
<thead class="thead-light">
<tr>
<th scope="col" class="col-lg-9">Error</th>
<th scope="col" class="col-md-3">Date</th>
</tr>
</thead>
<tbody>
<?php
foreach ($errors as $error) {
echo "<tr><td>{$error['error']}</td><td>{$error['date_checked']}</td></tr>";
}
?>
</tbody>
</table>
ALTER TABLE pods ADD fullversion text;
\ No newline at end of file
......@@ -4,6 +4,7 @@ CREATE TABLE pods (
name text,
softwarename text,
masterversion text,
fullversion text,
shortversion text,
stats_apikey text,
score int DEFAULT 50,
......
<?php
/**
* Pull pod info in parallel for all pods in db
* Use php-cgi or set higher time limits in php.ini
*/
declare(strict_types=1);
set_time_limit(0);
require_once __DIR__ . '/../boot.php';
if (!is_cli()) {
$referer = ($_SERVER['HTTP_REFERER'] ? parse_url($_SERVER['HTTP_REFERER'])['host'] : '');
if ($referer !== $_SERVER['SERVER_NAME']) {
header('HTTP/1.0 403 Forbidden');
exit;
}
}
use Carbon\Carbon;
use GeoIp2\Database\Reader;
use Longman\IPTools\Ip;
use Poduptime\PodStatus;
use RedBeanPHP\R;
use FastBill\ParallelProcessDispatcher\Process;
use FastBill\ParallelProcessDispatcher\Dispatcher;
if (!is_connected()) {
die('no internet');
}
$time_start = microtime(true);
try {
$sql = '
SELECT domain, score, date_created, weight, podmin_notify, podmin_notify_level, email, masterversion, shortversion, status, detectedlanguage
FROM pods
';
$pods = [];
if (getMeta('pods_updating')) {
die('already running');
}
updateMeta('pods_updating', true);
$sql .= ' WHERE status < ? ORDER BY id';
$pods = R::getAll($sql, [PodStatus::PAUSED]);
} catch (\RedBeanPHP\RedException $e) {
debug('Error in SQL query: ' . sprintf($e->getMessage()));
} catch (\MaxMind\Db\Reader\InvalidDatabaseException $e) {
debug('Invalid GeoIP database: ' . sprintf($e->getMessage()));
}
$serverprocesses = c('serverprocesses') ?: 1;
$dispatcher = new Dispatcher($serverprocesses);
foreach ($pods as $pod) {
$update = __DIR__ . '/update.php';
$dispatcher->addProcess(new Process(c('nicebin') . ' ' . c('phpcgibin') . " {$update} domain={$pod['domain']}"));
}
$dispatcher->dispatch();
updateMeta('pods_updated');
if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('masterversions_updated', 'date_created'))->diffInHours() > 48) {
require __DIR__ . '/update-masterversions.php';
}
if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('statstable_updated', 'date_created'))->diffInHours() > 48) {
require __DIR__ . '/update-monthly-stats.php';
}
if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('cacert_updated', 'date_created'))->diffInHours() > 400) {
copy('https://curl.haxx.se/ca/cacert.pem', c('cafullpath'));
updateMeta('cacert_updated');
}
if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('sitemap_updated', 'date_created'))->diffInHours() > 48) {
require __DIR__ . '/update-sitemap.php';
}
if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('geoip_updated', 'date_created'))->diffInHours() > 400) {
copy('compress.zlib://https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=' . c('maxmindkey') . '&suffix=tar.gz', 'geo.tar');
$archive = new PharData('geo.tar');
$basename = basename((string)$archive);
$archive->extractTo('.', $basename . '/GeoLite2-City.mmdb', true);
rename($basename . '/GeoLite2-City.mmdb', c('geoip2db'));
unlink('geo.tar');
rmdir($basename);
updateMeta('geoip_updated');
}
if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('federation_updated', 'date_created'))->diffInHours() > 9) {
require __DIR__ . '/update-remote-data.php';
}
require __DIR__ . '/backup.php';
$time_end = microtime(true);
$execution_time = ($time_end - $time_start) / 60;
updateMeta('pods_update_runtime', round($execution_time));
updateMeta('pods_updating', false);
$langhours = Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('languages_updated', 'date_created'))->diffInHours();
if ($langhours > 24) {
updateMeta('languages_updated');
}
......@@ -29,7 +29,7 @@ foreach ($softwares as $software => $details) {
try {
$m = R::dispense('masterversions');
$m['software'] = $software;
$m['version'] = $masterversion;
$m['version'] = preg_replace('/-.*$/', '', $masterversion);
if (isset($releasejson->published_at) && $releasedate = $releasejson ? $releasejson->published_at : '') {
$m['releasedate'] = $releasedate;
}
......@@ -52,7 +52,7 @@ foreach ($softwares as $software => $details) {
try {
$m = R::dispense('masterversions');
$m['software'] = $software;
$m['version'] = $masterversion;
$m['version'] = preg_replace('/-.*$/', '', $masterversion);
if (isset($releasejson[0]->commit->created_at) && $releasedate = $releasejson[0] ? $releasejson[0]->commit->created_at : '') {
$m['releasedate'] = $releasedate;
}
......@@ -66,7 +66,7 @@ foreach ($softwares as $software => $details) {
}
}
printf('%s:%s:%s ', $software, $masterversion, $devlastcommit ?: 'n/a');
printf('%s:%s:%s ', $software, $m['version'], $devlastcommit ?: 'n/a');
}
updateMeta('masterversions_updated');
......@@ -5,8 +5,11 @@
*/
declare(strict_types=1);
set_time_limit(0);
use RedBeanPHP\R;
use FastBill\ParallelProcessDispatcher\Process;
use FastBill\ParallelProcessDispatcher\Dispatcher;
require_once __DIR__ . '/../boot.php';
......@@ -47,26 +50,31 @@ $skippods = array_merge($existingpods, $triedpods);
$foundpods = [];
//pulling all nodes for now
$federationpods = curl('https://the-federation.info/graphql?query=%7Bnodes%7Bhost%20platform%7Bname%7Dprotocols%7Bname%7D%7D%7D&raw', false, 45);
$federationpods = curl('https://the-federation.info/pods.json', false, 45);
if ($pods = json_decode($federationpods['body'] ?: '', true)) {
foreach ($pods['data']['nodes'] ?? [] as $poddata) {
$protocols = array_column($poddata['protocols'] ?? [], 'name');
foreach ($pods['pods'] ?? [] as $key => $poddata) {
$foundpods[] = strtolower(preg_replace('/(https?:\/\/)?(:(\d*))?/', '', $poddata['host']));
}
}
$diasppods = curl('https://diasp.org/pods.json', false, 45);
if ($pods = json_decode($diasppods['body'] ?: '', true)) {
foreach ($pods ?? [] as $poddata) {
foreach ($pods ?? [] as $key => $poddata) {
$foundpods[] = strtolower(preg_replace('/(https?:\/\/)?(:(\d*))?/', '', $poddata['host']));
}
}
$results = array_diff($foundpods, $skippods);
$serverprocesses = c('serverprocesses') ?: 1;
$dispatcher = new Dispatcher($serverprocesses);
foreach ($results as $result) {
echo ' domain ' . $result . ' trying... ';
$add = __DIR__ . '/add.php';
echo exec("php-cgi {$add} domain={$result}") . "\r\n";
$dispatcher->addProcess(new Process(c('nicebin') . ' ' . c('phpcgibin') . " {$add} domain={$result}"));
updateMeta('add_attempt', $result);
}
$dispatcher->dispatch();
updateMeta('federation_updated');
......@@ -12,7 +12,7 @@ require_once __DIR__ . '/../boot.php';
try {
$pods = R::getAll('
SELECT domain, date_updated::TIMESTAMP::DATE
SELECT softwarename, domain, date_updated::TIMESTAMP::DATE
FROM pods
WHERE date_updated IS NOT NULL
ORDER BY date_updated DESC
......@@ -23,7 +23,12 @@ try {
foreach ($pods as $pod) {
$xml .= "\t<url>\n";
$xml .= "\t\t<loc>https://podupti.me/{$pod['domain']}</loc>\n";
if ($pod['softwarename']) {
$softwareclean = preg_replace("/[^a-zA-Z]/", "", $pod['softwarename']);
$xml .= "\t\t<loc>https://{$softwareclean}.podupti.me/{$pod['domain']}</loc>\n";
} else {
$xml .= "\t\t<loc>https://podupti.me/{$pod['domain']}</loc>\n";
}
$xml .= "\t\t<lastmod>{$pod['date_updated']}</lastmod>\n";
$xml .= "\t\t<changefreq>hourly</changefreq>\n";
$xml .= "\t</url>\n";
......
......@@ -29,7 +29,7 @@ if (!is_connected()) {
$time_start = microtime(true);
$debug = isset($_GET['debug']) || (isset($argv) && in_array('debug', $argv, true));
$init = isset($_GET['init']) || (isset($argv) && in_array('init', $argv, true));
$init = isset($_GET['init']) || (isset($argv) && in_array('init', $argv, true));
$sqldebug = isset($_GET['sqldebug']) || (isset($argv) && in_array('sqldebug', $argv, true));
$develop = isset($_GET['develop']) || (isset($argv) && in_array('develop', $argv, true));
$write = !(isset($_GET['nowrite']) || (isset($argv) && in_array('nowrite', $argv, true)));
......@@ -47,7 +47,7 @@ try {
$reader = new Reader(c('geoip2db'));
$sql = '
SELECT domain, score, date_created, weight, podmin_notify, podmin_notify_level, email, masterversion, shortversion, status, detectedlanguage
SELECT domain, score, date_created, weight, podmin_notify, podmin_notify_level, email, masterversion, shortversion, fullversion, status, detectedlanguage, date_updated
FROM pods
';
......@@ -81,12 +81,14 @@ foreach ($pods as $pod) {