Commit c175345f authored by dmorley's avatar dmorley

Merge branch 'develop' into podmin-notice-freedom

# Conflicts:
#	CHANGELOG.md
parents 80c8be9b 79d78010
......@@ -8,6 +8,8 @@ The format is based on [Keep a Changelog] and this project adheres to [Semantic
- Added `pghost` config to set database port
- Added `CONTRIBUTING.md`
- Podmin can choose at what fail score to send the notice out
- Podmin email shares details on why pod is failing
- Only retrieve location data for remote servers / IPs
### Changed
- Introduce proper changelog format (#189)
- Moved DB migration scripts into `db` folder
......@@ -16,6 +18,11 @@ The format is based on [Keep a Changelog] and this project adheres to [Semantic
- Use pretty URLs (see nginx.example)
- Open pod URLs in a new tab
- Use detectlanguage.com API for language guess
- Only use JSON data, ignore HTML when returned
- Allow curl redirect on home page check
- Score now goes to -5000 before a pod is removed so dead pods get checked a while then removed for good
- Move functions to dedicated file to allow reuse
- Backup script rewrite
### Deprecated
### Removed
### Fixed
......
......@@ -9,33 +9,6 @@ declare(strict_types=1);
use DetectLanguage\DetectLanguage;
use RedBeanPHP\R;
/**
* Helper to get config values.
*
* @param null|string $param
* @param null|mixed $default
*
* @return mixed|null
*/
function c(string $param = null, $default = null)
{
static $config;
if ($config === null) {
$config = require __DIR__ . '/config.php';
is_array($config) || die('Invalid config format.');
}
if ($param === null) {
return $config;
}
if (array_key_exists($param, $config)) {
return $config[$param];
}
return $default;
}
define('PODUPTIME', microtime(true));
require_once __DIR__ . '/vendor/autoload.php';
......
......@@ -22,13 +22,15 @@
"maxmind-db/reader": "^1.3",
"matriphe/iso-639": "^1.2",
"rinvex/country": "^3.1",
"detectlanguage/detectlanguage": "^2.2"
"detectlanguage/detectlanguage": "^2.2",
"longman/ip-tools": "^1.2"
},
"require-dev": {
"squizlabs/php_codesniffer": "^3.3"
},
"autoload": {
"classmap": ["lib"]
"classmap": ["lib"],
"files": ["lib/functions.php"]
},
"scripts": {
"check-code": [
......
......@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "ae2037263bedfa647fb7fe96311a84da",
"content-hash": "9ac3659b3b8b6f4a29e9a44e0dfd105f",
"packages": [
{
"name": "commerceguys/enum",
......@@ -289,6 +289,62 @@
],
"time": "2018-07-30T20:23:10+00:00"
},
{
"name": "longman/ip-tools",
"version": "1.2.1",
"source": {
"type": "git",
"url": "https://github.com/akalongman/php-ip-tools.git",
"reference": "6c050dfbf91811d14b9b3aa31fb7116eac0f0a18"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/akalongman/php-ip-tools/zipball/6c050dfbf91811d14b9b3aa31fb7116eac0f0a18",
"reference": "6c050dfbf91811d14b9b3aa31fb7116eac0f0a18",
"shasum": ""
},
"require": {
"ext-bcmath": "*",
"php": ">=5.5"
},
"require-dev": {
"phpspec/phpspec": "~2.1",
"phpunit/phpunit": "~4.1",
"squizlabs/php_codesniffer": "~2.3"
},
"type": "library",
"autoload": {
"psr-4": {
"Longman\\IPTools\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Avtandil Kikabidze aka LONGMAN",
"email": "akalongman@gmail.com",
"homepage": "https://longman.me",
"role": "Developer"
}
],
"description": "PHP IP Tools for manipulation with IPv4 and IPv6",
"homepage": "https://github.com/akalongman/php-ip-tools",
"keywords": [
"IP",
"Match",
"compare",
"ipv4",
"ipv6",
"mask",
"subnet",
"tools",
"utilities"
],
"time": "2016-10-23T20:08:46+00:00"
},
{
"name": "matriphe/iso-639",
"version": "1.2",
......
......@@ -6,25 +6,45 @@
declare(strict_types=1);
if (PHP_SAPI !== 'cli') {
require_once __DIR__ . '/../boot.php';
if (!is_cli()) {
header('HTTP/1.0 403 Forbidden');
exit;
}
$c = require __DIR__ . '/../config.php';
$keep_for = 60 * 60 * 6; // 6 hours
$backup_file = c('backup_dir') . '/dump_' . date('Ymd_His') . '.sql';
printf("Making backup of '%s' to '%s'...", c('pgdb'), $backup_file);
system(sprintf(
'export PGPASSWORD=%3$s &&' .
'"%1$s" --clean --format=tar --username=%2$s %4$s >> "%5$s"',
c('pg_dump_dir') . '/pg_dump',
c('pguser'),
c('pgpass'),
c('pgdb'),
$backup_file
), $exit_code);
printf(" %s\n", $exit_code === 0 ? 'Success!' : 'Failed.');
$keep = (60 * 60 * 6) * 1;
$dump_date = date('Ymd_Hs');
$file_name = "{$c['backup_dir']}/dump_{$dump_date}.sql";
system("export PGPASSWORD={$c['pgpass']} && {$c['pg_dump_dir']}/pg_dump --clean --format=tar --username={$c['pguser']} {$c['pgdb']} >> {$file_name}");
echo "pg backup of {$c['pgdb']} made";
$dirh = dir($c['backup_dir']);
$dirh = dir(c('backup_dir'));
while ($entry = $dirh->read()) {
$old_file_time = date('U') - $keep;
$file_created = filectime("{$c['backup_dir']}/{$entry}");
if ($file_created < $old_file_time && !is_dir($entry)) {
if (unlink("{$c['backup_dir']}/{$entry}")) {
echo 'Cleaned up old backups';
}
$file = c('backup_dir') . "/{$entry}";
// Skip dotfiles and non-files (folders, symlinks, etc.).
if ($entry[0] === '.' || !is_file($file)) {
continue;
}
if (filectime($file) + $keep_for > time()) {
//echo "Don't delete {$entry}\n";
continue;
}
printf(
"Removing old file '%s'... %s\n",
$entry,
unlink($file) ? 'Success!' : 'Failed.'
);
}
......@@ -6,15 +6,15 @@
declare(strict_types=1);
if (PHP_SAPI !== 'cli') {
header('HTTP/1.0 403 Forbidden');
exit;
}
use RedBeanPHP\R;
require_once __DIR__ . '/../boot.php';
if (!is_cli()) {
header('HTTP/1.0 403 Forbidden');
exit;
}
try {
$monthly_totals = R::getAll("
SELECT
......
......@@ -6,15 +6,15 @@
declare(strict_types=1);
if (PHP_SAPI !== 'cli') {
header('HTTP/1.0 403 Forbidden');
exit;
}
use RedBeanPHP\R;
require_once __DIR__ . '/../boot.php';
if (!is_cli()) {
header('HTTP/1.0 403 Forbidden');
exit;
}
try {
$sql = '
SELECT domain, status
......
......@@ -6,15 +6,15 @@
declare(strict_types=1);
if (PHP_SAPI !== 'cli') {
header('HTTP/1.0 403 Forbidden');
exit;
}
use RedBeanPHP\R;
require_once __DIR__ . '/../boot.php';
if (!is_cli()) {
header('HTTP/1.0 403 Forbidden');
exit;
}
$softwares = [
'diaspora' => ['repo' => 'diaspora/diaspora', 'gitsite' => 'api.github.com', 'gittype' => 'github', 'devbranch' => 'develop'],
'friendica' => ['repo' => 'friendica/friendica', 'gitsite' => 'api.github.com', 'gittype' => 'github', 'devbranch' => 'develop'],
......
......@@ -6,7 +6,9 @@
declare(strict_types=1);
if (!in_array(PHP_SAPI, ['cgi-fcgi', 'cli'])) {
require_once __DIR__ . '/../boot.php';
if (!is_cli()) {
$referer = ($_SERVER['HTTP_REFERER'] ? parse_url($_SERVER['HTTP_REFERER'])['host'] : '');
if ($referer !== $_SERVER['SERVER_NAME']) {
header('HTTP/1.0 403 Forbidden');
......@@ -15,8 +17,8 @@ if (!in_array(PHP_SAPI, ['cgi-fcgi', 'cli'])) {
}
use Carbon\Carbon;
use DetectLanguage\DetectLanguage;
use GeoIp2\Database\Reader;
use Longman\IPTools\Ip;
use Poduptime\PodStatus;
use RedBeanPHP\R;
......@@ -24,14 +26,12 @@ $debug = isset($_GET['debug']) || (isset($argv) && in_array('debug', $argv, t
$sqldebug = isset($_GET['sqldebug']) || (isset($argv) && in_array('sqldebug', $argv, true));
$develop = isset($_GET['develop']) || (isset($argv) && in_array('develop', $argv, true));
$write = !(isset($_GET['nowrite']) || (isset($argv) && in_array('nowrite', $argv, true)));
$newline = PHP_SAPI === 'cli' ? "\n\n" : '<br><br>';
$newline = is_cli() ? "\n\n" : '<br><br>';
$_domain = $_GET['domain'] ?? null;
// Must have a domain, except if called from CLI.
$_domain || PHP_SAPI === 'cli' || die('No valid input');
require_once __DIR__ . '/../boot.php';
$_domain || is_cli() || die('No valid input');
$sqldebug && R::fancyDebug(true);
......@@ -49,11 +49,11 @@ try {
if ($_domain) {
$sql .= ' WHERE domain = ?';
$pods = R::getAll($sql, [$_domain]);
} elseif (PHP_SAPI === 'cli' && (isset($argv) && in_array('Check_System_Deleted', $argv, true))) {
$sql .= ' WHERE status = ?';
} elseif (is_cli() && (isset($argv) && in_array('Check_System_Deleted', $argv, true))) {
$sql .= ' WHERE status = ? ORDER BY id';
$pods = R::getAll($sql, [PodStatus::SYSTEM_DELETED]);
} elseif (PHP_SAPI === 'cli') {
$sql .= ' WHERE status < ?';
} elseif (is_cli()) {
$sql .= ' WHERE status < ? ORDER BY id';
$pods = R::getAll($sql, [PodStatus::PAUSED]);
}
} catch (\RedBeanPHP\RedException $e) {
......@@ -86,7 +86,7 @@ foreach ($pods as $pod) {
die('Error in SQL query: ' . $e->getMessage());
}
_debug('Domain', $domain);
debug('Domain', $domain);
$user_rating = 0;
......@@ -94,38 +94,39 @@ foreach ($pods as $pod) {
$user_rating = round(array_sum($user_ratings) / count($user_ratings), 2);
}
$nodeinfo_meta = _curl("https://{$domain}/.well-known/nodeinfo");
$nodeinfo_meta = curl("https://{$domain}/.well-known/nodeinfo");
// Default link to fetch node info.
$nodeinfo_url = "https://{$domain}/nodeinfo/1.0";
if (!isset($nodeinfo_meta['error'])) {
$info = json_decode($nodeinfo_meta['body'], true);
if ($info = json_decode($nodeinfo_meta['body'] ?: '', true)) {
$nodeinfo_url = max($info['links'])['href'];
}
_debug('Nodeinfo link', $nodeinfo_url);
$nodeinfo = _curl($nodeinfo_url);
debug('Nodeinfo link', $nodeinfo_url);
$nodeinfo = curl($nodeinfo_url);
$outputssl = $nodeinfo['body'];
$outputsslerror = $nodeinfo['error'];
$info = $nodeinfo['info'];
$httpcode = $nodeinfo['code'];
$conntime = $nodeinfo['conntime'];
$nstime = $nodeinfo['nstime'];
$latency = $conntime - $nstime;
$sslexpire = $info[0]['Expire date'] ?? null;
_debug('Nodeinfo output', $outputssl, true);
_debug('Nodeinfo output error', $outputsslerror, true);
_debug('Cert expire date', $sslexpire);
_debug('Conntime', $conntime);
_debug('NStime', $nstime);
_debug('Latency', $latency);
debug('Nodeinfo output', $outputssl, true);
debug('Nodeinfo output error', $outputsslerror, true);
debug('Nodeinfo service http response code', $httpcode);
debug('Cert expire date', $sslexpire);
debug('Conntime', $conntime);
debug('NStime', $nstime);
debug('Latency', $latency);
$jsonssl = ($outputssl ? json_decode($outputssl) : null);
$jsonssl = json_decode($outputssl ?: '');
if ($jsonssl !== null) {
$version = $jsonssl->software->version ?? 0;
$version = $jsonssl->software->version ?? 0;
preg_match_all('((?:\d(.|-)?)+(\.)\d+\.*)', $version, $sversion);
$shortversion = $sversion[0][0] ?? '0.0.0.0';
$signup = ($jsonssl->openRegistrations ?? false) === true;
......@@ -171,7 +172,7 @@ foreach ($pods as $pod) {
die('Error in SQL query: ' . $e->getMessage());
}
_debug('Version code', $shortversion);
debug('Version code', $shortversion);
try {
$masterdata = R::getRow('
......@@ -189,13 +190,13 @@ foreach ($pods as $pod) {
$devlastcommit = ($masterdata['devlastcommit'] ?? date('Y-m-d H:i:s'));
$releasedate = ($masterdata['releasedate'] ?? date('Y-m-d H:i:s'));
_debug('Masterversion', $masterversion);
debug('Masterversion', $masterversion);
$masterversioncheck = explode('.', $masterversion);
$shortversioncheck = (strpos($shortversion, '.') ? explode('.', $shortversion) : implode('.', ['0', preg_replace('/\D/', '', $shortversion), '0']));
//this is still off with a pod with v1 as total version. cant explode that, won't have a [0] or [1] later to use either
_debug('Days since master code release', date_diff(new DateTime($releasedate), new DateTime())->format('%d'));
debug('Days since master code release', date_diff(new DateTime($releasedate), new DateTime())->format('%d'));
try {
$lastpodupdates = R::getRow('
......@@ -213,7 +214,7 @@ foreach ($pods as $pod) {
$lastdatechecked = ($lastpodupdates['date_checked'] ?? date('Y-m-d H:i:s'));
$devlastdays = $devlastcommit ? date_diff(new DateTime($devlastcommit), new DateTime())->format('%a') : 30;//tmp//if no dev branch then what?
_debug('Dev last commit was ', $devlastdays);
debug('Dev last commit was ', $devlastdays);
$updategap = date_diff(new DateTime($lastdatechecked), new DateTime($devlastcommit))->format('%a');
......@@ -222,28 +223,33 @@ foreach ($pods as $pod) {
if ($updategap + $devlastdays > 400) {
_debug('Outdated More than 400 days', 'Yes');
debug('Outdated More than 400 days', 'Yes');
$podminhelp = 'Your code base seems too out of date to be used. Last time you updated was ' . $updategap;
$score -= 2;
}
} elseif (($masterversioncheck[1] - $shortversioncheck[1]) > 1) {
///tmp/If pod is two versions off AND it's been more than 60 days since that release came out AND your on the master production branch
_debug('Outdated second decimal > 1', 'Yes');
debug('Outdated second decimal > 1', 'Yes');
$score -= 2;
$updategap = date_diff(new DateTime($lastdatechecked), new DateTime($releasedate))->format('%a');
$podminhelp = 'Your code base seems too out of date to be used. Current version is ' . $masterversion . ' and you are running ' . $shortversion;
} elseif ($updategap - date_diff(new DateTime($releasedate), new DateTime())->format('%a') > 400) {
_debug('Outdated more than 400 days since x ', 'Yes');
debug('Outdated more than 400 days since x ', 'Yes');
$score -= 2;
$updategap = date_diff(new DateTime($lastdatechecked), new DateTime($releasedate))->format('%a');
$podminhelp = 'Your code base seems too out of date to be used. Last time you updated was ' . $updategap;
} else {
$updategap = date_diff(new DateTime($lastdatechecked), new DateTime($releasedate))->format('%a');
}
_debug('Pod code was updated after ', $updategap);
debug('Pod code was updated after ', $updategap);
$status = PodStatus::UP;
}
......@@ -255,14 +261,15 @@ foreach ($pods as $pod) {
if (!$language_snippet) {
$detectedlanguage = null;
--$score;
$podminhelp = 'Unable to render the html on your main page https://' . $domain;
} elseif ($debug || Carbon::now()->hour === 12) {
$detectedlanguage = detectWebsiteLanguageFromSnippet($language_snippet);
}
_debug('Detected Language', $detectedlanguage);
debug('Detected Language', $detectedlanguage);
if (!$jsonssl || !$language_snippet) {
_debug('Connection', 'Can not connect to pod');
debug('Connection', 'Can not connect to pod');
try {
$c = R::dispense('checks');
......@@ -283,7 +290,7 @@ foreach ($pods as $pod) {
$status = PodStatus::DOWN;
}
_debug('Signup Open', $signup);
debug('Signup Open', $signup);
$dnsserver = c('dnsserver') ?: '1.1.1.1';
$delv = new NPM\Xec\Command("delv @{$dnsserver} {$domain}");
......@@ -303,10 +310,11 @@ foreach ($pods as $pod) {
$iplookupv6 = explode(PHP_EOL, trim($delv->execute(['AAAA'], null, 15)->stdout));
$ipv6 = (bool) preg_grep('/\s+IN\s+AAAA\s+.*/', $iplookupv6);
_debug('IPv4', $ip);
_debug('IPv6', $ipv6);
debug('IPv4', $ip);
debug('IPv6', $ipv6);
if ($ip) {
// todo: Temporary workaround (see https://github.com/akalongman/php-ip-tools/issues/8)
if (Ip::isValid($ip) && Ip::isRemote($ip)) {
$geo = $reader->city($ip);
$countryname = ($geo->country->name ?? null) ?: null;
$country = ($geo->country->isoCode ?? null) ?: null;
......@@ -315,7 +323,7 @@ foreach ($pods as $pod) {
$lat = ($geo->location->latitude ?? null) ?: 0;
$long = ($geo->location->longitude ?? null) ?: 0;
_debug('Location', json_encode($geo->raw), true);
debug('Location', json_encode($geo->raw), true);
}
echo $newline;
$statslastdate = date('Y-m-d H:i:s');
......@@ -339,27 +347,36 @@ foreach ($pods as $pod) {
die('Error in SQL query: ' . $e->getMessage());
}
_debug('Uptime', $uptime);
debug('Uptime', $uptime);
if ($score == ($notify_level - 1) && $notify && !$develop && $dbscore == $notify_level) {
$to = $email;
$headers = ['From: ' . c('adminemail'), 'Bcc: ' . c('adminemail')];
$subject = 'Monitoring notice from poduptime';
$message = 'Notice for ' . $domain . '. Your score is ' . $score . ' and your pod will fall off the list soon.';
$message = 'Notice for ' . $domain . '. Your score is ' . $score . ' and your pod will fall off the list soon. HTTP response of ' . $httpcode . '. ';
if ($outputsslerror) {
$message .= 'SSL Error ' . $outputsslerror;
}
if ($podminhelp) {
$message .= ' ' . $podminhelp;
}
@mail($to, $subject, $message, implode("\r\n", $headers));
_debug('Mail Notice', 'sent to ' . $email);
debug('Mail Message body', $message);
debug('Mail Notice', 'sent to ' . $email);
}
if ($score > 100) {
$score = 100;
} elseif ($score < 0) {
$score = 0;
} elseif ($score < -6000) {
$status = PodStatus::SYSTEM_DELETED;
}
$weightedscore = ($uptime + $score - (10 - $weight)) / 2;
_debug('Score', $score);
_debug('Weighted Score', $weightedscore);
debug('Score', $score);
debug('Weighted Score', $weightedscore);
try {
$p = R::findOne('pods', 'domain = ?', [$domain]);
......@@ -416,129 +433,4 @@ foreach ($pods as $pod) {
echo 'Success ' . $domain;
echo $newline;
echo $newline;
}
/**
* Output a debug message and variable value
*
* @param string $label
* @param mixed $var
* @param bool $dump
*/
function _debug($label, $var = null, $dump = false)
{
global $debug, $newline;
if (!$debug) {
return;
}
$output = (string) $var;
if ($dump || is_array($var)) {
$output = print_r($var, true);
} elseif (is_bool($var)) {
$output = $var ? 'true' : 'false';
}
printf('%s: %s%s', $label, $output, $newline);
}
/**
* Execute cURL request and return array of data.
*
* @param string $url
*
* @return array
*/
function _curl(string $url): array
{
$chss = curl_init();
curl_setopt($chss, CURLOPT_URL, $url);
curl_setopt($chss, CURLOPT_CONNECTTIMEOUT, 10);
curl_setopt($chss, CURLOPT_TIMEOUT, 20);
curl_setopt($chss, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($chss, CURLOPT_CERTINFO, 1);
curl_setopt($chss, CURLOPT_CAINFO, c('cafullpath'));
$data = [
'body' => curl_exec($chss),
'error' => curl_error($chss),
'info' => curl_getinfo($chss, CURLINFO_CERTINFO),
'conntime' => curl_getinfo($chss, CURLINFO_CONNECT_TIME),
'nstime' => curl_getinfo($chss, CURLINFO_NAMELOOKUP_TIME),
];
curl_close($chss);
return $data;
}
/**
* Get a language snippet from a given URL.
*
* @param string $url
*
* @return null|string
*/
function getWebsiteLanguageSnippetFromUrl(string $url): ?string
{
$curl = _curl($url);
if (!$curl['body']) {
return null;
}
libxml_use_internal_errors(true);
$d = new DOMDocument;
$d->loadHTML($curl['body']);
$snippet = $d->getElementsByTagName('title')->item(0)->textContent ?? '';
for ($type = 1; $type < 6; $type++) {
foreach ($d->getElementsByTagName('h' . $type) as $h) {
// Ignore possibly generic "JavaScript required" texts.
if (stripos($h->textContent, 'javascript') === false) {
$snippet .= ' ' . $h->textContent;
}
}
}
// Get descriptions of meta tags.
foreach ($d->getElementsByTagName('meta') as $meta) {
if (strtolower($meta->getAttribute('name')) === 'description') {
$snippet .= ' ' . $meta->getAttribute('value');
}
}
return $snippet;
}
/**
* Detect the language of the given text snippet.
*