Commit 5602a42b authored by dmorley's avatar dmorley Committed by noplanman
Browse files

Pull cleanup round 900

parent f267bd8f
...@@ -7,6 +7,8 @@ The format is based on [Keep a Changelog] and this project adheres to [Semantic ...@@ -7,6 +7,8 @@ The format is based on [Keep a Changelog] and this project adheres to [Semantic
- Config syntax has changed to array style (#155) - Config syntax has changed to array style (#155)
- Added `pghost` config to set database port - Added `pghost` config to set database port
- Added `CONTRIBUTING.md` - Added `CONTRIBUTING.md`
- Podmin email shares details on why pod is failing
- Only retrieve location data for remote servers / IPs
### Changed ### Changed
- Introduce proper changelog format (#189) - Introduce proper changelog format (#189)
- Moved DB migration scripts into `db` folder - Moved DB migration scripts into `db` folder
...@@ -15,6 +17,11 @@ The format is based on [Keep a Changelog] and this project adheres to [Semantic ...@@ -15,6 +17,11 @@ The format is based on [Keep a Changelog] and this project adheres to [Semantic
- Use pretty URLs (see nginx.example) - Use pretty URLs (see nginx.example)
- Open pod URLs in a new tab - Open pod URLs in a new tab
- Use detectlanguage.com API for language guess - Use detectlanguage.com API for language guess
- Only use JSON data, ignore HTML when returned
- Allow curl redirect on home page check
- Score now goes to -5000 before a pod is removed so dead pods get checked a while then removed for good
- Move functions to dedicated file to allow reuse
- Backup script rewrite
### Deprecated ### Deprecated
### Removed ### Removed
### Fixed ### Fixed
......
...@@ -9,33 +9,6 @@ declare(strict_types=1); ...@@ -9,33 +9,6 @@ declare(strict_types=1);
use DetectLanguage\DetectLanguage; use DetectLanguage\DetectLanguage;
use RedBeanPHP\R; use RedBeanPHP\R;
/**
* Helper to get config values.
*
* @param null|string $param
* @param null|mixed $default
*
* @return mixed|null
*/
function c(string $param = null, $default = null)
{
static $config;
if ($config === null) {
$config = require __DIR__ . '/config.php';
is_array($config) || die('Invalid config format.');
}
if ($param === null) {
return $config;
}
if (array_key_exists($param, $config)) {
return $config[$param];
}
return $default;
}
define('PODUPTIME', microtime(true)); define('PODUPTIME', microtime(true));
require_once __DIR__ . '/vendor/autoload.php'; require_once __DIR__ . '/vendor/autoload.php';
......
...@@ -22,13 +22,15 @@ ...@@ -22,13 +22,15 @@
"maxmind-db/reader": "^1.3", "maxmind-db/reader": "^1.3",
"matriphe/iso-639": "^1.2", "matriphe/iso-639": "^1.2",
"rinvex/country": "^3.1", "rinvex/country": "^3.1",
"detectlanguage/detectlanguage": "^2.2" "detectlanguage/detectlanguage": "^2.2",
"longman/ip-tools": "^1.2"
}, },
"require-dev": { "require-dev": {
"squizlabs/php_codesniffer": "^3.3" "squizlabs/php_codesniffer": "^3.3"
}, },
"autoload": { "autoload": {
"classmap": ["lib"] "classmap": ["lib"],
"files": ["lib/functions.php"]
}, },
"scripts": { "scripts": {
"check-code": [ "check-code": [
......
...@@ -4,7 +4,7 @@ ...@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically" "This file is @generated automatically"
], ],
"content-hash": "ae2037263bedfa647fb7fe96311a84da", "content-hash": "9ac3659b3b8b6f4a29e9a44e0dfd105f",
"packages": [ "packages": [
{ {
"name": "commerceguys/enum", "name": "commerceguys/enum",
...@@ -289,6 +289,62 @@ ...@@ -289,6 +289,62 @@
], ],
"time": "2018-07-30T20:23:10+00:00" "time": "2018-07-30T20:23:10+00:00"
}, },
{
"name": "longman/ip-tools",
"version": "1.2.1",
"source": {
"type": "git",
"url": "https://github.com/akalongman/php-ip-tools.git",
"reference": "6c050dfbf91811d14b9b3aa31fb7116eac0f0a18"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/akalongman/php-ip-tools/zipball/6c050dfbf91811d14b9b3aa31fb7116eac0f0a18",
"reference": "6c050dfbf91811d14b9b3aa31fb7116eac0f0a18",
"shasum": ""
},
"require": {
"ext-bcmath": "*",
"php": ">=5.5"
},
"require-dev": {
"phpspec/phpspec": "~2.1",
"phpunit/phpunit": "~4.1",
"squizlabs/php_codesniffer": "~2.3"
},
"type": "library",
"autoload": {
"psr-4": {
"Longman\\IPTools\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Avtandil Kikabidze aka LONGMAN",
"email": "akalongman@gmail.com",
"homepage": "https://longman.me",
"role": "Developer"
}
],
"description": "PHP IP Tools for manipulation with IPv4 and IPv6",
"homepage": "https://github.com/akalongman/php-ip-tools",
"keywords": [
"IP",
"Match",
"compare",
"ipv4",
"ipv6",
"mask",
"subnet",
"tools",
"utilities"
],
"time": "2016-10-23T20:08:46+00:00"
},
{ {
"name": "matriphe/iso-639", "name": "matriphe/iso-639",
"version": "1.2", "version": "1.2",
......
...@@ -6,25 +6,45 @@ ...@@ -6,25 +6,45 @@
declare(strict_types=1); declare(strict_types=1);
if (PHP_SAPI !== 'cli') { require_once __DIR__ . '/../boot.php';
if (!is_cli()) {
header('HTTP/1.0 403 Forbidden'); header('HTTP/1.0 403 Forbidden');
exit; exit;
} }
$c = require __DIR__ . '/../config.php'; $keep_for = 60 * 60 * 6; // 6 hours
$backup_file = c('backup_dir') . '/dump_' . date('Ymd_His') . '.sql';
printf("Making backup of '%s' to '%s'...", c('pgdb'), $backup_file);
system(sprintf(
'export PGPASSWORD=%3$s &&' .
'"%1$s" --clean --format=tar --username=%2$s %4$s >> "%5$s"',
c('pg_dump_dir') . '/pg_dump',
c('pguser'),
c('pgpass'),
c('pgdb'),
$backup_file
), $exit_code);
printf(" %s\n", $exit_code === 0 ? 'Success!' : 'Failed.');
$keep = (60 * 60 * 6) * 1; $dirh = dir(c('backup_dir'));
$dump_date = date('Ymd_Hs');
$file_name = "{$c['backup_dir']}/dump_{$dump_date}.sql";
system("export PGPASSWORD={$c['pgpass']} && {$c['pg_dump_dir']}/pg_dump --clean --format=tar --username={$c['pguser']} {$c['pgdb']} >> {$file_name}");
echo "pg backup of {$c['pgdb']} made";
$dirh = dir($c['backup_dir']);
while ($entry = $dirh->read()) { while ($entry = $dirh->read()) {
$old_file_time = date('U') - $keep; $file = c('backup_dir') . "/{$entry}";
$file_created = filectime("{$c['backup_dir']}/{$entry}");
if ($file_created < $old_file_time && !is_dir($entry)) { // Skip dotfiles and non-files (folders, symlinks, etc.).
if (unlink("{$c['backup_dir']}/{$entry}")) { if ($entry[0] === '.' || !is_file($file)) {
echo 'Cleaned up old backups'; continue;
}
} }
if (filectime($file) + $keep_for > time()) {
//echo "Don't delete {$entry}\n";
continue;
}
printf(
"Removing old file '%s'... %s\n",
$entry,
unlink($file) ? 'Success!' : 'Failed.'
);
} }
...@@ -6,15 +6,15 @@ ...@@ -6,15 +6,15 @@
declare(strict_types=1); declare(strict_types=1);
if (PHP_SAPI !== 'cli') {
header('HTTP/1.0 403 Forbidden');
exit;
}
use RedBeanPHP\R; use RedBeanPHP\R;
require_once __DIR__ . '/../boot.php'; require_once __DIR__ . '/../boot.php';
if (!is_cli()) {
header('HTTP/1.0 403 Forbidden');
exit;
}
try { try {
$monthly_totals = R::getAll(" $monthly_totals = R::getAll("
SELECT SELECT
......
...@@ -6,15 +6,15 @@ ...@@ -6,15 +6,15 @@
declare(strict_types=1); declare(strict_types=1);
if (PHP_SAPI !== 'cli') {
header('HTTP/1.0 403 Forbidden');
exit;
}
use RedBeanPHP\R; use RedBeanPHP\R;
require_once __DIR__ . '/../boot.php'; require_once __DIR__ . '/../boot.php';
if (!is_cli()) {
header('HTTP/1.0 403 Forbidden');
exit;
}
try { try {
$sql = ' $sql = '
SELECT domain, status SELECT domain, status
......
...@@ -6,15 +6,15 @@ ...@@ -6,15 +6,15 @@
declare(strict_types=1); declare(strict_types=1);
if (PHP_SAPI !== 'cli') {
header('HTTP/1.0 403 Forbidden');
exit;
}
use RedBeanPHP\R; use RedBeanPHP\R;
require_once __DIR__ . '/../boot.php'; require_once __DIR__ . '/../boot.php';
if (!is_cli()) {
header('HTTP/1.0 403 Forbidden');
exit;
}
$softwares = [ $softwares = [
'diaspora' => ['repo' => 'diaspora/diaspora', 'gitsite' => 'api.github.com', 'gittype' => 'github', 'devbranch' => 'develop'], 'diaspora' => ['repo' => 'diaspora/diaspora', 'gitsite' => 'api.github.com', 'gittype' => 'github', 'devbranch' => 'develop'],
'friendica' => ['repo' => 'friendica/friendica', 'gitsite' => 'api.github.com', 'gittype' => 'github', 'devbranch' => 'develop'], 'friendica' => ['repo' => 'friendica/friendica', 'gitsite' => 'api.github.com', 'gittype' => 'github', 'devbranch' => 'develop'],
......
...@@ -6,7 +6,9 @@ ...@@ -6,7 +6,9 @@
declare(strict_types=1); declare(strict_types=1);
if (!in_array(PHP_SAPI, ['cgi-fcgi', 'cli'])) { require_once __DIR__ . '/../boot.php';
if (!is_cli()) {
$referer = ($_SERVER['HTTP_REFERER'] ? parse_url($_SERVER['HTTP_REFERER'])['host'] : ''); $referer = ($_SERVER['HTTP_REFERER'] ? parse_url($_SERVER['HTTP_REFERER'])['host'] : '');
if ($referer !== $_SERVER['SERVER_NAME']) { if ($referer !== $_SERVER['SERVER_NAME']) {
header('HTTP/1.0 403 Forbidden'); header('HTTP/1.0 403 Forbidden');
...@@ -15,8 +17,8 @@ if (!in_array(PHP_SAPI, ['cgi-fcgi', 'cli'])) { ...@@ -15,8 +17,8 @@ if (!in_array(PHP_SAPI, ['cgi-fcgi', 'cli'])) {
} }
use Carbon\Carbon; use Carbon\Carbon;
use DetectLanguage\DetectLanguage;
use GeoIp2\Database\Reader; use GeoIp2\Database\Reader;
use Longman\IPTools\Ip;
use Poduptime\PodStatus; use Poduptime\PodStatus;
use RedBeanPHP\R; use RedBeanPHP\R;
...@@ -24,14 +26,12 @@ $debug = isset($_GET['debug']) || (isset($argv) && in_array('debug', $argv, t ...@@ -24,14 +26,12 @@ $debug = isset($_GET['debug']) || (isset($argv) && in_array('debug', $argv, t
$sqldebug = isset($_GET['sqldebug']) || (isset($argv) && in_array('sqldebug', $argv, true)); $sqldebug = isset($_GET['sqldebug']) || (isset($argv) && in_array('sqldebug', $argv, true));
$develop = isset($_GET['develop']) || (isset($argv) && in_array('develop', $argv, true)); $develop = isset($_GET['develop']) || (isset($argv) && in_array('develop', $argv, true));
$write = !(isset($_GET['nowrite']) || (isset($argv) && in_array('nowrite', $argv, true))); $write = !(isset($_GET['nowrite']) || (isset($argv) && in_array('nowrite', $argv, true)));
$newline = PHP_SAPI === 'cli' ? "\n\n" : '<br><br>'; $newline = is_cli() ? "\n\n" : '<br><br>';
$_domain = $_GET['domain'] ?? null; $_domain = $_GET['domain'] ?? null;
// Must have a domain, except if called from CLI. // Must have a domain, except if called from CLI.
$_domain || PHP_SAPI === 'cli' || die('No valid input'); $_domain || is_cli() || die('No valid input');
require_once __DIR__ . '/../boot.php';
$sqldebug && R::fancyDebug(true); $sqldebug && R::fancyDebug(true);
...@@ -49,11 +49,11 @@ try { ...@@ -49,11 +49,11 @@ try {
if ($_domain) { if ($_domain) {
$sql .= ' WHERE domain = ?'; $sql .= ' WHERE domain = ?';
$pods = R::getAll($sql, [$_domain]); $pods = R::getAll($sql, [$_domain]);
} elseif (PHP_SAPI === 'cli' && (isset($argv) && in_array('Check_System_Deleted', $argv, true))) { } elseif (is_cli() && (isset($argv) && in_array('Check_System_Deleted', $argv, true))) {
$sql .= ' WHERE status = ?'; $sql .= ' WHERE status = ? ORDER BY id';
$pods = R::getAll($sql, [PodStatus::SYSTEM_DELETED]); $pods = R::getAll($sql, [PodStatus::SYSTEM_DELETED]);
} elseif (PHP_SAPI === 'cli') { } elseif (is_cli()) {
$sql .= ' WHERE status < ?'; $sql .= ' WHERE status < ? ORDER BY id';
$pods = R::getAll($sql, [PodStatus::PAUSED]); $pods = R::getAll($sql, [PodStatus::PAUSED]);
} }
} catch (\RedBeanPHP\RedException $e) { } catch (\RedBeanPHP\RedException $e) {
...@@ -85,7 +85,7 @@ foreach ($pods as $pod) { ...@@ -85,7 +85,7 @@ foreach ($pods as $pod) {
die('Error in SQL query: ' . $e->getMessage()); die('Error in SQL query: ' . $e->getMessage());
} }
_debug('Domain', $domain); debug('Domain', $domain);
$user_rating = 0; $user_rating = 0;
...@@ -93,38 +93,39 @@ foreach ($pods as $pod) { ...@@ -93,38 +93,39 @@ foreach ($pods as $pod) {
$user_rating = round(array_sum($user_ratings) / count($user_ratings), 2); $user_rating = round(array_sum($user_ratings) / count($user_ratings), 2);
} }
$nodeinfo_meta = _curl("https://{$domain}/.well-known/nodeinfo"); $nodeinfo_meta = curl("https://{$domain}/.well-known/nodeinfo");
// Default link to fetch node info. // Default link to fetch node info.
$nodeinfo_url = "https://{$domain}/nodeinfo/1.0"; $nodeinfo_url = "https://{$domain}/nodeinfo/1.0";
if (!isset($nodeinfo_meta['error'])) { if ($info = json_decode($nodeinfo_meta['body'] ?: '', true)) {
$info = json_decode($nodeinfo_meta['body'], true);
$nodeinfo_url = max($info['links'])['href']; $nodeinfo_url = max($info['links'])['href'];
} }
_debug('Nodeinfo link', $nodeinfo_url); debug('Nodeinfo link', $nodeinfo_url);
$nodeinfo = _curl($nodeinfo_url);
$nodeinfo = curl($nodeinfo_url);
$outputssl = $nodeinfo['body']; $outputssl = $nodeinfo['body'];
$outputsslerror = $nodeinfo['error']; $outputsslerror = $nodeinfo['error'];
$info = $nodeinfo['info']; $info = $nodeinfo['info'];
$httpcode = $nodeinfo['code'];
$conntime = $nodeinfo['conntime']; $conntime = $nodeinfo['conntime'];
$nstime = $nodeinfo['nstime']; $nstime = $nodeinfo['nstime'];
$latency = $conntime - $nstime; $latency = $conntime - $nstime;
$sslexpire = $info[0]['Expire date'] ?? null; $sslexpire = $info[0]['Expire date'] ?? null;
_debug('Nodeinfo output', $outputssl, true); debug('Nodeinfo output', $outputssl, true);
_debug('Nodeinfo output error', $outputsslerror, true); debug('Nodeinfo output error', $outputsslerror, true);
_debug('Cert expire date', $sslexpire); debug('Nodeinfo service http response code', $httpcode);
_debug('Conntime', $conntime); debug('Cert expire date', $sslexpire);
_debug('NStime', $nstime); debug('Conntime', $conntime);
_debug('Latency', $latency); debug('NStime', $nstime);
debug('Latency', $latency);
$jsonssl = ($outputssl ? json_decode($outputssl) : null); $jsonssl = json_decode($outputssl ?: '');
if ($jsonssl !== null) { if ($jsonssl !== null) {
$version = $jsonssl->software->version ?? 0; $version = $jsonssl->software->version ?? 0;
preg_match_all('((?:\d(.|-)?)+(\.)\d+\.*)', $version, $sversion); preg_match_all('((?:\d(.|-)?)+(\.)\d+\.*)', $version, $sversion);
$shortversion = $sversion[0][0] ?? '0.0.0.0'; $shortversion = $sversion[0][0] ?? '0.0.0.0';
$signup = ($jsonssl->openRegistrations ?? false) === true; $signup = ($jsonssl->openRegistrations ?? false) === true;
...@@ -170,7 +171,7 @@ foreach ($pods as $pod) { ...@@ -170,7 +171,7 @@ foreach ($pods as $pod) {
die('Error in SQL query: ' . $e->getMessage()); die('Error in SQL query: ' . $e->getMessage());
} }
_debug('Version code', $shortversion); debug('Version code', $shortversion);
try { try {
$masterdata = R::getRow(' $masterdata = R::getRow('
...@@ -188,13 +189,13 @@ foreach ($pods as $pod) { ...@@ -188,13 +189,13 @@ foreach ($pods as $pod) {
$devlastcommit = ($masterdata['devlastcommit'] ?? date('Y-m-d H:i:s')); $devlastcommit = ($masterdata['devlastcommit'] ?? date('Y-m-d H:i:s'));
$releasedate = ($masterdata['releasedate'] ?? date('Y-m-d H:i:s')); $releasedate = ($masterdata['releasedate'] ?? date('Y-m-d H:i:s'));
_debug('Masterversion', $masterversion); debug('Masterversion', $masterversion);
$masterversioncheck = explode('.', $masterversion); $masterversioncheck = explode('.', $masterversion);
$shortversioncheck = (strpos($shortversion, '.') ? explode('.', $shortversion) : implode('.', ['0', preg_replace('/\D/', '', $shortversion), '0'])); $shortversioncheck = (strpos($shortversion, '.') ? explode('.', $shortversion) : implode('.', ['0', preg_replace('/\D/', '', $shortversion), '0']));
//this is still off with a pod with v1 as total version. cant explode that, won't have a [0] or [1] later to use either //this is still off with a pod with v1 as total version. cant explode that, won't have a [0] or [1] later to use either
_debug('Days since master code release', date_diff(new DateTime($releasedate), new DateTime())->format('%d')); debug('Days since master code release', date_diff(new DateTime($releasedate), new DateTime())->format('%d'));
try { try {
$lastpodupdates = R::getRow(' $lastpodupdates = R::getRow('
...@@ -212,7 +213,7 @@ foreach ($pods as $pod) { ...@@ -212,7 +213,7 @@ foreach ($pods as $pod) {
$lastdatechecked = ($lastpodupdates['date_checked'] ?? date('Y-m-d H:i:s')); $lastdatechecked = ($lastpodupdates['date_checked'] ?? date('Y-m-d H:i:s'));
$devlastdays = $devlastcommit ? date_diff(new DateTime($devlastcommit), new DateTime())->format('%a') : 30;//tmp//if no dev branch then what? $devlastdays = $devlastcommit ? date_diff(new DateTime($devlastcommit), new DateTime())->format('%a') : 30;//tmp//if no dev branch then what?
_debug('Dev last commit was ', $devlastdays); debug('Dev last commit was ', $devlastdays);
$updategap = date_diff(new DateTime($lastdatechecked), new DateTime($devlastcommit))->format('%a'); $updategap = date_diff(new DateTime($lastdatechecked), new DateTime($devlastcommit))->format('%a');
...@@ -221,28 +222,33 @@ foreach ($pods as $pod) { ...@@ -221,28 +222,33 @@ foreach ($pods as $pod) {
if ($updategap + $devlastdays > 400) { if ($updategap + $devlastdays > 400) {
_debug('Outdated More than 400 days', 'Yes'); debug('Outdated More than 400 days', 'Yes');
$podminhelp = 'Your code base seems too out of date to be used. Last time you updated was ' . $updategap;
$score -= 2; $score -= 2;
} }
} elseif (($masterversioncheck[1] - $shortversioncheck[1]) > 1) { } elseif (($masterversioncheck[1] - $shortversioncheck[1]) > 1) {
///tmp/If pod is two versions off AND it's been more than 60 days since that release came out AND your on the master production branch ///tmp/If pod is two versions off AND it's been more than 60 days since that release came out AND your on the master production branch
_debug('Outdated second decimal > 1', 'Yes'); debug('Outdated second decimal > 1', 'Yes');
$score -= 2; $score -= 2;
$updategap = date_diff(new DateTime($lastdatechecked), new DateTime($releasedate))->format('%a'); $updategap = date_diff(new DateTime($lastdatechecked), new DateTime($releasedate))->format('%a');
$podminhelp = 'Your code base seems too out of date to be used. Current version is ' . $masterversion . ' and you are running ' . $shortversion;
} elseif ($updategap - date_diff(new DateTime($releasedate), new DateTime())->format('%a') > 400) { } elseif ($updategap - date_diff(new DateTime($releasedate), new DateTime())->format('%a') > 400) {
_debug('Outdated more than 400 days since x ', 'Yes'); debug('Outdated more than 400 days since x ', 'Yes');
$score -= 2; $score -= 2;
$updategap = date_diff(new DateTime($lastdatechecked), new DateTime($releasedate))->format('%a'); $updategap = date_diff(new DateTime($lastdatechecked), new DateTime($releasedate))->format('%a');
$podminhelp = 'Your code base seems too out of date to be used. Last time you updated was ' . $updategap;
} else { } else {
$updategap = date_diff(new DateTime($lastdatechecked), new DateTime($releasedate))->format('%a'); $updategap = date_diff(new DateTime($lastdatechecked), new DateTime($releasedate))->format('%a');
} }
_debug('Pod code was updated after ', $updategap); debug('Pod code was updated after ', $updategap);
$status = PodStatus::UP; $status = PodStatus::UP;
} }
...@@ -254,14 +260,15 @@ foreach ($pods as $pod) { ...@@ -254,14 +260,15 @@ foreach ($pods as $pod) {
if (!$language_snippet) { if (!$language_snippet) {
$detectedlanguage = null; $detectedlanguage = null;
--$score; --$score;
$podminhelp = 'Unable to render the html on your main page https://' . $domain;
} elseif ($debug || Carbon::now()->hour === 12) { } elseif ($debug || Carbon::now()->hour === 12) {
$detectedlanguage = detectWebsiteLanguageFromSnippet($language_snippet); $detectedlanguage = detectWebsiteLanguageFromSnippet($language_snippet);
} }
_debug('Detected Language', $detectedlanguage); debug('Detected Language', $detectedlanguage);