Commit f267bd8f authored by noplanman's avatar noplanman
Browse files

Merge branch 'targetandlog' into 'develop'

Target and Log and CA

See merge request !206
parents c0f90add c694a60b
......@@ -10,17 +10,27 @@ The format is based on [Keep a Changelog] and this project adheres to [Semantic
### Changed
- Introduce proper changelog format (#189)
- Moved DB migration scripts into `db` folder
- Use Curl for all http calls
- Use filter dropdowns for pre-defined columns
- Use pretty URLs (see nginx.example)
- Open pod URLs in a new tab
- Use detectlanguage.com API for language guess
### Deprecated
### Removed
### Fixed
- Notify podmins just once at 50 when pod failing (#186)
- Add missing meta and PHP module requirements to `composer.json`
### Security
## [2.3.1] - 2018-08-05
### Added
- Podmins can link directly to their pod via `https://podupti.me/domain.name` for stats and to allow users to rate easier
- Wizard to help you filter the columns to what you need (#145)
- Cookie used to remember last pod you clicked
### Changed
- Now one table with a basic default view you can customize (#171)
- Switch to a library for country to lat long lookup
- Switch GeoIP from built in PHP to library and use newer Maxmind database file
## [2.3.0] - 2018-07-19
:exclamation: DB migrations required! (see [SQL migration script][2.2.0-sql-migration])
......
......@@ -10,7 +10,6 @@ OS Dependencies:
```
php7.2 php7.2-curl php7.2-pgsql php-geoip php7.2-cli php7.2-common php7.2-bcmath php7.2-json php7.2-readline php7.2-mbstring php7.2-xml php-cgi git curl postgresql postgresql-contrib dnsutils bind9 npm nodejs composer
```
GeoIP needs setup normally with a dat file
Yarn is a separate install: https://yarnpkg.com
......@@ -54,13 +53,14 @@ run `db/pull.sh debug` to debug output
run `db/pull.sh sqldebug` to debug sql
run `db/pull.sh develop` to run without email alerts to end users
run `db/pull.sh Check_System_Deleted` to re-check system deleted pods as needed
These commands can be combined
# To Upgrade:
```
git pull
yarn install
composer install
psql -U podupuser podupdb < db/migrationXXX.sql (see db/version.md for proper migration versions)
psql -U podupuser podupdb < db/migrations/xxx.sql (see db/migrations/README.md for proper migration versions)
```
# Status
......
......@@ -6,6 +6,7 @@
declare(strict_types=1);
use DetectLanguage\DetectLanguage;
use RedBeanPHP\R;
/**
......@@ -39,6 +40,10 @@ define('PODUPTIME', microtime(true));
require_once __DIR__ . '/vendor/autoload.php';
// Initialise language detection.
DetectLanguage::setApiKey(c('dlkey'));
DetectLanguage::setSecure(true);
// Set up global DB connection.
R::setup(
sprintf(
......
{
"name": "diasporg/poduptime",
"description": "Poduptime is software to get live stats and data on listed Diaspora Pods.",
"license": "AGPL-3.0-or-later",
"config": {
"platform": {
"php": "7.2"
......@@ -6,16 +9,20 @@
},
"require": {
"php": "^7.2",
"gabordemooij/redbean": "^5.0",
"nesbot/carbon": "^1.31",
"ext-curl": "*",
"ext-dom": "*",
"ext-json": "*",
"ext-libxml": "*",
"gabordemooij/redbean": "^5.1",
"nesbot/carbon": "^1.33",
"commerceguys/enum": "^1.0",
"noplanman/xec": "0.1.0",
"jaybizzle/crawler-detect": "1.*",
"patrickschur/language-detection": "^3.3",
"jaybizzle/crawler-detect": "^1.2",
"geoip2/geoip2": "^2.9",
"maxmind-db/reader": "~1.0",
"maxmind-db/reader": "^1.3",
"matriphe/iso-639": "^1.2",
"rinvex/country": "^3.1"
"rinvex/country": "^3.1",
"detectlanguage/detectlanguage": "^2.2"
},
"require-dev": {
"squizlabs/php_codesniffer": "^3.3"
......
{
"_readme": [
"This file locks the dependencies of your project to a known state",
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "b4dea96de8556414616e04d0f09aa8fa",
"content-hash": "ae2037263bedfa647fb7fe96311a84da",
"packages": [
{
"name": "commerceguys/enum",
......@@ -46,16 +46,16 @@
},
{
"name": "composer/ca-bundle",
"version": "1.1.1",
"version": "1.1.2",
"source": {
"type": "git",
"url": "https://github.com/composer/ca-bundle.git",
"reference": "d2c0a83b7533d6912e8d516756ebd34f893e9169"
"reference": "46afded9720f40b9dc63542af4e3e43a1177acb0"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/composer/ca-bundle/zipball/d2c0a83b7533d6912e8d516756ebd34f893e9169",
"reference": "d2c0a83b7533d6912e8d516756ebd34f893e9169",
"url": "https://api.github.com/repos/composer/ca-bundle/zipball/46afded9720f40b9dc63542af4e3e43a1177acb0",
"reference": "46afded9720f40b9dc63542af4e3e43a1177acb0",
"shasum": ""
},
"require": {
......@@ -98,7 +98,54 @@
"ssl",
"tls"
],
"time": "2018-03-29T19:57:20+00:00"
"time": "2018-08-08T08:57:40+00:00"
},
{
"name": "detectlanguage/detectlanguage",
"version": "2.2.0",
"source": {
"type": "git",
"url": "https://github.com/detectlanguage/detectlanguage-php.git",
"reference": "a410dc1cfb31a9e332bd779c1cb27b1af2ab9b18"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/detectlanguage/detectlanguage-php/zipball/a410dc1cfb31a9e332bd779c1cb27b1af2ab9b18",
"reference": "a410dc1cfb31a9e332bd779c1cb27b1af2ab9b18",
"shasum": ""
},
"require": {
"ext-json": "*",
"php": ">=5.3.0"
},
"require-dev": {
"phpunit/phpunit": "3.7.*"
},
"type": "library",
"autoload": {
"psr-0": {
"DetectLanguage": "lib/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Laurynas Butkus"
}
],
"description": "Language Detection API PHP Client",
"homepage": "https://github.com/detectlanguage/detectlanguage-php",
"keywords": [
"api",
"client",
"detect",
"detection",
"language"
],
"time": "2018-03-28T20:56:36+00:00"
},
{
"name": "gabordemooij/redbean",
......@@ -195,16 +242,16 @@
},
{
"name": "jaybizzle/crawler-detect",
"version": "v1.2.65",
"version": "v1.2.66",
"source": {
"type": "git",
"url": "https://github.com/JayBizzle/Crawler-Detect.git",
"reference": "1ea1c897018c93e3c2ac76e008519085b24e339e"
"reference": "55ad4b140d821c19dbaf1a6ea9d9e5ca6444ecc1"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/1ea1c897018c93e3c2ac76e008519085b24e339e",
"reference": "1ea1c897018c93e3c2ac76e008519085b24e339e",
"url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/55ad4b140d821c19dbaf1a6ea9d9e5ca6444ecc1",
"reference": "55ad4b140d821c19dbaf1a6ea9d9e5ca6444ecc1",
"shasum": ""
},
"require": {
......@@ -240,7 +287,7 @@
"crawlerdetect",
"php crawler detect"
],
"time": "2018-07-24T17:53:06+00:00"
"time": "2018-07-30T20:23:10+00:00"
},
{
"name": "matriphe/iso-639",
......@@ -390,16 +437,16 @@
},
{
"name": "nesbot/carbon",
"version": "1.32.0",
"version": "1.33.0",
"source": {
"type": "git",
"url": "https://github.com/briannesbitt/Carbon.git",
"reference": "64563e2b9f69e4db1b82a60e81efa327a30ff343"
"reference": "55667c1007a99e82030874b1bb14d24d07108413"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/briannesbitt/Carbon/zipball/64563e2b9f69e4db1b82a60e81efa327a30ff343",
"reference": "64563e2b9f69e4db1b82a60e81efa327a30ff343",
"url": "https://api.github.com/repos/briannesbitt/Carbon/zipball/55667c1007a99e82030874b1bb14d24d07108413",
"reference": "55667c1007a99e82030874b1bb14d24d07108413",
"shasum": ""
},
"require": {
......@@ -441,7 +488,7 @@
"datetime",
"time"
],
"time": "2018-07-05T06:59:26+00:00"
"time": "2018-08-07T08:39:47+00:00"
},
{
"name": "noplanman/xec",
......@@ -495,52 +542,6 @@
],
"time": "2017-03-12T19:16:23+00:00"
},
{
"name": "patrickschur/language-detection",
"version": "v3.3.0",
"source": {
"type": "git",
"url": "https://github.com/patrickschur/language-detection.git",
"reference": "21a2e7a1b9bf6bff578ac11c6dcf3d3668aeccdf"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/patrickschur/language-detection/zipball/21a2e7a1b9bf6bff578ac11c6dcf3d3668aeccdf",
"reference": "21a2e7a1b9bf6bff578ac11c6dcf3d3668aeccdf",
"shasum": ""
},
"require": {
"ext-mbstring": "*",
"php": "^7"
},
"require-dev": {
"phpunit/phpunit": "^6"
},
"type": "library",
"autoload": {
"psr-4": {
"LanguageDetection\\": "src/LanguageDetection"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Patrick Schur",
"email": "patrick_schur@outlook.de"
}
],
"description": "A language detection library for PHP. Detects the language from a given text string.",
"homepage": "https://github.com/patrickschur/language-detection",
"keywords": [
"detect",
"detection",
"language"
],
"time": "2018-02-01T17:12:47+00:00"
},
{
"name": "rinvex/country",
"version": "v3.1.0",
......@@ -617,16 +618,16 @@
},
{
"name": "symfony/polyfill-mbstring",
"version": "v1.8.0",
"version": "v1.9.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-mbstring.git",
"reference": "3296adf6a6454a050679cde90f95350ad604b171"
"reference": "d0cd638f4634c16d8df4508e847f14e9e43168b8"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/3296adf6a6454a050679cde90f95350ad604b171",
"reference": "3296adf6a6454a050679cde90f95350ad604b171",
"url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/d0cd638f4634c16d8df4508e847f14e9e43168b8",
"reference": "d0cd638f4634c16d8df4508e847f14e9e43168b8",
"shasum": ""
},
"require": {
......@@ -638,7 +639,7 @@
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "1.8-dev"
"dev-master": "1.9-dev"
}
},
"autoload": {
......@@ -672,20 +673,20 @@
"portable",
"shim"
],
"time": "2018-04-26T10:06:28+00:00"
"time": "2018-08-06T14:22:27+00:00"
},
{
"name": "symfony/translation",
"version": "v4.1.2",
"version": "v4.1.3",
"source": {
"type": "git",
"url": "https://github.com/symfony/translation.git",
"reference": "2dd74d6b2dcbd46a93971e6ce7d245cf3123e957"
"reference": "6fcd1bd44fd6d7181e6ea57a6f4e08a09b29ef65"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/translation/zipball/2dd74d6b2dcbd46a93971e6ce7d245cf3123e957",
"reference": "2dd74d6b2dcbd46a93971e6ce7d245cf3123e957",
"url": "https://api.github.com/repos/symfony/translation/zipball/6fcd1bd44fd6d7181e6ea57a6f4e08a09b29ef65",
"reference": "6fcd1bd44fd6d7181e6ea57a6f4e08a09b29ef65",
"shasum": ""
},
"require": {
......@@ -741,22 +742,22 @@
],
"description": "Symfony Translation Component",
"homepage": "https://symfony.com",
"time": "2018-07-23T08:20:20+00:00"
"time": "2018-07-26T11:24:31+00:00"
}
],
"packages-dev": [
{
"name": "squizlabs/php_codesniffer",
"version": "3.3.0",
"version": "3.3.1",
"source": {
"type": "git",
"url": "https://github.com/squizlabs/PHP_CodeSniffer.git",
"reference": "d86873af43b4aa9d1f39a3601cc0cfcf02b25266"
"reference": "628a481780561150481a9ec74709092b9759b3ec"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/d86873af43b4aa9d1f39a3601cc0cfcf02b25266",
"reference": "d86873af43b4aa9d1f39a3601cc0cfcf02b25266",
"url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/628a481780561150481a9ec74709092b9759b3ec",
"reference": "628a481780561150481a9ec74709092b9759b3ec",
"shasum": ""
},
"require": {
......@@ -794,7 +795,7 @@
"phpcs",
"standards"
],
"time": "2018-06-06T23:58:19+00:00"
"time": "2018-07-26T23:47:18+00:00"
}
],
"aliases": [],
......@@ -803,7 +804,11 @@
"prefer-stable": false,
"prefer-lowest": false,
"platform": {
"php": "^7.2"
"php": "^7.2",
"ext-curl": "*",
"ext-dom": "*",
"ext-json": "*",
"ext-libxml": "*"
},
"platform-dev": [],
"platform-overrides": {
......
......@@ -43,4 +43,7 @@ return [
//Geolite2-city database file in mmdb format - full file path (pull.sh will update this monthly)
'geoip2db' => '',
//detectlanguage.com api key
'dlkey' => '',
];
......@@ -14,8 +14,9 @@ if (!in_array(PHP_SAPI, ['cgi-fcgi', 'cli'])) {
}
}
use Carbon\Carbon;
use DetectLanguage\DetectLanguage;
use GeoIp2\Database\Reader;
use LanguageDetection\Language;
use Poduptime\PodStatus;
use RedBeanPHP\R;
......@@ -39,7 +40,7 @@ try {
$reader = new Reader(c('geoip2db'));
$sql = '
SELECT domain, score, date_created, weight, podmin_notify, email, masterversion, shortversion, status
SELECT domain, score, date_created, weight, podmin_notify, email, masterversion, shortversion, status, detectedlanguage
FROM pods
';
......@@ -72,6 +73,7 @@ foreach ($pods as $pod) {
$masterv = $pod['masterversion'];
$shortv = $pod['shortversion'];
$dbstatus = $pod['status'];
$language = $pod['detectedlanguage'];
try {
$ratings = R::getAll('
......@@ -91,25 +93,24 @@ foreach ($pods as $pod) {
$user_rating = round(array_sum($user_ratings) / count($user_ratings), 2);
}
// Default link to fetch node info.
$link = "https://{$domain}/nodeinfo/1.0";
$nodeinfo_meta = _curl("https://{$domain}/.well-known/nodeinfo");
extract(_curl("https://{$domain}/.well-known/nodeinfo"));
$outputwellknown = $curl_body;
// Default link to fetch node info.
$nodeinfo_url = "https://{$domain}/nodeinfo/1.0";
if ($outputwellknown !== false) {
$info = json_decode($outputwellknown, true);
$link = max($info['links'])['href'];
if (!isset($nodeinfo_meta['error'])) {
$info = json_decode($nodeinfo_meta['body'], true);
$nodeinfo_url = max($info['links'])['href'];
}
_debug('Nodeinfo link', $link);
extract(_curl($link));
_debug('Nodeinfo link', $nodeinfo_url);
$nodeinfo = _curl($nodeinfo_url);
$outputssl = $curl_body;
$outputsslerror = $curl_error;
$info = $curl_info;
$conntime = $curl_conntime;
$nstime = $curl_nstime;
$outputssl = $nodeinfo['body'];
$outputsslerror = $nodeinfo['error'];
$info = $nodeinfo['info'];
$conntime = $nodeinfo['conntime'];
$nstime = $nodeinfo['nstime'];
$latency = $conntime - $nstime;
$sslexpire = $info[0]['Expire date'] ?? null;
......@@ -123,7 +124,7 @@ foreach ($pods as $pod) {
$jsonssl = ($outputssl ? json_decode($outputssl) : null);
if ($jsonssl !== null) {
$version = $jsonssl->software->version ?? 0;
$version = $jsonssl->software->version ?? 0;
preg_match_all('((?:\d(.|-)?)+(\.)\d+\.*)', $version, $sversion);
$shortversion = $sversion[0][0] ?? '0.0.0.0';
$signup = ($jsonssl->openRegistrations ?? false) === true;
......@@ -246,25 +247,20 @@ foreach ($pods as $pod) {
$status = PodStatus::UP;
}
$d = new DOMDocument;
libxml_use_internal_errors(true);
extract(_curl("https://{$domain}/"));
$outputbody = $curl_body;
($outputbody ? $d->loadHTML($outputbody) : $d->loadHTML('<html></html>'));
$body = $d->getElementsByTagName('html')->item(0);
if ($body) {
$ld = new Language;
$detectedlanguage = key($ld->detect($body->nodeValue)->bestResults()->close());
} else {
$score -= 1;
// Default to the already saved language.
$detectedlanguage = $language;
$language_snippet = getWebsiteLanguageSnippetFromUrl("https://{$domain}/");
if (!$language_snippet) {
$detectedlanguage = null;
--$score;
} elseif ($debug || Carbon::now()->hour === 12) {
$detectedlanguage = detectWebsiteLanguageFromSnippet($language_snippet);
}
_debug('Detected Language', $detectedlanguage);
if (!$jsonssl || !$body) {
if (!$jsonssl || !$language_snippet) {
_debug('Connection', 'Can not connect to pod');
try {
......@@ -282,7 +278,7 @@ foreach ($pods as $pod) {
die('Error in SQL query: ' . $e->getMessage());
}
$score -= 1;
--$score;
$status = PodStatus::DOWN;
}
......@@ -356,7 +352,7 @@ foreach ($pods as $pod) {
if ($score > 100) {
$score = 100;
} elseif ($score < 0) {
$score = 0;
$score = 0;
$status = PodStatus::SYSTEM_DELETED;
}
$weightedscore = ($uptime + $score - (10 - $weight)) / 2;
......@@ -447,23 +443,101 @@ function _debug($label, $var = null, $dump = false)
printf('%s: %s%s', $label, $output, $newline);
}
function _curl($url)
/**
* Execute cURL request and return array of data.
*
* @param string $url
*
* @return array
*/
function _curl(string $url): array
{
global $cafullpath;
$chss = curl_init();
curl_setopt($chss, CURLOPT_URL, $url);
curl_setopt($chss, CURLOPT_CONNECTTIMEOUT, 10);
curl_setopt($chss, CURLOPT_TIMEOUT, 20);
curl_setopt($chss, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($chss, CURLOPT_CERTINFO, 1);
curl_setopt($chss, CURLOPT_CAINFO, $cafullpath);
return [
'curl_body' => curl_exec($chss),
'curl_error' => curl_error($chss),
'curl_info' => curl_getinfo($chss, CURLINFO_CERTINFO),
'curl_conntime' => curl_getinfo($chss, CURLINFO_CONNECT_TIME),
'curl_nstime' => curl_getinfo($chss, CURLINFO_NAMELOOKUP_TIME)
curl_setopt($chss, CURLOPT_CAINFO, c('cafullpath'));
$data = [
'body' => curl_exec($chss),
'error' => curl_error($chss),
'info' => curl_getinfo($chss, CURLINFO_CERTINFO),
'conntime' => curl_getinfo($chss, CURLINFO_CONNECT_TIME),
'nstime' => curl_getinfo($chss, CURLINFO_NAMELOOKUP_TIME),
];
curl_close($chss);
return $data;
}
/**
* Get a language snippet from a given URL.
*
* @param string $url
*
* @return null|string
*/
function getWebsiteLanguageSnippetFromUrl(string $url): ?string
{
$curl = _curl($url);
if (!$curl['body']) {
return null;
}
libxml_use_internal_errors(true);
$d = new DOMDocument;
$d->loadHTML($curl['body']);
$snippet = $d->getElementsByTagName('title')->item(0)->textContent ?? '';
for ($type = 1; $type < 6; $type++) {
foreach ($d->getElementsByTagName('h' . $type) as $h) {
// Ignore possibly generic "JavaScript required" texts.
if (stripos($h->textContent, 'javascript') === false) {
$snippet .= ' ' . $h->textContent;
}
}
}
// Get descriptions of meta tags.
foreach ($d->getElementsByTagName('meta') as $meta) {
if (strtolower($meta->getAttribute('name')) === 'description') {
$snippet .= ' ' . $meta->getAttribute('value');
}
}
return $snippet;
}
/**
* Detect the language of the given text snippet.
*
* @param string $snippet
*
* @return null|string
*/
function detectWebsiteLanguageFromSnippet(string $snippet): ?string
{
if (!$snippet) {
return null;
}
return DetectLanguage::simpleDetect($snippet);
}
/**
* Detect the website language of the given URL.