Commit f267bd8f authored by noplanman's avatar noplanman

Merge branch 'targetandlog' into 'develop'

Target and Log and CA

See merge request !206
parents c0f90add c694a60b
...@@ -10,17 +10,27 @@ The format is based on [Keep a Changelog] and this project adheres to [Semantic ...@@ -10,17 +10,27 @@ The format is based on [Keep a Changelog] and this project adheres to [Semantic
### Changed ### Changed
- Introduce proper changelog format (#189) - Introduce proper changelog format (#189)
- Moved DB migration scripts into `db` folder - Moved DB migration scripts into `db` folder
- Use Curl for all http calls
- Use filter dropdowns for pre-defined columns
- Use pretty URLs (see nginx.example)
- Open pod URLs in a new tab
- Use detectlanguage.com API for language guess
### Deprecated ### Deprecated
### Removed ### Removed
### Fixed ### Fixed
- Notify podmins just once at 50 when pod failing (#186)
- Add missing meta and PHP module requirements to `composer.json`
### Security ### Security
## [2.3.1] - 2018-08-05 ## [2.3.1] - 2018-08-05
### Added ### Added
- Podmins can link directly to their pod via `https://podupti.me/domain.name` for stats and to allow users to rate easier - Podmins can link directly to their pod via `https://podupti.me/domain.name` for stats and to allow users to rate easier
- Wizard to help you filter the columns to what you need (#145) - Wizard to help you filter the columns to what you need (#145)
- Cookie used to remember last pod you clicked
### Changed ### Changed
- Now one table with a basic default view you can customize (#171) - Now one table with a basic default view you can customize (#171)
- Switch to a library for country to lat long lookup
- Switch GeoIP from built in PHP to library and use newer Maxmind database file
## [2.3.0] - 2018-07-19 ## [2.3.0] - 2018-07-19
:exclamation: DB migrations required! (see [SQL migration script][2.2.0-sql-migration]) :exclamation: DB migrations required! (see [SQL migration script][2.2.0-sql-migration])
......
...@@ -10,7 +10,6 @@ OS Dependencies: ...@@ -10,7 +10,6 @@ OS Dependencies:
``` ```
php7.2 php7.2-curl php7.2-pgsql php-geoip php7.2-cli php7.2-common php7.2-bcmath php7.2-json php7.2-readline php7.2-mbstring php7.2-xml php-cgi git curl postgresql postgresql-contrib dnsutils bind9 npm nodejs composer php7.2 php7.2-curl php7.2-pgsql php-geoip php7.2-cli php7.2-common php7.2-bcmath php7.2-json php7.2-readline php7.2-mbstring php7.2-xml php-cgi git curl postgresql postgresql-contrib dnsutils bind9 npm nodejs composer
``` ```
GeoIP needs setup normally with a dat file
Yarn is a separate install: https://yarnpkg.com Yarn is a separate install: https://yarnpkg.com
...@@ -54,13 +53,14 @@ run `db/pull.sh debug` to debug output ...@@ -54,13 +53,14 @@ run `db/pull.sh debug` to debug output
run `db/pull.sh sqldebug` to debug sql run `db/pull.sh sqldebug` to debug sql
run `db/pull.sh develop` to run without email alerts to end users run `db/pull.sh develop` to run without email alerts to end users
run `db/pull.sh Check_System_Deleted` to re-check system deleted pods as needed run `db/pull.sh Check_System_Deleted` to re-check system deleted pods as needed
These commands can be combined
# To Upgrade: # To Upgrade:
``` ```
git pull git pull
yarn install yarn install
composer install composer install
psql -U podupuser podupdb < db/migrationXXX.sql (see db/version.md for proper migration versions) psql -U podupuser podupdb < db/migrations/xxx.sql (see db/migrations/README.md for proper migration versions)
``` ```
# Status # Status
......
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
declare(strict_types=1); declare(strict_types=1);
use DetectLanguage\DetectLanguage;
use RedBeanPHP\R; use RedBeanPHP\R;
/** /**
...@@ -39,6 +40,10 @@ define('PODUPTIME', microtime(true)); ...@@ -39,6 +40,10 @@ define('PODUPTIME', microtime(true));
require_once __DIR__ . '/vendor/autoload.php'; require_once __DIR__ . '/vendor/autoload.php';
// Initialise language detection.
DetectLanguage::setApiKey(c('dlkey'));
DetectLanguage::setSecure(true);
// Set up global DB connection. // Set up global DB connection.
R::setup( R::setup(
sprintf( sprintf(
......
{ {
"name": "diasporg/poduptime",
"description": "Poduptime is software to get live stats and data on listed Diaspora Pods.",
"license": "AGPL-3.0-or-later",
"config": { "config": {
"platform": { "platform": {
"php": "7.2" "php": "7.2"
...@@ -6,16 +9,20 @@ ...@@ -6,16 +9,20 @@
}, },
"require": { "require": {
"php": "^7.2", "php": "^7.2",
"gabordemooij/redbean": "^5.0", "ext-curl": "*",
"nesbot/carbon": "^1.31", "ext-dom": "*",
"ext-json": "*",
"ext-libxml": "*",
"gabordemooij/redbean": "^5.1",
"nesbot/carbon": "^1.33",
"commerceguys/enum": "^1.0", "commerceguys/enum": "^1.0",
"noplanman/xec": "0.1.0", "noplanman/xec": "0.1.0",
"jaybizzle/crawler-detect": "1.*", "jaybizzle/crawler-detect": "^1.2",
"patrickschur/language-detection": "^3.3",
"geoip2/geoip2": "^2.9", "geoip2/geoip2": "^2.9",
"maxmind-db/reader": "~1.0", "maxmind-db/reader": "^1.3",
"matriphe/iso-639": "^1.2", "matriphe/iso-639": "^1.2",
"rinvex/country": "^3.1" "rinvex/country": "^3.1",
"detectlanguage/detectlanguage": "^2.2"
}, },
"require-dev": { "require-dev": {
"squizlabs/php_codesniffer": "^3.3" "squizlabs/php_codesniffer": "^3.3"
......
This diff is collapsed.
...@@ -43,4 +43,7 @@ return [ ...@@ -43,4 +43,7 @@ return [
//Geolite2-city database file in mmdb format - full file path (pull.sh will update this monthly) //Geolite2-city database file in mmdb format - full file path (pull.sh will update this monthly)
'geoip2db' => '', 'geoip2db' => '',
//detectlanguage.com api key
'dlkey' => '',
]; ];
...@@ -14,8 +14,9 @@ if (!in_array(PHP_SAPI, ['cgi-fcgi', 'cli'])) { ...@@ -14,8 +14,9 @@ if (!in_array(PHP_SAPI, ['cgi-fcgi', 'cli'])) {
} }
} }
use Carbon\Carbon;
use DetectLanguage\DetectLanguage;
use GeoIp2\Database\Reader; use GeoIp2\Database\Reader;
use LanguageDetection\Language;
use Poduptime\PodStatus; use Poduptime\PodStatus;
use RedBeanPHP\R; use RedBeanPHP\R;
...@@ -39,7 +40,7 @@ try { ...@@ -39,7 +40,7 @@ try {
$reader = new Reader(c('geoip2db')); $reader = new Reader(c('geoip2db'));
$sql = ' $sql = '
SELECT domain, score, date_created, weight, podmin_notify, email, masterversion, shortversion, status SELECT domain, score, date_created, weight, podmin_notify, email, masterversion, shortversion, status, detectedlanguage
FROM pods FROM pods
'; ';
...@@ -72,6 +73,7 @@ foreach ($pods as $pod) { ...@@ -72,6 +73,7 @@ foreach ($pods as $pod) {
$masterv = $pod['masterversion']; $masterv = $pod['masterversion'];
$shortv = $pod['shortversion']; $shortv = $pod['shortversion'];
$dbstatus = $pod['status']; $dbstatus = $pod['status'];
$language = $pod['detectedlanguage'];
try { try {
$ratings = R::getAll(' $ratings = R::getAll('
...@@ -91,25 +93,24 @@ foreach ($pods as $pod) { ...@@ -91,25 +93,24 @@ foreach ($pods as $pod) {
$user_rating = round(array_sum($user_ratings) / count($user_ratings), 2); $user_rating = round(array_sum($user_ratings) / count($user_ratings), 2);
} }
// Default link to fetch node info. $nodeinfo_meta = _curl("https://{$domain}/.well-known/nodeinfo");
$link = "https://{$domain}/nodeinfo/1.0";
extract(_curl("https://{$domain}/.well-known/nodeinfo")); // Default link to fetch node info.
$outputwellknown = $curl_body; $nodeinfo_url = "https://{$domain}/nodeinfo/1.0";
if ($outputwellknown !== false) { if (!isset($nodeinfo_meta['error'])) {
$info = json_decode($outputwellknown, true); $info = json_decode($nodeinfo_meta['body'], true);
$link = max($info['links'])['href']; $nodeinfo_url = max($info['links'])['href'];
} }
_debug('Nodeinfo link', $link); _debug('Nodeinfo link', $nodeinfo_url);
extract(_curl($link)); $nodeinfo = _curl($nodeinfo_url);
$outputssl = $curl_body; $outputssl = $nodeinfo['body'];
$outputsslerror = $curl_error; $outputsslerror = $nodeinfo['error'];
$info = $curl_info; $info = $nodeinfo['info'];
$conntime = $curl_conntime; $conntime = $nodeinfo['conntime'];
$nstime = $curl_nstime; $nstime = $nodeinfo['nstime'];
$latency = $conntime - $nstime; $latency = $conntime - $nstime;
$sslexpire = $info[0]['Expire date'] ?? null; $sslexpire = $info[0]['Expire date'] ?? null;
...@@ -123,7 +124,7 @@ foreach ($pods as $pod) { ...@@ -123,7 +124,7 @@ foreach ($pods as $pod) {
$jsonssl = ($outputssl ? json_decode($outputssl) : null); $jsonssl = ($outputssl ? json_decode($outputssl) : null);
if ($jsonssl !== null) { if ($jsonssl !== null) {
$version = $jsonssl->software->version ?? 0; $version = $jsonssl->software->version ?? 0;
preg_match_all('((?:\d(.|-)?)+(\.)\d+\.*)', $version, $sversion); preg_match_all('((?:\d(.|-)?)+(\.)\d+\.*)', $version, $sversion);
$shortversion = $sversion[0][0] ?? '0.0.0.0'; $shortversion = $sversion[0][0] ?? '0.0.0.0';
$signup = ($jsonssl->openRegistrations ?? false) === true; $signup = ($jsonssl->openRegistrations ?? false) === true;
...@@ -246,25 +247,20 @@ foreach ($pods as $pod) { ...@@ -246,25 +247,20 @@ foreach ($pods as $pod) {
$status = PodStatus::UP; $status = PodStatus::UP;
} }
$d = new DOMDocument; // Default to the already saved language.
libxml_use_internal_errors(true); $detectedlanguage = $language;
extract(_curl("https://{$domain}/"));
$outputbody = $curl_body; $language_snippet = getWebsiteLanguageSnippetFromUrl("https://{$domain}/");
($outputbody ? $d->loadHTML($outputbody) : $d->loadHTML('<html></html>')); if (!$language_snippet) {
$body = $d->getElementsByTagName('html')->item(0);
if ($body) {
$ld = new Language;
$detectedlanguage = key($ld->detect($body->nodeValue)->bestResults()->close());
} else {
$score -= 1;
$detectedlanguage = null; $detectedlanguage = null;
--$score;
} elseif ($debug || Carbon::now()->hour === 12) {
$detectedlanguage = detectWebsiteLanguageFromSnippet($language_snippet);
} }
_debug('Detected Language', $detectedlanguage); _debug('Detected Language', $detectedlanguage);
if (!$jsonssl || !$body) { if (!$jsonssl || !$language_snippet) {
_debug('Connection', 'Can not connect to pod'); _debug('Connection', 'Can not connect to pod');
try { try {
...@@ -282,7 +278,7 @@ foreach ($pods as $pod) { ...@@ -282,7 +278,7 @@ foreach ($pods as $pod) {
die('Error in SQL query: ' . $e->getMessage()); die('Error in SQL query: ' . $e->getMessage());
} }
$score -= 1; --$score;
$status = PodStatus::DOWN; $status = PodStatus::DOWN;
} }
...@@ -356,7 +352,7 @@ foreach ($pods as $pod) { ...@@ -356,7 +352,7 @@ foreach ($pods as $pod) {
if ($score > 100) { if ($score > 100) {
$score = 100; $score = 100;
} elseif ($score < 0) { } elseif ($score < 0) {
$score = 0; $score = 0;
$status = PodStatus::SYSTEM_DELETED; $status = PodStatus::SYSTEM_DELETED;
} }
$weightedscore = ($uptime + $score - (10 - $weight)) / 2; $weightedscore = ($uptime + $score - (10 - $weight)) / 2;
...@@ -447,23 +443,101 @@ function _debug($label, $var = null, $dump = false) ...@@ -447,23 +443,101 @@ function _debug($label, $var = null, $dump = false)
printf('%s: %s%s', $label, $output, $newline); printf('%s: %s%s', $label, $output, $newline);
} }
function _curl($url) /**
* Execute cURL request and return array of data.
*
* @param string $url
*
* @return array
*/
function _curl(string $url): array
{ {
global $cafullpath;
$chss = curl_init(); $chss = curl_init();
curl_setopt($chss, CURLOPT_URL, $url); curl_setopt($chss, CURLOPT_URL, $url);
curl_setopt($chss, CURLOPT_CONNECTTIMEOUT, 10); curl_setopt($chss, CURLOPT_CONNECTTIMEOUT, 10);
curl_setopt($chss, CURLOPT_TIMEOUT, 20); curl_setopt($chss, CURLOPT_TIMEOUT, 20);
curl_setopt($chss, CURLOPT_RETURNTRANSFER, 1); curl_setopt($chss, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($chss, CURLOPT_CERTINFO, 1); curl_setopt($chss, CURLOPT_CERTINFO, 1);
curl_setopt($chss, CURLOPT_CAINFO, $cafullpath); curl_setopt($chss, CURLOPT_CAINFO, c('cafullpath'));
return [ $data = [
'curl_body' => curl_exec($chss), 'body' => curl_exec($chss),
'curl_error' => curl_error($chss), 'error' => curl_error($chss),
'curl_info' => curl_getinfo($chss, CURLINFO_CERTINFO), 'info' => curl_getinfo($chss, CURLINFO_CERTINFO),
'curl_conntime' => curl_getinfo($chss, CURLINFO_CONNECT_TIME), 'conntime' => curl_getinfo($chss, CURLINFO_CONNECT_TIME),
'curl_nstime' => curl_getinfo($chss, CURLINFO_NAMELOOKUP_TIME) 'nstime' => curl_getinfo($chss, CURLINFO_NAMELOOKUP_TIME),
]; ];
curl_close($chss); curl_close($chss);
return $data;
}
/**
* Get a language snippet from a given URL.
*
* @param string $url
*
* @return null|string
*/
function getWebsiteLanguageSnippetFromUrl(string $url): ?string
{
$curl = _curl($url);
if (!$curl['body']) {
return null;
}
libxml_use_internal_errors(true);
$d = new DOMDocument;
$d->loadHTML($curl['body']);
$snippet = $d->getElementsByTagName('title')->item(0)->textContent ?? '';
for ($type = 1; $type < 6; $type++) {
foreach ($d->getElementsByTagName('h' . $type) as $h) {
// Ignore possibly generic "JavaScript required" texts.
if (stripos($h->textContent, 'javascript') === false) {
$snippet .= ' ' . $h->textContent;
}
}
}
// Get descriptions of meta tags.
foreach ($d->getElementsByTagName('meta') as $meta) {
if (strtolower($meta->getAttribute('name')) === 'description') {
$snippet .= ' ' . $meta->getAttribute('value');
}
}
return $snippet;
}
/**
* Detect the language of the given text snippet.
*
* @param string $snippet
*
* @return null|string
*/
function detectWebsiteLanguageFromSnippet(string $snippet): ?string
{
if (!$snippet) {
return null;
}
return DetectLanguage::simpleDetect($snippet);
}
/**
* Detect the website language of the given URL.
*
* @param string $url
*
* @return null|string
*/
function detectWebsiteLanguageFromUrl(string $url): ?string
{
return detectWebsiteLanguageFromSnippet(
getWebsiteLanguageSnippetFromUrl($url)
);
} }
...@@ -12,6 +12,8 @@ use RedBeanPHP\R; ...@@ -12,6 +12,8 @@ use RedBeanPHP\R;
require_once __DIR__ . '/boot.php'; require_once __DIR__ . '/boot.php';
$iso = new Matriphe\ISO639\ISO639;
try { try {
$pods = R::getAll(' $pods = R::getAll('
SELECT domain, dnssec, podmin_statement, sslexpire, masterversion, shortversion, softwarename, daysmonitored, monthsmonitored, score, signup, name, country, countryname, city, state, detectedlanguage, uptime_alltime, active_users_halfyear, active_users_monthly, service_facebook, service_twitter, service_tumblr, service_wordpress, service_xmpp, latency, date_updated, ipv6, total_users, local_posts, comment_counts, userrating, status SELECT domain, dnssec, podmin_statement, sslexpire, masterversion, shortversion, softwarename, daysmonitored, monthsmonitored, score, signup, name, country, countryname, city, state, detectedlanguage, uptime_alltime, active_users_halfyear, active_users_monthly, service_facebook, service_twitter, service_tumblr, service_wordpress, service_xmpp, latency, date_updated, ipv6, total_users, local_posts, comment_counts, userrating, status
...@@ -31,9 +33,9 @@ foreach ($pods as $pod) { ...@@ -31,9 +33,9 @@ foreach ($pods as $pod) {
$tip = "Over the last {$humanmonitored} uptime was {$pod['uptime_alltime']}% and response time from Los Angeles was {$pod['latency']}ms."; $tip = "Over the last {$humanmonitored} uptime was {$pod['uptime_alltime']}% and response time from Los Angeles was {$pod['latency']}ms.";
if (($_COOKIE['domain'] ?? null) === $pod['domain']) { if (($_COOKIE['domain'] ?? null) === $pod['domain']) {
echo '<tr><td title="This is the last pod you visited from this site. ' . $tip . '" data-placement="right" data-toggle="tooltip" class="bg-secondary"><a class="text-warning url" target="_self" href="/go.php?domain=' . $pod['domain'] . '">' . $pod['domain'] . '</a></td>'; echo '<tr><td title="This is the last pod you visited from this site. ' . $tip . '" data-placement="right" data-toggle="tooltip" class="bg-secondary"><a class="text-warning url" target="_pod" href="/go.php?domain=' . $pod['domain'] . '">' . $pod['domain'] . '</a></td>';
} else { } else {
echo '<tr><td data-placement="right" title="' . $tip . '" data-toggle="tooltip"><a class="text-success url" target="_self" href="/go.php?domain=' . $pod['domain'] . '">' . $pod['domain'] . '</a></td>'; echo '<tr><td data-placement="right" title="' . $tip . '" data-toggle="tooltip"><a class="text-success url" target="_pod" href="/go.php?domain=' . $pod['domain'] . '">' . $pod['domain'] . '</a></td>';
} }
if ($pod['shortversion'] > $pod['masterversion']) { if ($pod['shortversion'] > $pod['masterversion']) {
...@@ -77,7 +79,7 @@ foreach ($pods as $pod) { ...@@ -77,7 +79,7 @@ foreach ($pods as $pod) {
} }
echo '<td>' . $pod['city'] . '</td>'; echo '<td>' . $pod['city'] . '</td>';
echo '<td>' . $pod['state'] . '</td>'; echo '<td>' . $pod['state'] . '</td>';
echo '<td>' . ($pod['detectedlanguage'] ? strtoupper($pod['detectedlanguage']) : '') . '</td>'; echo '<td data-toggle="tooltip" title="' . ($pod['detectedlanguage'] ? $iso->languageByCode1($pod['detectedlanguage']) : '') . '">' . ($pod['detectedlanguage'] ? strtoupper($pod['detectedlanguage']) : '') . '</td>';
echo '<td class="text-truncate">'; echo '<td class="text-truncate">';
$pod['service_facebook'] && print '<div class="smlogo smlogo-facebook" data-toggle="tooltip" title="Publish to Facebook"></div>'; $pod['service_facebook'] && print '<div class="smlogo smlogo-facebook" data-toggle="tooltip" title="Publish to Facebook"></div>';
$pod['service_twitter'] && print '<div class="smlogo smlogo-twitter" data-toggle="tooltip" title="Publish to Twitter"></div>'; $pod['service_twitter'] && print '<div class="smlogo smlogo-twitter" data-toggle="tooltip" title="Publish to Twitter"></div>';
......
...@@ -114,7 +114,7 @@ EOF; ...@@ -114,7 +114,7 @@ EOF;
echo '<div class="row m-1 p-1"><div class="col-9">Countries we found pods in:</div><div class="col-8 p-3">'; echo '<div class="row m-1 p-1"><div class="col-9">Countries we found pods in:</div><div class="col-8 p-3">';
foreach ($countries as $country) { foreach ($countries as $country) {
printf( printf(
'<label><input class="ml-2" type="radio" name="country" value="%1$s" /> %2$s %3$s</label><br>', '<label class="m-0"><input class="ml-2" type="radio" name="country" value="%1$s" /> %2$s %3$s</label><br>',
$country, $country,
country($country)->getName(), country($country)->getName(),
country($country)->getEmoji() country($country)->getEmoji()
...@@ -132,7 +132,7 @@ EOF; ...@@ -132,7 +132,7 @@ EOF;
foreach ($languages as $language) { foreach ($languages as $language) {
printf( printf(
'<label><input class="ml-2" type="radio" name="language" value="%1$s" /> %2$s</label><br>', '<label><input class="ml-2" type="radio" name="language" value="%1$s" /> %2$s</label><br>',
$language, strtoupper($language),
$iso->languageByCode1($language) $iso->languageByCode1($language)
); );
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment