Commit f267bd8f authored by Armando Lüscher's avatar Armando Lüscher

Merge branch 'targetandlog' into 'develop'

Target and Log and CA

See merge request !206
parents c0f90add c694a60b
......@@ -10,17 +10,27 @@ The format is based on [Keep a Changelog] and this project adheres to [Semantic
### Changed
- Introduce proper changelog format (#189)
- Moved DB migration scripts into `db` folder
- Use Curl for all http calls
- Use filter dropdowns for pre-defined columns
- Use pretty URLs (see nginx.example)
- Open pod URLs in a new tab
- Use detectlanguage.com API for language guess
### Deprecated
### Removed
### Fixed
- Notify podmins just once at 50 when pod failing (#186)
- Add missing meta and PHP module requirements to `composer.json`
### Security
## [2.3.1] - 2018-08-05
### Added
- Podmins can link directly to their pod via `https://podupti.me/domain.name` for stats and to allow users to rate easier
- Wizard to help you filter the columns to what you need (#145)
- Cookie used to remember last pod you clicked
### Changed
- Now one table with a basic default view you can customize (#171)
- Switch to a library for country to lat long lookup
- Switch GeoIP from built in PHP to library and use newer Maxmind database file
## [2.3.0] - 2018-07-19
:exclamation: DB migrations required! (see [SQL migration script][2.2.0-sql-migration])
......
......@@ -10,7 +10,6 @@ OS Dependencies:
```
php7.2 php7.2-curl php7.2-pgsql php-geoip php7.2-cli php7.2-common php7.2-bcmath php7.2-json php7.2-readline php7.2-mbstring php7.2-xml php-cgi git curl postgresql postgresql-contrib dnsutils bind9 npm nodejs composer
```
GeoIP needs setup normally with a dat file
Yarn is a separate install: https://yarnpkg.com
......@@ -54,13 +53,14 @@ run `db/pull.sh debug` to debug output
run `db/pull.sh sqldebug` to debug sql
run `db/pull.sh develop` to run without email alerts to end users
run `db/pull.sh Check_System_Deleted` to re-check system deleted pods as needed
These commands can be combined
# To Upgrade:
```
git pull
yarn install
composer install
psql -U podupuser podupdb < db/migrationXXX.sql (see db/version.md for proper migration versions)
psql -U podupuser podupdb < db/migrations/xxx.sql (see db/migrations/README.md for proper migration versions)
```
# Status
......
......@@ -6,6 +6,7 @@
declare(strict_types=1);
use DetectLanguage\DetectLanguage;
use RedBeanPHP\R;
/**
......@@ -39,6 +40,10 @@ define('PODUPTIME', microtime(true));
require_once __DIR__ . '/vendor/autoload.php';
// Initialise language detection.
DetectLanguage::setApiKey(c('dlkey'));
DetectLanguage::setSecure(true);
// Set up global DB connection.
R::setup(
sprintf(
......
{
"name": "diasporg/poduptime",
"description": "Poduptime is software to get live stats and data on listed Diaspora Pods.",
"license": "AGPL-3.0-or-later",
"config": {
"platform": {
"php": "7.2"
......@@ -6,16 +9,20 @@
},
"require": {
"php": "^7.2",
"gabordemooij/redbean": "^5.0",
"nesbot/carbon": "^1.31",
"ext-curl": "*",
"ext-dom": "*",
"ext-json": "*",
"ext-libxml": "*",
"gabordemooij/redbean": "^5.1",
"nesbot/carbon": "^1.33",
"commerceguys/enum": "^1.0",
"noplanman/xec": "0.1.0",
"jaybizzle/crawler-detect": "1.*",
"patrickschur/language-detection": "^3.3",
"jaybizzle/crawler-detect": "^1.2",
"geoip2/geoip2": "^2.9",
"maxmind-db/reader": "~1.0",
"maxmind-db/reader": "^1.3",
"matriphe/iso-639": "^1.2",
"rinvex/country": "^3.1"
"rinvex/country": "^3.1",
"detectlanguage/detectlanguage": "^2.2"
},
"require-dev": {
"squizlabs/php_codesniffer": "^3.3"
......
This diff is collapsed.
......@@ -43,4 +43,7 @@ return [
//Geolite2-city database file in mmdb format - full file path (pull.sh will update this monthly)
'geoip2db' => '',
//detectlanguage.com api key
'dlkey' => '',
];
......@@ -14,8 +14,9 @@ if (!in_array(PHP_SAPI, ['cgi-fcgi', 'cli'])) {
}
}
use Carbon\Carbon;
use DetectLanguage\DetectLanguage;
use GeoIp2\Database\Reader;
use LanguageDetection\Language;
use Poduptime\PodStatus;
use RedBeanPHP\R;
......@@ -39,7 +40,7 @@ try {
$reader = new Reader(c('geoip2db'));
$sql = '
SELECT domain, score, date_created, weight, podmin_notify, email, masterversion, shortversion, status
SELECT domain, score, date_created, weight, podmin_notify, email, masterversion, shortversion, status, detectedlanguage
FROM pods
';
......@@ -72,6 +73,7 @@ foreach ($pods as $pod) {
$masterv = $pod['masterversion'];
$shortv = $pod['shortversion'];
$dbstatus = $pod['status'];
$language = $pod['detectedlanguage'];
try {
$ratings = R::getAll('
......@@ -91,25 +93,24 @@ foreach ($pods as $pod) {
$user_rating = round(array_sum($user_ratings) / count($user_ratings), 2);
}
// Default link to fetch node info.
$link = "https://{$domain}/nodeinfo/1.0";
$nodeinfo_meta = _curl("https://{$domain}/.well-known/nodeinfo");
extract(_curl("https://{$domain}/.well-known/nodeinfo"));
$outputwellknown = $curl_body;
// Default link to fetch node info.
$nodeinfo_url = "https://{$domain}/nodeinfo/1.0";
if ($outputwellknown !== false) {
$info = json_decode($outputwellknown, true);
$link = max($info['links'])['href'];
if (!isset($nodeinfo_meta['error'])) {
$info = json_decode($nodeinfo_meta['body'], true);
$nodeinfo_url = max($info['links'])['href'];
}
_debug('Nodeinfo link', $link);
extract(_curl($link));
_debug('Nodeinfo link', $nodeinfo_url);
$nodeinfo = _curl($nodeinfo_url);
$outputssl = $curl_body;
$outputsslerror = $curl_error;
$info = $curl_info;
$conntime = $curl_conntime;
$nstime = $curl_nstime;
$outputssl = $nodeinfo['body'];
$outputsslerror = $nodeinfo['error'];
$info = $nodeinfo['info'];
$conntime = $nodeinfo['conntime'];
$nstime = $nodeinfo['nstime'];
$latency = $conntime - $nstime;
$sslexpire = $info[0]['Expire date'] ?? null;
......@@ -123,7 +124,7 @@ foreach ($pods as $pod) {
$jsonssl = ($outputssl ? json_decode($outputssl) : null);
if ($jsonssl !== null) {
$version = $jsonssl->software->version ?? 0;
$version = $jsonssl->software->version ?? 0;
preg_match_all('((?:\d(.|-)?)+(\.)\d+\.*)', $version, $sversion);
$shortversion = $sversion[0][0] ?? '0.0.0.0';
$signup = ($jsonssl->openRegistrations ?? false) === true;
......@@ -246,25 +247,20 @@ foreach ($pods as $pod) {
$status = PodStatus::UP;
}
$d = new DOMDocument;
libxml_use_internal_errors(true);
extract(_curl("https://{$domain}/"));
$outputbody = $curl_body;
($outputbody ? $d->loadHTML($outputbody) : $d->loadHTML('<html></html>'));
$body = $d->getElementsByTagName('html')->item(0);
if ($body) {
$ld = new Language;
$detectedlanguage = key($ld->detect($body->nodeValue)->bestResults()->close());
} else {
$score -= 1;
// Default to the already saved language.
$detectedlanguage = $language;
$language_snippet = getWebsiteLanguageSnippetFromUrl("https://{$domain}/");
if (!$language_snippet) {
$detectedlanguage = null;
--$score;
} elseif ($debug || Carbon::now()->hour === 12) {
$detectedlanguage = detectWebsiteLanguageFromSnippet($language_snippet);
}
_debug('Detected Language', $detectedlanguage);
if (!$jsonssl || !$body) {
if (!$jsonssl || !$language_snippet) {
_debug('Connection', 'Can not connect to pod');
try {
......@@ -282,7 +278,7 @@ foreach ($pods as $pod) {
die('Error in SQL query: ' . $e->getMessage());
}
$score -= 1;
--$score;
$status = PodStatus::DOWN;
}
......@@ -356,7 +352,7 @@ foreach ($pods as $pod) {
if ($score > 100) {
$score = 100;
} elseif ($score < 0) {
$score = 0;
$score = 0;
$status = PodStatus::SYSTEM_DELETED;
}
$weightedscore = ($uptime + $score - (10 - $weight)) / 2;
......@@ -447,23 +443,101 @@ function _debug($label, $var = null, $dump = false)
printf('%s: %s%s', $label, $output, $newline);
}
function _curl($url)
/**
* Execute cURL request and return array of data.
*
* @param string $url
*
* @return array
*/
function _curl(string $url): array
{
global $cafullpath;
$chss = curl_init();
curl_setopt($chss, CURLOPT_URL, $url);
curl_setopt($chss, CURLOPT_CONNECTTIMEOUT, 10);
curl_setopt($chss, CURLOPT_TIMEOUT, 20);
curl_setopt($chss, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($chss, CURLOPT_CERTINFO, 1);
curl_setopt($chss, CURLOPT_CAINFO, $cafullpath);
return [
'curl_body' => curl_exec($chss),
'curl_error' => curl_error($chss),
'curl_info' => curl_getinfo($chss, CURLINFO_CERTINFO),
'curl_conntime' => curl_getinfo($chss, CURLINFO_CONNECT_TIME),
'curl_nstime' => curl_getinfo($chss, CURLINFO_NAMELOOKUP_TIME)
curl_setopt($chss, CURLOPT_CAINFO, c('cafullpath'));
$data = [
'body' => curl_exec($chss),
'error' => curl_error($chss),
'info' => curl_getinfo($chss, CURLINFO_CERTINFO),
'conntime' => curl_getinfo($chss, CURLINFO_CONNECT_TIME),
'nstime' => curl_getinfo($chss, CURLINFO_NAMELOOKUP_TIME),
];
curl_close($chss);
return $data;
}
/**
* Get a language snippet from a given URL.
*
* @param string $url
*
* @return null|string
*/
function getWebsiteLanguageSnippetFromUrl(string $url): ?string
{
$curl = _curl($url);
if (!$curl['body']) {
return null;
}
libxml_use_internal_errors(true);
$d = new DOMDocument;
$d->loadHTML($curl['body']);
$snippet = $d->getElementsByTagName('title')->item(0)->textContent ?? '';
for ($type = 1; $type < 6; $type++) {
foreach ($d->getElementsByTagName('h' . $type) as $h) {
// Ignore possibly generic "JavaScript required" texts.
if (stripos($h->textContent, 'javascript') === false) {
$snippet .= ' ' . $h->textContent;
}
}
}
// Get descriptions of meta tags.
foreach ($d->getElementsByTagName('meta') as $meta) {
if (strtolower($meta->getAttribute('name')) === 'description') {
$snippet .= ' ' . $meta->getAttribute('value');
}
}
return $snippet;
}
/**
* Detect the language of the given text snippet.
*
* @param string $snippet
*
* @return null|string
*/
function detectWebsiteLanguageFromSnippet(string $snippet): ?string
{
if (!$snippet) {
return null;
}
return DetectLanguage::simpleDetect($snippet);
}
/**
* Detect the website language of the given URL.
*
* @param string $url
*
* @return null|string
*/
function detectWebsiteLanguageFromUrl(string $url): ?string
{
return detectWebsiteLanguageFromSnippet(
getWebsiteLanguageSnippetFromUrl($url)
);
}
......@@ -12,6 +12,8 @@ use RedBeanPHP\R;
require_once __DIR__ . '/boot.php';
$iso = new Matriphe\ISO639\ISO639;
try {
$pods = R::getAll('
SELECT domain, dnssec, podmin_statement, sslexpire, masterversion, shortversion, softwarename, daysmonitored, monthsmonitored, score, signup, name, country, countryname, city, state, detectedlanguage, uptime_alltime, active_users_halfyear, active_users_monthly, service_facebook, service_twitter, service_tumblr, service_wordpress, service_xmpp, latency, date_updated, ipv6, total_users, local_posts, comment_counts, userrating, status
......@@ -31,9 +33,9 @@ foreach ($pods as $pod) {
$tip = "Over the last {$humanmonitored} uptime was {$pod['uptime_alltime']}% and response time from Los Angeles was {$pod['latency']}ms.";
if (($_COOKIE['domain'] ?? null) === $pod['domain']) {
echo '<tr><td title="This is the last pod you visited from this site. ' . $tip . '" data-placement="right" data-toggle="tooltip" class="bg-secondary"><a class="text-warning url" target="_self" href="/go.php?domain=' . $pod['domain'] . '">' . $pod['domain'] . '</a></td>';
echo '<tr><td title="This is the last pod you visited from this site. ' . $tip . '" data-placement="right" data-toggle="tooltip" class="bg-secondary"><a class="text-warning url" target="_pod" href="/go.php?domain=' . $pod['domain'] . '">' . $pod['domain'] . '</a></td>';
} else {
echo '<tr><td data-placement="right" title="' . $tip . '" data-toggle="tooltip"><a class="text-success url" target="_self" href="/go.php?domain=' . $pod['domain'] . '">' . $pod['domain'] . '</a></td>';
echo '<tr><td data-placement="right" title="' . $tip . '" data-toggle="tooltip"><a class="text-success url" target="_pod" href="/go.php?domain=' . $pod['domain'] . '">' . $pod['domain'] . '</a></td>';
}
if ($pod['shortversion'] > $pod['masterversion']) {
......@@ -77,7 +79,7 @@ foreach ($pods as $pod) {
}
echo '<td>' . $pod['city'] . '</td>';
echo '<td>' . $pod['state'] . '</td>';
echo '<td>' . ($pod['detectedlanguage'] ? strtoupper($pod['detectedlanguage']) : '') . '</td>';
echo '<td data-toggle="tooltip" title="' . ($pod['detectedlanguage'] ? $iso->languageByCode1($pod['detectedlanguage']) : '') . '">' . ($pod['detectedlanguage'] ? strtoupper($pod['detectedlanguage']) : '') . '</td>';
echo '<td class="text-truncate">';
$pod['service_facebook'] && print '<div class="smlogo smlogo-facebook" data-toggle="tooltip" title="Publish to Facebook"></div>';
$pod['service_twitter'] && print '<div class="smlogo smlogo-twitter" data-toggle="tooltip" title="Publish to Twitter"></div>';
......
......@@ -114,7 +114,7 @@ EOF;
echo '<div class="row m-1 p-1"><div class="col-9">Countries we found pods in:</div><div class="col-8 p-3">';
foreach ($countries as $country) {
printf(
'<label><input class="ml-2" type="radio" name="country" value="%1$s" /> %2$s %3$s</label><br>',
'<label class="m-0"><input class="ml-2" type="radio" name="country" value="%1$s" /> %2$s %3$s</label><br>',
$country,
country($country)->getName(),
country($country)->getEmoji()
......@@ -132,7 +132,7 @@ EOF;
foreach ($languages as $language) {
printf(
'<label><input class="ml-2" type="radio" name="language" value="%1$s" /> %2$s</label><br>',
$language,
strtoupper($language),
$iso->languageByCode1($language)
);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment