Unverified Commit ea73a847 authored by Armando Lüscher's avatar Armando Lüscher Committed by GitHub

Merge pull request #165 from diasporg/autolanguage

detect language from body text of main page
parents 16af008b 7ee8f61c
......@@ -2,10 +2,12 @@
## Podmins
* Can no longer access db/pull.php to test their pod, they can however get to a debug screen from the edit pod area
* Language is detected based on your homepage, edit your homepage to non-en if that is what you use
## DB
* Add development and release dates to masterversions table https://github.com/diasporg/Poduptime/issues/143
* Store full country name, store days monitored each pod
* Store detectedlanguage
* DB migrations see db/version.md
* rename table rating_comments to ratingcomments for redbean support
......
......@@ -4,7 +4,7 @@ Poduptime is software to get live stats and data on listed Diaspora Pods.
Dependencies:
```
php7.2 php7.2-curl php7.2-pgsql php-geoip php7.2-cli php7.2-common php7.2-json php7.2-readline php-cgi git curl postgresql postgresql-contrib wget dnsutils bind9 npm nodejs nodejs-legacy composer yarn
php7.2 php7.2-curl php7.2-pgsql php-geoip php7.2-cli php7.2-common php7.2-json php7.2-readline php7.2-mbstring php7.2-xml php-cgi git curl postgresql postgresql-contrib wget dnsutils bind9 npm nodejs nodejs-legacy composer yarn
```
To Install:
......
......@@ -5,13 +5,14 @@
}
},
"require": {
"noplanman/xec": "0.1.0",
"php" : "^7.2",
"gabordemooij/redbean": "^5.0",
"nesbot/carbon": "^1.31",
"commerceguys/enum": "^1.0",
"noplanman/xec": "0.1.0",
"jaybizzle/crawler-detect" :"1.*",
"twbs/bootstrap": "^4.1.0",
"php" : "^7.2",
"commerceguys/enum": "^1.0",
"nesbot/carbon": "^1.31"
"patrickschur/language-detection": "^3.3"
},
"autoload": {
"classmap": ["lib"]
......
{
"_readme": [
"This file locks the dependencies of your project to a known state",
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "d9076f2351a2d4457ed852c180ed1864",
"content-hash": "b4a8beac9ae1722bdbae0b3d81d7eb37",
"packages": [
{
"name": "commerceguys/enum",
......@@ -189,61 +189,6 @@
],
"time": "2018-06-25T13:15:16+00:00"
},
{
"name": "nesbot/carbon",
"version": "1.31.1",
"source": {
"type": "git",
"url": "https://github.com/briannesbitt/Carbon.git",
"reference": "385780c8ca2dbfd25452666e3f55e8dc1df58c41"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/briannesbitt/Carbon/zipball/385780c8ca2dbfd25452666e3f55e8dc1df58c41",
"reference": "385780c8ca2dbfd25452666e3f55e8dc1df58c41",
"shasum": ""
},
"require": {
"php": ">=5.3.9",
"symfony/translation": "~2.6 || ~3.0 || ~4.0"
},
"require-dev": {
"friendsofphp/php-cs-fixer": "~2",
"phpunit/phpunit": "^4.8.35 || ^5.7"
},
"type": "library",
"extra": {
"laravel": {
"providers": [
"Carbon\\Laravel\\ServiceProvider"
]
}
},
"autoload": {
"psr-4": {
"": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Brian Nesbitt",
"email": "brian@nesbot.com",
"homepage": "http://nesbot.com"
}
],
"description": "A simple API extension for DateTime.",
"homepage": "http://carbon.nesbot.com",
"keywords": [
"date",
"datetime",
"time"
],
"time": "2018-06-25T13:15:16+00:00"
},
{
"name": "noplanman/xec",
"version": "v0.1.0",
......@@ -297,26 +242,30 @@
"time": "2017-03-12T19:16:23+00:00"
},
{
"name": "twbs/bootstrap",
"version": "v4.1.0",
"name": "patrickschur/language-detection",
"version": "v3.3.0",
"source": {
"type": "git",
"url": "https://github.com/twbs/bootstrap.git",
"reference": "8f7bd419935adfcd53c471a0202083464800619e"
"url": "https://github.com/patrickschur/language-detection.git",
"reference": "21a2e7a1b9bf6bff578ac11c6dcf3d3668aeccdf"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/twbs/bootstrap/zipball/8f7bd419935adfcd53c471a0202083464800619e",
"reference": "8f7bd419935adfcd53c471a0202083464800619e",
"url": "https://api.github.com/repos/patrickschur/language-detection/zipball/21a2e7a1b9bf6bff578ac11c6dcf3d3668aeccdf",
"reference": "21a2e7a1b9bf6bff578ac11c6dcf3d3668aeccdf",
"shasum": ""
},
"replace": {
"twitter/bootstrap": "self.version"
"require": {
"ext-mbstring": "*",
"php": "^7"
},
"require-dev": {
"phpunit/phpunit": "^6"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "3.3.x-dev"
"autoload": {
"psr-4": {
"LanguageDetection\\": "src/LanguageDetection"
}
},
"notification-url": "https://packagist.org/downloads/",
......@@ -325,27 +274,18 @@
],
"authors": [
{
"name": "Jacob Thornton",
"email": "jacobthornton@gmail.com"
},
{
"name": "Mark Otto",
"email": "markdotto@gmail.com"
"name": "Patrick Schur",
"email": "patrick_schur@outlook.de"
}
],
"description": "The most popular front-end framework for developing responsive, mobile first projects on the web.",
"homepage": "https://getbootstrap.com/",
"description": "A language detection library for PHP. Detects the language from a given text string.",
"homepage": "https://github.com/patrickschur/language-detection",
"keywords": [
"JS",
"css",
"framework",
"front-end",
"mobile-first",
"responsive",
"sass",
"web"
"detect",
"detection",
"language"
],
"time": "2018-04-09T16:00:23+00:00"
"time": "2018-02-01T17:12:47+00:00"
},
{
"name": "symfony/polyfill-mbstring",
......@@ -474,6 +414,57 @@
"description": "Symfony Translation Component",
"homepage": "https://symfony.com",
"time": "2018-06-22T08:59:39+00:00"
},
{
"name": "twbs/bootstrap",
"version": "v4.1.1",
"source": {
"type": "git",
"url": "https://github.com/twbs/bootstrap.git",
"reference": "779ad9f174ea5ab7e755f6df0ec9e5912d67dd16"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/twbs/bootstrap/zipball/779ad9f174ea5ab7e755f6df0ec9e5912d67dd16",
"reference": "779ad9f174ea5ab7e755f6df0ec9e5912d67dd16",
"shasum": ""
},
"replace": {
"twitter/bootstrap": "self.version"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "3.3.x-dev"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Jacob Thornton",
"email": "jacobthornton@gmail.com"
},
{
"name": "Mark Otto",
"email": "markdotto@gmail.com"
}
],
"description": "The most popular front-end framework for developing responsive, mobile first projects on the web.",
"homepage": "https://getbootstrap.com/",
"keywords": [
"JS",
"css",
"framework",
"front-end",
"mobile-first",
"responsive",
"sass",
"web"
],
"time": "2018-04-30T16:04:33+00:00"
}
],
"packages-dev": [],
......@@ -482,7 +473,9 @@
"stability-flags": [],
"prefer-stable": false,
"prefer-lowest": false,
"platform": [],
"platform": {
"php": "^7.2"
},
"platform-dev": [],
"platform-overrides": {
"php": "7.2"
......
......@@ -2,3 +2,4 @@ ALTER TABLE masterversions ADD devlastcommit timestamp;
ALTER TABLE masterversions ADD releasedate timestamp;
ALTER TABLE pods ADD daysmonitored int, ADD countryname text;
ALTER TABLE rating_comments RENAME TO ratingcomments;
ALTER TABLE pods ADD detectedlanguage text;
......@@ -7,6 +7,7 @@ if ($_SERVER['SERVER_ADDR'] !== $_SERVER['REMOTE_ADDR']) {
}
use RedBeanPHP\R;
use LanguageDetection\Language;
$debug = isset($_GET['debug']) || (isset($argv) && in_array('debug', $argv, true));
$sqldebug = isset($_GET['sqldebug']) || (isset($argv) && in_array('sqldebug', $argv, true));
......@@ -91,6 +92,16 @@ foreach ($pods as $pod) {
$admin_rating = -1;
}
$d = new DOMDocument;
libxml_use_internal_errors(true);
$d->loadHTMLFile('https://' . $domain);
$body = $d->getElementsByTagName('body')->item(0);
if ($body->nodeValue) {
$ld = new Language;
$detectedlanguage = strtoupper(key($ld->detect($body->nodeValue)->bestResults()->close()));
_debug('Detected Language', $detectedlanguage);
}
if ($infos = json_decode(file_get_contents('https://' . $domain . '/.well-known/nodeinfo'), true)) {
$link = max($infos['links'])['href'];
} else {
......@@ -337,6 +348,7 @@ foreach ($pods as $pod) {
$p['state'] = $state;
$p['lat'] = $lat;
$p['long'] = $long;
$p['detectedlanguage'] = $detectedlanguage;
$p['userrating'] = $user_rating;
$p['masterversion'] = $masterversion;
$p['weightedscore'] = $weightedscore;
......
......@@ -12,6 +12,7 @@ CREATE TABLE pods (
userrating decimal DEFAULT 0,
hidden boolean DEFAULT true,
ip text,
detectedlanguage text,
country text,
countryname text,
city text,
......
......@@ -7,7 +7,7 @@ defined('PODUPTIME') || die();
try {
$pods = R::getAll('
SELECT domain, dnssec, podmin_statement, sslexpire, masterversion, shortversion, softwarename, daysmonitored, monthsmonitored, score, signup, name, country, countryname, city, state, lat, long, uptime_alltime, active_users_halfyear, active_users_monthly, service_facebook, service_twitter, service_tumblr, service_wordpress, service_xmpp, latency, date_updated, ipv6, total_users, local_posts, comment_counts, userrating, status
SELECT domain, dnssec, podmin_statement, sslexpire, masterversion, shortversion, softwarename, daysmonitored, monthsmonitored, score, signup, name, country, countryname, city, state, lat, long, detectedlanguage, uptime_alltime, active_users_halfyear, active_users_monthly, service_facebook, service_twitter, service_tumblr, service_wordpress, service_xmpp, latency, date_updated, ipv6, total_users, local_posts, comment_counts, userrating, status
FROM pods
WHERE status < ?
ORDER BY weightedscore DESC
......@@ -40,6 +40,7 @@ try {
<th><a data-toggle="tooltip" data-placement="bottom" title="System Score on a 100 point scale.">Score</a></th>
<th><a data-toggle="tooltip" data-placement="bottom" title="Does this domain use DNSSEC.">DNSSEC</a></th>
<th <?php echo ($country_code ? 'data-placeholder="Try: $country_code"' : 'data-placeholder="Try: US"') ?>><a data-toggle="tooltip" data-placement="bottom" title="Pod location, based on IP Geolocation.">Country</a></th>
<th><a data-toggle="tooltip" data-placement="bottom" title="Pod language detected from their main page text.">Language</a></th>
<th class="filter-false"><a data-toggle="tooltip" data-placement="bottom" title="External Social Networks this pod can post to.">Services</a></th>
<th><a data-toggle="tooltip" data-placement="bottom" title="Click for more information about this pod from the pod host (podmin).">Info</a></th>
</tr>
......@@ -92,6 +93,7 @@ try {
} else {
echo '<td data-toggle="tooltip" data-placement="bottom" title="Country: ' . ($pod['countryname'] ?? 'n/a') . ' City: ' . ($pod['city'] ?? 'n/a') . ' State: ' . ($pod['state'] ?? 'n/a') . '">' . $pod['country'] . '</td>';
}
echo '<td>' . ($pod['detectedlanguage'] ? $pod['detectedlanguage'] : '') . '</td>';
echo '<td>';
$pod['service_facebook'] && print '<div class="smlogo smlogo-facebook" title="Publish to Facebook"></div>';
$pod['service_twitter'] && print '<div class="smlogo smlogo-twitter" title="Publish to Twitter"></div>';
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment