Commit 9b7f6936 authored by dmorley's avatar dmorley
Browse files

Merge branch 'develop' into 'master'

Dev to Master for release

See merge request !242
parents a18b81d4 d41ea376
Pipeline #1818 passed with stage
in 2 minutes and 39 seconds
......@@ -9,6 +9,17 @@ The format is based on [Keep a Changelog] and this project adheres to [Semantic
### Fixed
### Security
## [2.6.1] - 2019-06-30
### Added
### Changed
- only check pods once a month found in the wild via crawl
- curl function timeout can be changed from 15 second default
### Deprecated
### Removed
### Fixed
- Error out if detectlanguage.com issue vs die out on script
### Security
## [2.6.0] - 2019-04-13
:exclamation: DB migrations required! (see [SQL migration script][2.6.0-sql-migration])
### Added
......@@ -186,6 +197,7 @@ The format is based on [Keep a Changelog] and this project adheres to [Semantic
[2.2.0-sql-migration]: https://git.feneas.org/diasporg/Poduptime/blob/master/db/migrations/2.1.4-2.2.0.sql
[Unreleased]: https://git.feneas.org/diasporg/Poduptime/compare/master...develop
[2.6.1]: https://git.feneas.org/diasporg/Poduptime/compare/2.6.0...2.6.1
[2.6.0]: https://git.feneas.org/diasporg/Poduptime/compare/2.5.2...2.6.0
[2.5.2]: https://git.feneas.org/diasporg/Poduptime/compare/2.5.1...2.5.2
[2.5.1]: https://git.feneas.org/diasporg/Poduptime/compare/2.5.0...2.5.1
......
......@@ -149,16 +149,16 @@
},
{
"name": "gabordemooij/redbean",
"version": "v5.2",
"version": "v5.3",
"source": {
"type": "git",
"url": "https://github.com/gabordemooij/redbean.git",
"reference": "4ec3ee400bfbe4c52da4a376de9a6c338b5a8819"
"reference": "913c8072a13105d7d702ee88e762796773d212fe"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/gabordemooij/redbean/zipball/4ec3ee400bfbe4c52da4a376de9a6c338b5a8819",
"reference": "4ec3ee400bfbe4c52da4a376de9a6c338b5a8819",
"url": "https://api.github.com/repos/gabordemooij/redbean/zipball/913c8072a13105d7d702ee88e762796773d212fe",
"reference": "913c8072a13105d7d702ee88e762796773d212fe",
"shasum": ""
},
"require": {
......@@ -186,7 +186,7 @@
"keywords": [
"orm"
],
"time": "2018-10-26T09:06:03+00:00"
"time": "2019-03-15T09:34:00+00:00"
},
{
"name": "geoip2/geoip2",
......@@ -242,16 +242,16 @@
},
{
"name": "jaybizzle/crawler-detect",
"version": "v1.2.78",
"version": "v1.2.82",
"source": {
"type": "git",
"url": "https://github.com/JayBizzle/Crawler-Detect.git",
"reference": "0f91f34d375ddd5d7034b9eaf6b74acd0b2a0ed6"
"reference": "48522d4b3495ac8eef2a7298f128f35d7ea3f49b"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/0f91f34d375ddd5d7034b9eaf6b74acd0b2a0ed6",
"reference": "0f91f34d375ddd5d7034b9eaf6b74acd0b2a0ed6",
"url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/48522d4b3495ac8eef2a7298f128f35d7ea3f49b",
"reference": "48522d4b3495ac8eef2a7298f128f35d7ea3f49b",
"shasum": ""
},
"require": {
......@@ -287,7 +287,7 @@
"crawlerdetect",
"php crawler detect"
],
"time": "2019-01-15T21:21:27+00:00"
"time": "2019-05-21T20:51:49+00:00"
},
{
"name": "longman/ip-tools",
......@@ -493,16 +493,16 @@
},
{
"name": "nesbot/carbon",
"version": "1.36.2",
"version": "1.37.1",
"source": {
"type": "git",
"url": "https://github.com/briannesbitt/Carbon.git",
"reference": "cd324b98bc30290f233dd0e75e6ce49f7ab2a6c9"
"reference": "5be4fdf97076a685b23efdedfc2b73ad0c5eab70"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/briannesbitt/Carbon/zipball/cd324b98bc30290f233dd0e75e6ce49f7ab2a6c9",
"reference": "cd324b98bc30290f233dd0e75e6ce49f7ab2a6c9",
"url": "https://api.github.com/repos/briannesbitt/Carbon/zipball/5be4fdf97076a685b23efdedfc2b73ad0c5eab70",
"reference": "5be4fdf97076a685b23efdedfc2b73ad0c5eab70",
"shasum": ""
},
"require": {
......@@ -547,7 +547,7 @@
"datetime",
"time"
],
"time": "2018-12-28T10:07:33+00:00"
"time": "2019-04-19T10:27:42+00:00"
},
{
"name": "noplanman/xec",
......@@ -680,16 +680,16 @@
},
{
"name": "symfony/contracts",
"version": "v1.0.2",
"version": "v1.1.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/contracts.git",
"reference": "1aa7ab2429c3d594dd70689604b5cf7421254cdf"
"reference": "d3636025e8253c6144358ec0a62773cae588395b"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/contracts/zipball/1aa7ab2429c3d594dd70689604b5cf7421254cdf",
"reference": "1aa7ab2429c3d594dd70689604b5cf7421254cdf",
"url": "https://api.github.com/repos/symfony/contracts/zipball/d3636025e8253c6144358ec0a62773cae588395b",
"reference": "d3636025e8253c6144358ec0a62773cae588395b",
"shasum": ""
},
"require": {
......@@ -697,19 +697,22 @@
},
"require-dev": {
"psr/cache": "^1.0",
"psr/container": "^1.0"
"psr/container": "^1.0",
"symfony/polyfill-intl-idn": "^1.10"
},
"suggest": {
"psr/cache": "When using the Cache contracts",
"psr/container": "When using the Service contracts",
"symfony/cache-contracts-implementation": "",
"symfony/event-dispatcher-implementation": "",
"symfony/http-client-contracts-implementation": "",
"symfony/service-contracts-implementation": "",
"symfony/translation-contracts-implementation": ""
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "1.0-dev"
"dev-master": "1.1-dev"
}
},
"autoload": {
......@@ -744,20 +747,20 @@
"interoperability",
"standards"
],
"time": "2018-12-05T08:06:11+00:00"
"time": "2019-04-27T14:29:50+00:00"
},
{
"name": "symfony/polyfill-mbstring",
"version": "v1.10.0",
"version": "v1.11.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-mbstring.git",
"reference": "c79c051f5b3a46be09205c73b80b346e4153e494"
"reference": "fe5e94c604826c35a32fa832f35bd036b6799609"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/c79c051f5b3a46be09205c73b80b346e4153e494",
"reference": "c79c051f5b3a46be09205c73b80b346e4153e494",
"url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/fe5e94c604826c35a32fa832f35bd036b6799609",
"reference": "fe5e94c604826c35a32fa832f35bd036b6799609",
"shasum": ""
},
"require": {
......@@ -769,7 +772,7 @@
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "1.9-dev"
"dev-master": "1.11-dev"
}
},
"autoload": {
......@@ -803,20 +806,20 @@
"portable",
"shim"
],
"time": "2018-09-21T13:07:52+00:00"
"time": "2019-02-06T07:57:58+00:00"
},
{
"name": "symfony/translation",
"version": "v4.2.3",
"version": "v4.2.8",
"source": {
"type": "git",
"url": "https://github.com/symfony/translation.git",
"reference": "23fd7aac70d99a17a8e6473a41fec8fab3331050"
"reference": "181a426dd129cb496f12d7e7555f6d0b37a7615b"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/translation/zipball/23fd7aac70d99a17a8e6473a41fec8fab3331050",
"reference": "23fd7aac70d99a17a8e6473a41fec8fab3331050",
"url": "https://api.github.com/repos/symfony/translation/zipball/181a426dd129cb496f12d7e7555f6d0b37a7615b",
"reference": "181a426dd129cb496f12d7e7555f6d0b37a7615b",
"shasum": ""
},
"require": {
......@@ -838,7 +841,9 @@
"symfony/console": "~3.4|~4.0",
"symfony/dependency-injection": "~3.4|~4.0",
"symfony/finder": "~2.8|~3.0|~4.0",
"symfony/http-kernel": "~3.4|~4.0",
"symfony/intl": "~3.4|~4.0",
"symfony/var-dumper": "~3.4|~4.0",
"symfony/yaml": "~3.4|~4.0"
},
"suggest": {
......@@ -876,22 +881,22 @@
],
"description": "Symfony Translation Component",
"homepage": "https://symfony.com",
"time": "2019-01-27T23:11:39+00:00"
"time": "2019-05-01T12:55:36+00:00"
}
],
"packages-dev": [
{
"name": "squizlabs/php_codesniffer",
"version": "3.4.0",
"version": "3.4.2",
"source": {
"type": "git",
"url": "https://github.com/squizlabs/PHP_CodeSniffer.git",
"reference": "379deb987e26c7cd103a7b387aea178baec96e48"
"reference": "b8a7362af1cc1aadb5bd36c3defc4dda2cf5f0a8"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/379deb987e26c7cd103a7b387aea178baec96e48",
"reference": "379deb987e26c7cd103a7b387aea178baec96e48",
"url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/b8a7362af1cc1aadb5bd36c3defc4dda2cf5f0a8",
"reference": "b8a7362af1cc1aadb5bd36c3defc4dda2cf5f0a8",
"shasum": ""
},
"require": {
......@@ -924,12 +929,12 @@
}
],
"description": "PHP_CodeSniffer tokenizes PHP, JavaScript and CSS files and detects violations of a defined set of coding standards.",
"homepage": "http://www.squizlabs.com/php-codesniffer",
"homepage": "https://github.com/squizlabs/PHP_CodeSniffer",
"keywords": [
"phpcs",
"standards"
],
"time": "2018-12-19T23:57:18+00:00"
"time": "2019-04-10T23:49:02+00:00"
}
],
"aliases": [],
......
......@@ -90,6 +90,8 @@ if (!$stop) {
if ($info = json_decode($nodeinfo_meta['body'] ?: '', true)) {
if (count($info['links'], COUNT_RECURSIVE) === 2) {
$nodeinfo_url = $info['links']['href'];
} elseif (count($info['links'], COUNT_RECURSIVE) === 4) {
$nodeinfo_url = $info['links'][1]['href'];
} else {
$nodeinfo_url = max($info['links'])['href'];
}
......@@ -133,5 +135,6 @@ if (!$stop) {
echo 'Data successfully inserted! Your pod will be checked and live on the list in a few hours!';
} else {
echo 'Could not validate your pod, check your setup!<br>Take a look at your <a href="' . $nodeinfo_url . '">nodeinfo</a>';
updateMeta('add_attempt', $_domain);
}
}
......@@ -28,25 +28,41 @@ try {
//get all existing pod domains
$existingpods = array_column($pods, 'domain');
//get pods we have not tried to auto add for a while
try {
$sql = '
SELECT value
FROM meta
WHERE date_created > now() - interval \'1 month\'
AND name = \'add_attempt\'
';
$metaresult = R::getAll($sql);
} catch (\RedBeanPHP\RedException $e) {
die('Error in SQL query: ' . $e->getMessage());
}
$triedpods = array_column($metaresult, 'value');
$skippods = array_merge($existingpods, $triedpods);
$foundpods = [];
//pulling all nodes for now
$federationpods = curl('https://the-federation.info/graphql?query=%7Bnodes%7Bhost%20platform%7Bname%7Dprotocols%7Bname%7D%7D%7D&raw');
$federationpods = curl('https://the-federation.info/graphql?query=%7Bnodes%7Bhost%20platform%7Bname%7Dprotocols%7Bname%7D%7D%7D&raw', false, 45);
if ($pods = json_decode($federationpods['body'] ?: '', true)) {
foreach ($pods['data']['nodes'] ?? [] as $poddata) {
$protocols = array_column($poddata['protocols'] ?? [], 'name');
$foundpods[] = strtolower(preg_replace('(^https?://)', '', $poddata['host']));
$foundpods[] = strtolower(preg_replace('/(https?:\/\/)?(:(\d*))?/', '', $poddata['host']));
}
}
$diasppods = curl('https://diasp.org/pods.json');
$diasppods = curl('https://diasp.org/pods.json', false, 45);
if ($pods = json_decode($diasppods['body'] ?: '', true)) {
foreach ($pods ?? [] as $poddata) {
$foundpods[] = strtolower($poddata['host']);
$foundpods[] = strtolower(preg_replace('/(https?:\/\/)?(:(\d*))?/', '', $poddata['host']));
}
}
$results = array_diff($foundpods, $existingpods);
$results = array_diff($foundpods, $skippods);
foreach ($results as $result) {
echo ' domain ' . $result . ' trying... ';
$add = __DIR__ . '/add.php';
......
......@@ -116,6 +116,8 @@ foreach ($pods as $pod) {
if ($info = json_decode($nodeinfo_meta['body'] ?: '', true)) {
if (count($info['links'], COUNT_RECURSIVE) === 2) {
$nodeinfo_url = $info['links']['href'];
} elseif (count($info['links'], COUNT_RECURSIVE) === 4) {
$nodeinfo_url = $info['links'][1]['href'];
} else {
$nodeinfo_url = max($info['links'])['href'];
}
......@@ -279,10 +281,9 @@ foreach ($pods as $pod) {
} elseif ($debug || $langhours > 24) {
$detectedlanguage = detectWebsiteLanguageFromSnippet($language_snippet);
$languagesupdated = true;
debug('Detected Language', $detectedlanguage);
}
debug('Detected Language', $detectedlanguage);
if (!$jsonssl || !$language_snippet) {
debug('Connection', 'Can not connect to pod');
......
......@@ -105,12 +105,12 @@ function debug($label, $var = null, $dump = false): void
*
* @return array
*/
function curl(string $url, bool $follow = false): array
function curl(string $url, bool $follow = false, $timeout = 15): array
{
$chss = curl_init();
curl_setopt($chss, CURLOPT_URL, $url);
curl_setopt($chss, CURLOPT_CONNECTTIMEOUT, 5);
curl_setopt($chss, CURLOPT_TIMEOUT, 15);
curl_setopt($chss, CURLOPT_CONNECTTIMEOUT, $timeout);
curl_setopt($chss, CURLOPT_TIMEOUT, $timeout);
curl_setopt($chss, CURLOPT_RETURNTRANSFER, true);
curl_setopt($chss, CURLOPT_FAILONERROR, true);
curl_setopt($chss, CURLOPT_CERTINFO, true);
......@@ -183,8 +183,12 @@ function detectWebsiteLanguageFromSnippet(string $snippet): ?string
if (!$snippet) {
return null;
}
return DetectLanguage::simpleDetect($snippet);
try {
$language = DetectLanguage::simpleDetect($snippet);
} catch (\DetectLanguage\DetectLanguageError $e) {
debug('Error in DetectLanguage: ' . $e->getMessage());
}
return $language;
}
/**
......
......@@ -3,9 +3,9 @@
"@fortawesome/fontawesome-free@^5.2.0":
version "5.7.2"
resolved "https://registry.yarnpkg.com/@fortawesome/fontawesome-free/-/fontawesome-free-5.7.2.tgz#1498c3eb78ee7c78c5488418707de90aaf58d5d7"
integrity sha512-Ha4HshKdCVKgu4TVCtG8XyPPYdzTzNW4/fvPnn+LT7AosRABryhlRv4cc4+o84dgpvVJN9reN7jo/c+nYujFug==
version "5.8.2"
resolved "https://registry.yarnpkg.com/@fortawesome/fontawesome-free/-/fontawesome-free-5.8.2.tgz#fae4112c4fb47086f6ae167e32524b280000abd5"
integrity sha512-E4fDUF4fbu9AxKpaQQqCN3XBnNzb/5e0Gvd9OaQsYkK574LVI57v/EqqPfIm/mC7jYbxaPNrhvT5AF+Yzwyizg==
bootstrap@^4.1.3:
version "4.3.1"
......@@ -13,26 +13,26 @@ bootstrap@^4.1.3:
integrity sha512-rXqOmH1VilAt2DyPzluTi2blhk17bO7ef+zLLPlWvG494pDxcM234pJ8wTc/6R40UWizAIIMgxjvxZg5kmsbag==
chart.js@^2.7.2:
version "2.7.3"
resolved "https://registry.yarnpkg.com/chart.js/-/chart.js-2.7.3.tgz#cdb61618830bf216dc887e2f7b1b3c228b73c57e"
integrity sha512-3+7k/DbR92m6BsMUYP6M0dMsMVZpMnwkUyNSAbqolHKsbIzH2Q4LWVEHHYq7v0fmEV8whXE0DrjANulw9j2K5g==
version "2.8.0"
resolved "https://registry.yarnpkg.com/chart.js/-/chart.js-2.8.0.tgz#b703b10d0f4ec5079eaefdcd6ca32dc8f826e0e9"
integrity sha512-Di3wUL4BFvqI5FB5K26aQ+hvWh8wnP9A3DWGvXHVkO13D3DSnaSsdZx29cXlEsYKVkn1E2az+ZYFS4t0zi8x0w==
dependencies:
chartjs-color "^2.1.0"
moment "^2.10.2"
chartjs-color-string@^0.5.0:
version "0.5.0"
resolved "https://registry.yarnpkg.com/chartjs-color-string/-/chartjs-color-string-0.5.0.tgz#8d3752d8581d86687c35bfe2cb80ac5213ceb8c1"
integrity sha512-amWNvCOXlOUYxZVDSa0YOab5K/lmEhbFNKI55PWc4mlv28BDzA7zaoQTGxSBgJMHIW+hGX8YUrvw/FH4LyhwSQ==
chartjs-color-string@^0.6.0:
version "0.6.0"
resolved "https://registry.yarnpkg.com/chartjs-color-string/-/chartjs-color-string-0.6.0.tgz#1df096621c0e70720a64f4135ea171d051402f71"
integrity sha512-TIB5OKn1hPJvO7JcteW4WY/63v6KwEdt6udfnDE9iCAZgy+V4SrbSxoIbTw/xkUIapjEI4ExGtD0+6D3KyFd7A==
dependencies:
color-name "^1.0.0"
chartjs-color@^2.1.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/chartjs-color/-/chartjs-color-2.2.0.tgz#84a2fb755787ed85c39dd6dd8c7b1d88429baeae"
integrity sha1-hKL7dVeH7YXDndbdjHsdiEKbrq4=
version "2.3.0"
resolved "https://registry.yarnpkg.com/chartjs-color/-/chartjs-color-2.3.0.tgz#0e7e1e8dba37eae8415fd3db38bf572007dd958f"
integrity sha512-hEvVheqczsoHD+fZ+tfPUE+1+RbV6b+eksp2LwAhwRTVXEjCSEavvk+Hg3H6SZfGlPh/UfmWKGIvZbtobOEm3g==
dependencies:
chartjs-color-string "^0.5.0"
chartjs-color-string "^0.6.0"
color-convert "^0.5.3"
color-convert@^0.5.3:
......@@ -51,8 +51,8 @@ featherlight@^1.7.13:
integrity sha1-UA9TIEuntbtGUDcN2UrlllxVbpQ=
fork-awesome@ForkAwesome/Fork-Awesome:
version "1.1.5"
resolved "https://codeload.github.com/ForkAwesome/Fork-Awesome/tar.gz/5945975242464b243c80f5fdc32316b37f1c8dcb"
version "1.1.7"
resolved "https://codeload.github.com/ForkAwesome/Fork-Awesome/tar.gz/10fdbffbf1841759eb0f6b78229f92c7f599a8b3"
ion-rangeslider@^2.2.0:
version "2.3.0"
......@@ -62,9 +62,9 @@ ion-rangeslider@^2.2.0:
jquery ">=1.8"
jquery@>=1.2.6, jquery@>=1.8, jquery@^3.3.1:
version "3.3.1"
resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.3.1.tgz#958ce29e81c9790f31be7792df5d4d95fc57fbca"
integrity sha512-Ubldcmxp5np52/ENotGxlLe6aGMvmF4R8S6tZjsP6Knsaxd/xp3Zrh50cG93lR6nPXyUFwzN3ZSOQI0wRJNdGg==
version "3.4.1"
resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.4.1.tgz#714f1f8d9dde4bdfa55764ba37ef214630d80ef2"
integrity sha512-36+AdBzCL+y6qjw5Tx7HgzeGCzC81MDDgaUP8ld2zhx58HdqXGoBd+tHdrBMiyjGQs0Hxs/MLZTu/eHNJJuWPw==
leaflet.markercluster@^1.3.0:
version "1.4.1"
......@@ -72,9 +72,9 @@ leaflet.markercluster@^1.3.0:
integrity sha512-ZSEpE/EFApR0bJ1w/dUGwTSUvWlpalKqIzkaYdYB7jaftQA/Y2Jav+eT4CMtEYFj+ZK4mswP13Q2acnPBnhGOw==
leaflet@^1.3.3:
version "1.4.0"
resolved "https://registry.yarnpkg.com/leaflet/-/leaflet-1.4.0.tgz#d5f56eeb2aa32787c24011e8be4c77e362ae171b"
integrity sha512-x9j9tGY1+PDLN9pcWTx9/y6C5nezoTMB8BLK5jTakx+H7bPlnbCHfi9Hjg+Qt36sgDz/cb9lrSpNQXmk45Tvhw==
version "1.5.1"
resolved "https://registry.yarnpkg.com/leaflet/-/leaflet-1.5.1.tgz#9afb9d963d66c870066b1342e7a06f92840f46bf"
integrity sha512-ekM9KAeG99tYisNBg0IzEywAlp0hYI5XRipsqRXyRTeuU8jcuntilpp+eFf5gaE0xubc9RuSNIVtByEKwqFV0w==
moment@^2.10.2:
version "2.24.0"
......@@ -82,9 +82,9 @@ moment@^2.10.2:
integrity sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==
popper.js@^1.14.3:
version "1.14.7"
resolved "https://registry.yarnpkg.com/popper.js/-/popper.js-1.14.7.tgz#e31ec06cfac6a97a53280c3e55e4e0c860e7738e"
integrity sha512-4q1hNvoUre/8srWsH7hnoSJ5xVmIL4qgz+s4qf2TnJIMyZFUFMGH+9vE7mXynAlHSZ/NdTmmow86muD0myUkVQ==
version "1.15.0"
resolved "https://registry.yarnpkg.com/popper.js/-/popper.js-1.15.0.tgz#5560b99bbad7647e9faa475c6b8056621f5a4ff2"
integrity sha512-w010cY1oCUmI+9KwwlWki+r5jxKfTFDVoadl7MSrIujHU5MJ5OR6HTDj6Xo8aoR/QsA56x8jKjA59qGH4ELtrA==
tablesorter@^2.30.7:
version "2.31.1"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment