Unverified Commit 6da68df9 authored by dmorley's avatar dmorley Committed by GitHub

use API for pulls of master version - enforce some security (#159)

* use API for pulls of master version - enforce some security

* add migration

* fix podcrawler

* fix podcrawler

* add diasp.org json active pods
update composer to 7.2 php

* add diasp.org json active pods
update composer to 7.2 php

* remove wizard junk from this branch

* remove wizard junk from this branch

* Revamp masterversionpull fixes (#163)

* Use PHP_SAPI constant to save function call.

* Fix podcrawler to properly filter out only pods that support the diaspora protocol.

* Fix DB write logic and clean up pull.php code a bit.

* Use the best nodeinfo available

* Use the best nodeinfo available, need to default to 1.0

* same order each time vs random

* fix link

* little more universality

* freindica version edit, support as best as can

* using jasonrobinson.me as an example last updated at 2018-04-12 17:23:41.55 is now hidden, giving more time padding for dev branches.
parent 0dd92196
# 2x+
## Podmins
* Can no longer access db/pull.php to test their pod, they can however get to a debug screen from the edit pod area
## DB
* Add development and release dates to masterversions table https://github.com/diasporg/Poduptime/issues/143
## Cleanup
* Use the git API for release versions, check development releases on pods https://github.com/diasporg/Poduptime/issues/143
* Forbid access to files that should be cli only https://github.com/diasporg/Poduptime/issues/152
## End Users
* Show version and update in full view cleaner https://github.com/diasporg/Poduptime/issues/143
* Edit will send to email on file and be less delay, runner of site does not really have anyway to verify email address
# 2.2.0 # 2.2.0
## Podmins ## Podmins
......
...@@ -38,6 +38,7 @@ touch add.log in location you configured in config.php ...@@ -38,6 +38,7 @@ touch add.log in location you configured in config.php
run `db/pull.sh` manually or with cron to update your data run `db/pull.sh` manually or with cron to update your data
run `db/pull.sh debug` to debug output run `db/pull.sh debug` to debug output
run `db/pull.sh sqldebug` to debug sql
run `db/pull.sh develop` to run without email alerts to end users run `db/pull.sh develop` to run without email alerts to end users
run `db/pull.sh Check_System_Deleted` to re-check system deleted pods as needed run `db/pull.sh Check_System_Deleted` to re-check system deleted pods as needed
......
{ {
"config": {
"platform": {
"php": "7.2"
}
},
"require": { "require": {
"noplanman/xec": "0.1.0", "noplanman/xec": "0.1.0",
"gabordemooij/redbean": "^5.0", "gabordemooij/redbean": "^5.0",
"jaybizzle/crawler-detect" :"1.*", "jaybizzle/crawler-detect" :"1.*",
"commerceguys/enum": "^1.0" "commerceguys/enum": "^1.0",
"php" : "^7.2"
}, },
"autoload": { "autoload": {
"classmap": ["lib"] "classmap": ["lib"]
......
{ {
"_readme": [ "_readme": [
"This file locks the dependencies of your project to a known state", "This file locks the dependencies of your project to a known state",
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
"This file is @generated automatically" "This file is @generated automatically"
], ],
"content-hash": "b753491dc03e2084e9587d0bbafad98c", "content-hash": "e5874c9bc9ad133f4e558cf61972ae67",
"packages": [ "packages": [
{ {
"name": "commerceguys/enum", "name": "commerceguys/enum",
...@@ -87,23 +87,23 @@ ...@@ -87,23 +87,23 @@
}, },
{ {
"name": "jaybizzle/crawler-detect", "name": "jaybizzle/crawler-detect",
"version": "v1.2.62", "version": "v1.2.63",
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/JayBizzle/Crawler-Detect.git", "url": "https://github.com/JayBizzle/Crawler-Detect.git",
"reference": "f9767578e00f87a081835b49adc7c71074a5b46c" "reference": "3566bc69d0839ab2dfd739a660b00f25a6f02031"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/f9767578e00f87a081835b49adc7c71074a5b46c", "url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/3566bc69d0839ab2dfd739a660b00f25a6f02031",
"reference": "f9767578e00f87a081835b49adc7c71074a5b46c", "reference": "3566bc69d0839ab2dfd739a660b00f25a6f02031",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
"php": ">=5.3.0" "php": ">=5.3.0"
}, },
"require-dev": { "require-dev": {
"phpunit/phpunit": "4.8.*", "phpunit/phpunit": "^4.8|^5.5|^6.5",
"satooshi/php-coveralls": "1.*" "satooshi/php-coveralls": "1.*"
}, },
"type": "library", "type": "library",
...@@ -132,7 +132,7 @@ ...@@ -132,7 +132,7 @@
"crawlerdetect", "crawlerdetect",
"php crawler detect" "php crawler detect"
], ],
"time": "2018-03-27T18:25:43+00:00" "time": "2018-05-21T19:56:44+00:00"
}, },
{ {
"name": "noplanman/xec", "name": "noplanman/xec",
...@@ -194,5 +194,8 @@ ...@@ -194,5 +194,8 @@
"prefer-stable": false, "prefer-stable": false,
"prefer-lowest": false, "prefer-lowest": false,
"platform": [], "platform": [],
"platform-dev": [] "platform-dev": [],
"platform-overrides": {
"php": "7.2"
}
} }
...@@ -26,7 +26,6 @@ body { ...@@ -26,7 +26,6 @@ body {
.main { .main {
padding: 20px; padding: 20px;
} }
.placeholders { .placeholders {
margin-bottom: 30px; margin-bottom: 30px;
text-align: center; text-align: center;
...@@ -42,59 +41,63 @@ body { ...@@ -42,59 +41,63 @@ body {
display: inline-block; display: inline-block;
} }
.rating { .rating {
font-size:9px; font-size:9px;
} }
.tfont { .tfont {
font-size:12px; font-size:12px;
} }
.smlogo { .smlogo {
width: 16px; width: 16px;
height: 16px; height: 16px;
background: url('/images/smlogo.png') 0 0; background: url('/images/smlogo.png') 0 0;
display: inline-block; display: inline-block;
margin: 0 2px; margin: 0 2px;
} }
.smlogo-twitter { .smlogo-twitter {
background-position: 0 -776px; background-position: 0 -776px;
} }
.smlogo-facebook { .smlogo-facebook {
background-position: 0 -824px; background-position: 0 -824px;
} }
.smlogo-tumblr { .smlogo-tumblr {
background-position: 0 -792px; background-position: 0 -792px;
} }
.smlogo-wordpress { .smlogo-wordpress {
background-position: 0 -656px; background-position: 0 -656px;
} }
.smlogo-xmpp { .smlogo-xmpp {
background: none; background: none;
} }
.smlogo-xmpp img { .smlogo-xmpp img {
vertical-align: unset; vertical-align: unset;
}
#map {
height: 580px;
} }
#map { height: 580px; }
.mycluster { .mycluster {
width: 35px; width: 35px;
height: 35px; height: 35px;
background-color: blue; background-color: blue;
text-align: left; text-align: left;
font-size: 17px; font-size: 17px;
background: url('/bower_components/leaflet/dist/images/marker-icon-2x.png') repeat-y right bottom; background: url('/bower_components/leaflet/dist/images/marker-icon-2x.png') repeat-y right bottom;
background-size: 25px 37px; background-size: 25px 37px;
} }
.icon { .icon {
color: red; color: red;
} }
table { table {
empty-cells: show; empty-cells: show;
} }
#facebox { #facebox {
z-index: 1000 !important; z-index: 1000 !important;
} }
#facebox .content { #facebox .content {
width: 800px !important; width: 800px !important;
} }
.tablesorter-header { .tablesorter-header {
background-size: 10px 15px !important; background-size: 10px 15px !important;
}
.modal-xlg {
max-width: 92%;
} }
...@@ -64,7 +64,7 @@ foreach ($pods as $pod) { ...@@ -64,7 +64,7 @@ foreach ($pods as $pod) {
} }
echo <<<EOF echo <<<EOF
<form action="edit.php" method="get"> <form action="/?edit" method="get">
<input type="hidden" name="domain" value="{$_domain}"> <input type="hidden" name="domain" value="{$_domain}">
<input type="hidden" name="token" value="{$uuid}"> <input type="hidden" name="token" value="{$uuid}">
<label>Email <input type="text" size="20" name="email"></label><br> <label>Email <input type="text" size="20" name="email"></label><br>
...@@ -82,8 +82,14 @@ EOF; ...@@ -82,8 +82,14 @@ EOF;
} }
} }
if ($infos = json_decode(file_get_contents('https://' . $_domain . '/.well-known/nodeinfo'), true)) {
$link = max($infos['links'])['href'];
} else {
$link= 'https://' . $_domain . '/.well-known/nodeinfo';
}
$chss = curl_init(); $chss = curl_init();
curl_setopt($chss, CURLOPT_URL, 'https://' . $_domain . '/nodeinfo/1.0'); curl_setopt($chss, CURLOPT_URL, $link);
curl_setopt($chss, CURLOPT_POST, 0); curl_setopt($chss, CURLOPT_POST, 0);
curl_setopt($chss, CURLOPT_HEADER, 0); curl_setopt($chss, CURLOPT_HEADER, 0);
curl_setopt($chss, CURLOPT_CONNECTTIMEOUT, 5); curl_setopt($chss, CURLOPT_CONNECTTIMEOUT, 5);
...@@ -118,19 +124,17 @@ if (stristr($outputssl, 'openRegistrations')) { ...@@ -118,19 +124,17 @@ if (stristr($outputssl, 'openRegistrations')) {
$message_lines = [ $message_lines = [
'https://' . $_SERVER['HTTP_HOST'], 'https://' . $_SERVER['HTTP_HOST'],
'Pod: https://' . $_SERVER['HTTP_HOST'] . '/db/pull.php?debug=1&domain=' . $_domain, 'Your pod ' . $_domain . ' will not show up right away, as it needs to pass a few checks first.',
'',
'Your pod will not show up right away, as it needs to pass a few checks first.',
'Give it a few hours!', 'Give it a few hours!',
]; ];
@mail($to, $subject, implode("\r\n", $message_lines), implode("\r\n", $headers)); @mail($to, $subject, implode("\r\n", $message_lines), implode("\r\n", $headers));
} }
echo 'Data successfully inserted! Your pod will be reviewed and live on the list in a few hours!'; echo 'Data successfully inserted! Your pod will be checked and live on the list in a few hours!';
} else { } else {
$log->lwrite('Could not validate your pod, check your setup! ' . $_domain); $log->lwrite('Could not validate your pod, check your setup! ' . $_domain);
echo 'Could not validate your pod, check your setup!<br>Take a look at <a href="https://' . $_domain . '/nodeinfo/1.0">your /nodeinfo</a>'; echo 'Could not validate your pod, check your setup!<br>Take a look at <a href="' . $link . '">your /nodeinfo</a>';
} }
$log->lclose(); $log->lclose();
<?php <?php
if (PHP_SAPI !== 'cli') {
header('HTTP/1.0 403 Forbidden');
exit;
}
require_once __DIR__ . '/../config.php'; require_once __DIR__ . '/../config.php';
$keep = (60 * 60 * 6) * 1; $keep = (60 * 60 * 6) * 1;
......
...@@ -120,4 +120,24 @@ if ('save' === $_action) { ...@@ -120,4 +120,24 @@ if ('save' === $_action) {
<input type="hidden" name="token" value="<?php echo $_token; ?>"> <input type="hidden" name="token" value="<?php echo $_token; ?>">
<input type="submit" name="action" value="unpause"> <input type="submit" name="action" value="unpause">
</form> </form>
<button type="button" class="openBtn" value="<?php echo $_domain; ?>">Do a debug test pull of your pod</button>
<!-- Modal -->
<div class="modal fade" id="podpull" role="dialog">
<div class="modal-dialog modal-xlg">
<div class="modal-content">
<div class="modal-header">
<button type="button" class="close" data-dismiss="modal">&times;</button>
<h4 class="modal-title">Pod debug data</h4>
</div>
<div class="modal-body">
Loading.....
</div>
<div class="modal-footer">
<button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
</div>
</div>
</div>
</div>
<?php <?php
...@@ -38,16 +38,16 @@ if ($_email) { ...@@ -38,16 +38,16 @@ if ($_email) {
$subject = 'Temporary edit key for ' . $_SERVER['HTTP_HOST']; $subject = 'Temporary edit key for ' . $_SERVER['HTTP_HOST'];
$headers[] = 'Bcc: ' . $adminemail; $headers[] = 'Bcc: ' . $adminemail;
$expire = time() + 2700; $expire = time() + 2700;
$output = 'Link sent to your email'; $output = 'Link sent to your email.';
} elseif (!$pod['email']) { } elseif (!$pod['email']) {
die('domain is registered but no email associated, to add an email use the add a pod feature'); die('Domain is registered but no email associated, to add an email use the add a pod feature.');
} else { } else {
$to = $adminemail; $to = $pod['email'];
$subject = 'FORWARD REQUEST: Temporary edit key for ' . $_SERVER['HTTP_HOST']; $subject = 'Temporary edit key for ' . $_SERVER['HTTP_HOST'];
$message_lines[] = 'User trying to edit pod without email address.'; $message_lines[] = 'Looks like you did not enter your email address, be sure to update it if you forgot the one we have for you.';
$message_lines[] = 'Email found: ' . $pod['email']; $message_lines[] = 'Email found: ' . $pod['email'];
$expire = time() + 9700; $expire = time() + 2700;
$output = 'Link sent to administrator to review and verify, if approved they will forward the edit key to you.'; $output = 'Link sent to email we have for this pod on file.';
} }
try { try {
......
ALTER TABLE masterversions ADD devlastcommit timestamp;
ALTER TABLE masterversions ADD releasedate timestamp;
<?php <?php
if (PHP_SAPI !== 'cli') {
header('HTTP/1.0 403 Forbidden');
exit;
}
use RedBeanPHP\R; use RedBeanPHP\R;
require_once __DIR__ . '/../vendor/autoload.php'; require_once __DIR__ . '/../vendor/autoload.php';
......
<?php <?php
if (php_sapi_name() == "cli") { if (PHP_SAPI !== 'cli') {
$json = json_decode(file_get_contents('https://the-federation.info/pods.json')); header('HTTP/1.0 403 Forbidden');
if ($json) { exit;
foreach ($json->pods ?? [] as $poddata) { }
echo exec("php-cgi add.php domain={$poddata->host}") . "\r\n";
use RedBeanPHP\R;
require_once __DIR__ . '/../vendor/autoload.php';
require_once __DIR__ . '/../config.php';
define('PODUPTIME', microtime(true));
// Set up global DB connection.
R::setup("pgsql:host={$pghost};dbname={$pgdb}", $pguser, $pgpass, true);
R::testConnection() || die('Error in DB connection');
R::usePartialBeans(true);
try {
$sql = '
SELECT domain, status
FROM pods
';
$pods = R::getAll($sql);
} catch (\RedBeanPHP\RedException $e) {
die('Error in SQL query: ' . $e->getMessage());
}
//get all existing pod domains
$existingpods = array_column($pods, 'domain');
$foundpods = [];
//pulling all nodes for now
if ($pods = json_decode(file_get_contents('https://the-federation.info/graphql?query=%7Bnodes%7Bhost%20platform%7Bname%7Dprotocols%7Bname%7D%7D%7D&raw'), true)) {
foreach ($pods['data']['nodes'] ?? [] as $poddata) {
$protocols = array_column($poddata['protocols'] ?? [], 'name');
//limiting to diaspora compatible for now
if (in_array('diaspora', $protocols, true)) {
$foundpods[] = $poddata['host'];
} }
} }
} else { }
header('HTTP/1.0 403 Forbidden');
if ($pods = json_decode(file_get_contents('https://diasp.org/pods.json'), true)) {
foreach ($pods ?? [] as $poddata) {
$foundpods[] = $poddata['host'];
}
}
$results = array_diff($foundpods, $existingpods);
foreach ($results as $result) {
echo exec("php-cgi add.php domain={$result}") . "\r\n";
} }
<?php <?php
//* Copyright (c) 2017, David Morley. This file is licensed under the Affero General Public License version 3 or later. See the COPYRIGHT file. */ //* Copyright (c) 2017, David Morley. This file is licensed under the Affero General Public License version 3 or later. See the COPYRIGHT file. */
if (PHP_SAPI !== 'cli') {
header('HTTP/1.0 403 Forbidden');
exit;
}
use RedBeanPHP\R; use RedBeanPHP\R;
require_once __DIR__ . '/../vendor/autoload.php'; require_once __DIR__ . '/../vendor/autoload.php';
...@@ -14,32 +19,70 @@ R::testConnection() || die('Error in DB connection'); ...@@ -14,32 +19,70 @@ R::testConnection() || die('Error in DB connection');
R::usePartialBeans(true); R::usePartialBeans(true);
$softwares = [ $softwares = [
'diaspora' => ['url' => 'https://raw.githubusercontent.com/diaspora/diaspora/master/config/defaults.yml', 'regex' => '/number:.*"(.*)"/'], 'diaspora' => ['repo' => 'diaspora/diaspora', 'gitsite' => 'api.github.com', 'gittype' => 'github', 'devbranch' => 'develop'],
'friendica' => ['url' => 'https://raw.githubusercontent.com/friendica/friendica/master/boot.php', 'regex' => '/define.*\'FRIENDICA_VERSION\'.*\'(.*)\'/'], 'friendica' => ['repo' => 'friendica/friendica', 'gitsite' => 'api.github.com', 'gittype' => 'github', 'devbranch' => 'develop'],
'redmatrix' => ['url' => 'https://raw.githubusercontent.com/redmatrix/hubzilla/master/boot.php', 'regex' => '/define.*\'STD_VERSION\'.*\'(.*)\'/'], 'hubzilla' => ['repo' => 'hubzilla%2fcore', 'gitsite' => 'framagit.org', 'gittype' => 'gitlab', 'devbranch' => 'dev'],
'socialhome' => ['url' => 'https://raw.githubusercontent.com/jaywink/socialhome/master/socialhome/__init__.py', 'regex' => '/__version__ =.*"(.*)"/'], 'pleroma' => ['repo' => 'pleroma%2fpleroma', 'gitsite' => 'git.pleroma.social', 'gittype' => 'gitlab', 'devbranch' => 'develop'],
'social-relay' => ['url' => 'https://raw.githubusercontent.com/jaywink/social-relay/master/social_relay/config.py', 'regex' => '/VERSION.*"(.*)"/'], 'socialhome' => ['repo' => 'jaywink/socialhome', 'gitsite' => 'api.github.com', 'gittype' => 'github', 'devbranch' => ''],
'ganggo' => ['url' => 'https://raw.githubusercontent.com/ganggo/ganggo/master/package.json', 'regex' => '/"version": "(.*)"/'], 'social-relay' => ['repo' => 'jaywink/social-relay', 'gitsite' => 'api.github.com', 'gittype' => 'github', 'devbranch' => ''],
'ganggo' => ['repo' => 'ganggo/ganggo', 'gitsite' => 'api.github.com', 'gittype' => 'github', 'devbranch' => ''],
];
$opts = [
'http' => ['method' => 'GET', 'header' => ['User-Agent: Poduptime']]
]; ];
foreach ($softwares as $software => $details) { foreach ($softwares as $software => $details) {
$mv = curl_init(); if ($details['gittype'] == 'github') {
curl_setopt($mv, CURLOPT_URL, $details['url']); $context = stream_context_create($opts);
curl_setopt($mv, CURLOPT_CONNECTTIMEOUT, 5); $releasejson = json_decode(file_get_contents('https://' . $details["gitsite"] . '/repos/' . $details["repo"] . '/releases/latest', false, $context));
curl_setopt($mv, CURLOPT_RETURNTRANSFER, 1); if ($details["devbranch"]) {
$outputmv = curl_exec($mv); $commitjson = json_decode(file_get_contents('https://' . $details["gitsite"] . '/repos/' . $details["repo"] . '/commits/' . $details["devbranch"], false, $context));
curl_close($mv); } else {
$commitjson = '';
if ($masterversion = preg_match($details['regex'], $outputmv, $version) ? $version[1] : '') { }
try { if ($masterversion = $releasejson->tag_name ? str_replace('v', '', $releasejson->tag_name) : '') {
$m = R::dispense('masterversions'); try {
$m['software'] = $software; $m = R::dispense('masterversions');
$m['version'] = $masterversion; $m['software'] = $software;
R::store($m); $m['version'] = $masterversion;
} catch (\RedBeanPHP\RedException $e) { if ($releasedate = $releasejson ? $releasejson->published_at : '') {
die('Error in SQL query: ' . $e->getMessage()); $m['releasedate'] = $releasedate;
}
if ($devlastcommit = $commitjson ? $commitjson->commit->author->date : '') {
$m['devlastcommit'] = $devlastcommit;
}
R::store($m);
} catch (\RedBeanPHP\RedException $e) {
die('Error in SQL query: ' . $e->getMessage());
}
}
} elseif ($details['gittype'] == 'gitlab') {
$context = stream_context_create($opts);
$releasejson = json_decode(file_get_contents('https://' . $details["gitsite"] . '/api/v4/projects/' . $details["repo"] . '/repository/tags', false, $context));
if ($details["devbranch"]) {
$commitjson = json_decode(file_get_contents('https://' . $details["gitsite"] . '/api/v4/projects/' . $details["repo"] . '/repository/commits/' . $details["devbranch"], false, $context));
} else {
$commitjson = '';
}
if ($masterversion = $releasejson[0]->name ? str_replace('v', '', $releasejson[0]->name) : '') {
try {
$m = R::dispense('masterversions');
$m['software'] = $software;
$m['version'] = $masterversion;
if ($releasedate = $releasejson[0] ? $releasejson[0]->commit->created_at : '') {
$m['releasedate'] = $releasedate;
}
if ($devlastcommit = $commitjson ? $commitjson->created_at : '') {
$m['devlastcommit'] = $devlastcommit;
}
R::store($m);
} catch (\RedBeanPHP\RedException $e) {
die('Error in SQL query: ' . $e->getMessage());
}
}
} }
}
printf('%s:%s ', $software, $masterversion ?: 'n/a');
printf('%s:%s:%s ', $software, $masterversion, $devlastcommit ?: 'n/a');
} }
<?php <?php
//* Copyright (c) 2011, David Morley. This file is licensed under the Affero General Public License version 3 or later. See the COPYRIGHT file. */ //* Copyright (c) 2011, David Morley. This file is licensed under the Affero General Public License version 3 or later. See the COPYRIGHT file. */
if ($_SERVER['SERVER_ADDR'] !== $_SERVER['REMOTE_ADDR']) {
header('HTTP/1.0 403 Forbidden');
exit;
}
use RedBeanPHP\R; use RedBeanPHP\R;
$debug = isset($_GET['debug']) || (isset($argv) && in_array('debug', $argv, true)); $debug = isset($_GET['debug']) || (isset($argv) && in_array('debug', $argv, true));
$newline = PHP_SAPI === 'cli' ? "\n" : '<br>'; $sqldebug = isset($_GET['sqldebug']) || (isset($argv) && in_array('sqldebug', $argv, true));
$write = !(isset($_GET['nowrite']) || (isset($argv) && in_array('nowrite', $argv, true)));
$newline = PHP_SAPI === 'cli' ? "\n\n" : '<br><br>';
$_domain = $_GET['domain'] ?? null; $_domain = $_GET['domain'] ?? null;
...@@ -18,6 +25,7 @@ define('PODUPTIME', microtime(true)); ...@@ -18,6 +25,7 @@ define('PODUPTIME', microtime(true));
// Set up global DB connection. // Set up global DB connection.
R::setup("pgsql:host={$pghost};dbname={$pgdb}", $pguser, $pgpass, true);