Commit 88b1997e authored by David Morley's avatar David Morley Committed by GitHub
Browse files

Fixpullnbackup (#79)

* fixes for issue 63

* example

* add install note

* quotes

* Fixpullnbackup fixes (#81)

* Fix a few warnings and correct data type for boolean variables.

* Boolean data type for $dnssec, $signup and $ipv6.
Use only $signup instead of also $registrations_open.
Use only $outputssl instead of also $outputresults.

* Weird PSQL requires boolean to be passed as int, else it gives an error: `invalid input syntax for type boolean: ""`

* Remove unnecessary unsets.

* Remove unnecessary foreach loop.

* Fix alignment (visual only).
[skip ci]

* To determine if the request was successful, check for a valid JSON to parse instead of searching the response for a string.

* Rename query result variables to prevent breaking the main loop.

* Add a small debug function to make debug output cleaner and easier to use.

* Add debug flag. Browser: `pull.php?debug`, CLI: `pull.php debug`

* Remove old code that prevents markers from overlapping each other, as mapbox nicely separates them.
parent b5537788
...@@ -42,6 +42,7 @@ psql -u podupuser podupdb < db/tables.sql ...@@ -42,6 +42,7 @@ psql -u podupuser podupdb < db/tables.sql
Edit `config.php` to add your DB and file settings. Edit `config.php` to add your DB and file settings.
touch add.log in location you configured in config.php touch add.log in location you configured in config.php
run db/pull.sh manually or with cron to update your data
============================ ============================
......
<?php <?php
//$debug = isset($_GET['debug']);
$debug = true;
//$debug = isset($argv[1])?1:0;
//* Copyright (c) 2011, David Morley. This file is licensed under the Affero General Public License version 3 or later. See the COPYRIGHT file. */ //* Copyright (c) 2011, David Morley. This file is licensed under the Affero General Public License version 3 or later. See the COPYRIGHT file. */
$debug = isset($_GET['debug']) || (isset($argv) && in_array('debug', $argv, true));
$newline = PHP_SAPI === 'cli' ? "\n" : '<br>';
// Other parameters. // Other parameters.
$_domain = $_GET['domain'] ?? ''; $_domain = $_GET['domain'] ?? '';
...@@ -21,9 +21,7 @@ $outputmv = curl_exec($mv); ...@@ -21,9 +21,7 @@ $outputmv = curl_exec($mv);
curl_close($mv); curl_close($mv);
preg_match('/number: "(.*?)"/', $outputmv, $version); preg_match('/number: "(.*?)"/', $outputmv, $version);
$dmasterversion = trim($version[1], '"'); $dmasterversion = trim($version[1], '"');
if ($debug) { _debug('Diaspora Masterversion', $dmasterversion);
echo 'Diaspora Masterversion: ' . $dmasterversion . '<br>';
}
//get master code version for freindica pods //get master code version for freindica pods
$mv = curl_init(); $mv = curl_init();
...@@ -37,16 +35,14 @@ $outputmv = curl_exec($mv); ...@@ -37,16 +35,14 @@ $outputmv = curl_exec($mv);
curl_close($mv); curl_close($mv);
preg_match('/FRIENDICA_VERSION\', \'(.*?)\'/', $outputmv, $version); preg_match('/FRIENDICA_VERSION\', \'(.*?)\'/', $outputmv, $version);
$fmasterversion = trim($version[1], '"'); $fmasterversion = trim($version[1], '"');
if ($debug) { _debug('Friendica Masterversion: ' . $fmasterversion);
echo 'Frendica Masterversion: ' . $fmasterversion . '<br>';
}
$dbh = pg_connect("dbname=$pgdb user=$pguser password=$pgpass"); $dbh = pg_connect("dbname=$pgdb user=$pguser password=$pgpass");
$dbh || die('Error in connection: ' . pg_last_error()); $dbh || die('Error in connection: ' . pg_last_error());
//foreach pod check it and update db //foreach pod check it and update db
if ($_domain) { if ($_domain) {
$sql = 'SELECT domain,stats_apikey,score,date_created,weight FROM pods WHERE domain = $1'; $sql = 'SELECT domain,stats_apikey,score,date_created,adminrating,weight FROM pods WHERE domain = $1';
//$sleep = '0'; //$sleep = '0';
$result = pg_query_params($dbh, $sql, [$_domain]); $result = pg_query_params($dbh, $sql, [$_domain]);
} elseif (PHP_SAPI === 'cli') { } elseif (PHP_SAPI === 'cli') {
...@@ -58,21 +54,17 @@ if ($_domain) { ...@@ -58,21 +54,17 @@ if ($_domain) {
} }
$result || die('Error in SQL query1: ' . pg_last_error()); $result || die('Error in SQL query1: ' . pg_last_error());
while ($row = pg_fetch_all($result)) { while ($row = pg_fetch_assoc($result)) {
$numrows = pg_num_rows($result); $domain = $row['domain'];
for ($i = 0; $i < $numrows; $i++) { $score = (int) $row['score'];
$domain = $row[$i]['domain']; $dateadded = $row['date_created'];
$score = (int) $row[$i]['score']; $admindb = (int) $row['adminrating'];
$dateadded = $row[$i]['date_created']; $weight = $row['weight'];
$admindb = (int) $row[$i]['adminrating'];
$weight = $row[$i]['weight'];
$sqlforr = 'SELECT admin,rating FROM rating_comments WHERE domain = $1'; $sqlforr = 'SELECT admin,rating FROM rating_comments WHERE domain = $1';
$ratings = pg_query_params($dbh, $sqlforr, [$domain]); $ratings = pg_query_params($dbh, $sqlforr, [$domain]);
$ratings || die('Error in SQL query2: ' . pg_last_error()); $ratings || die('Error in SQL query2: ' . pg_last_error());
if ($debug) { _debug('Domain', $domain);
echo 'Domain: ' . $domain . '<br>';
}
$user_ratings = []; $user_ratings = [];
$admin_ratings = []; $admin_ratings = [];
...@@ -86,26 +78,9 @@ while ($row = pg_fetch_all($result)) { ...@@ -86,26 +78,9 @@ while ($row = pg_fetch_all($result)) {
$user_rating = empty($user_ratings) ? 0 : max(10, round(array_sum($user_ratings) / count($user_ratings), 2)); $user_rating = empty($user_ratings) ? 0 : max(10, round(array_sum($user_ratings) / count($user_ratings), 2));
$admin_rating = empty($admin_ratings) ? 0 : max(10, round(array_sum($admin_ratings) / count($admin_ratings), 2)); $admin_rating = empty($admin_ratings) ? 0 : max(10, round(array_sum($admin_ratings) / count($admin_ratings), 2));
if ($admindb == - 1) { if ($admindb == -1) {
$admin_rating = - 1; $admin_rating = -1;
} }
unset($name);
unset($total_users);
unset($active_users_halfyear);
unset($active_users_monthly);
unset($local_posts);
unset($registrations_open);
unset($comment_counts);
unset($service_facebook);
unset($service_twitter);
unset($service_tumblr);
unset($service_wordpess);
unset($service_xmpp);
unset($shortversion);
unset($dverr);
unset($xdver);
unset($softwarename);
unset($outputsslerror);
$chss = curl_init(); $chss = curl_init();
curl_setopt($chss, CURLOPT_URL, 'https://' . $domain . '/nodeinfo/1.0'); curl_setopt($chss, CURLOPT_URL, 'https://' . $domain . '/nodeinfo/1.0');
...@@ -123,47 +98,37 @@ while ($row = pg_fetch_all($result)) { ...@@ -123,47 +98,37 @@ while ($row = pg_fetch_all($result)) {
$sslexpire = $info[0]['Expire date']; $sslexpire = $info[0]['Expire date'];
curl_close($chss); curl_close($chss);
if ($debug) { _debug('Nodeinfo output', $outputssl, true);
var_dump($outputssl); _debug('Nodeinfo output error', $outputsslerror, true);
}
if (!$outpulssl && !$domain) { //get new json from nodeinfo
$jsonssl = json_decode($outputssl);
if (!$jsonssl && !$domain) {
continue; continue;
echo 'no connection to pod'; /*echo 'no connection to pod';
$sql = 'INSERT INTO checks (domain, online, error) VALUES ($1, $2, $3)'; $sql = 'INSERT INTO checks (domain, online, error) VALUES ($1, $2, $3)';
$result = pg_query_params($dbh, $sql, [$domain, false, $outputsslerror]); $result = pg_query_params($dbh, $sql, [$domain, false, $outputsslerror]);
$result || die('Error in SQL query: ' . pg_last_error()); $result || die('Error in SQL query: ' . pg_last_error());*/
} }
if ($outputssl) {
$secure = 'true';
$outputresults = $outputssl;
$sql = 'INSERT INTO checks (domain, online) VALUES ($1, $2)'; $sql_checks = 'INSERT INTO checks (domain, online) VALUES ($1, $2)';
$result = pg_query_params($dbh, $sql, [$domain, true]); $result_checks = pg_query_params($dbh, $sql_checks, [$domain, true]);
$result || die('Error in SQL query: ' . pg_last_error()); $result_checks || die('Error in SQL query: ' . pg_last_error());
} else { $signup = false;
$secure = 'false'; if ($jsonssl !== null) {
$outputresults = $output;
}
if (stristr($outputresults, 'openRegistrations')) {
$score += 1; $score += 1;
if ($debug) {
echo 'Secure: ' . $secure . '<br>';
}
//get new json from nodeinfo
$jsonssl = json_decode($outputresults);
var_dump($jsonssl); var_dump($jsonssl);
if ($jsonssl->openRegistrations === true) { if ($jsonssl->openRegistrations === true) {
$registrations_open = 1; $signup = true;
} }
$xdver = $jsonssl->software->version ?? 0; $xdver = $jsonssl->software->version ?? 0;
$dverr = explode('-', trim($xdver)); $dverr = explode('-', trim($xdver));
$shortversion = $dverr[0]; $shortversion = $dverr[0];
if ($debug) { _debug('Version code', $shortversion);
echo ' <br> Version code: ' . $shortversion . '<br>';
}
$shortversion || $score -= 2; $shortversion || $score -= 2;
$softwarename = $jsonssl->software->name ?? 'null'; $softwarename = $jsonssl->software->name ?? 'null';
...@@ -173,54 +138,41 @@ while ($row = pg_fetch_all($result)) { ...@@ -173,54 +138,41 @@ while ($row = pg_fetch_all($result)) {
$active_users_monthly = $jsonssl->usage->users->activeMonth ?? 0; $active_users_monthly = $jsonssl->usage->users->activeMonth ?? 0;
$local_posts = $jsonssl->usage->localPosts ?? 0; $local_posts = $jsonssl->usage->localPosts ?? 0;
$comment_counts = $jsonssl->usage->localComments ?? 0; $comment_counts = $jsonssl->usage->localComments ?? 0;
$service_facebook = in_array('facebook', $jsonssl->services->outbound, true) ? 'true' : 'false'; $service_facebook = in_array('facebook', $jsonssl->services->outbound, true);
$service_twitter = in_array('twitter', $jsonssl->services->outbound, true) ? 'true' : 'false'; $service_twitter = in_array('twitter', $jsonssl->services->outbound, true);
$service_tumblr = in_array('tumblr', $jsonssl->services->outbound, true) ? 'true' : 'false'; $service_tumblr = in_array('tumblr', $jsonssl->services->outbound, true);
$service_wordpress = in_array('wordpress', $jsonssl->services->outbound, true) ? 'true' : 'false'; $service_wordpress = in_array('wordpress', $jsonssl->services->outbound, true);
$service_xmpp = $jsonssl->metadata->xmppChat === true ? 'true' : 'false'; $service_xmpp = $jsonssl->metadata->xmppChat === true;
} else { } else {
$secure = 'false';
$score -= 1; $score -= 1;
$dver = '.connect error'; $dver = '.connect error';
$shortversion = 0; $shortversion = 0;
//could also be a ssl pod with a bad cert, I think its ok to call that a dead pod now //could also be a ssl pod with a bad cert, I think its ok to call that a dead pod now
} }
$signup = $registrations_open;
if ($debug) { _debug('Signup Open', $signup);
echo '<br>Signup Open: ' . $signup . '<br>';
}
$ip6 = exec(escapeshellcmd('dig +nocmd ' . $domain . ' aaaa +noall +short')); $ip6 = exec(escapeshellcmd('dig +nocmd ' . $domain . ' aaaa +noall +short'));
exec(escapeshellcmd('delv ' . $domain), $iplookup); exec(escapeshellcmd('delv ' . $domain), $iplookup);
$dnssec = in_array('; fully validated', $iplookup) ? 't' : 'f' ; $dnssec = in_array('; fully validated', $iplookup);
preg_match('/A(.*)/', $iplookup[1], $version); preg_match('/A(.*)/', $iplookup[1], $version);
$ip = trim($version[1]); $ip = trim($version[1]);
$ipv6 = strpos($ip6, ':') === false ? 'no' : 'yes'; $ipv6 = strpos($ip6, ':') !== false;
_debug('IP', $ip);
if ($debug) {
echo 'IP: ' . $ip . '<br>';
}
$location = geoip_record_by_name($ip); $location = geoip_record_by_name($ip);
if ($debug) { _debug('Location', $location, true);
echo ' Location: ';
var_dump($location);
echo '<br>';
}
if ($location) { if ($location) {
$country = !empty($location['country_code']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['country_code']) : null; $country = !empty($location['country_code']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['country_code']) : null;
$city = !empty($location['city']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['city']) : null; $city = !empty($location['city']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['city']) : null;
$state = !empty($location['region']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['region']) : null; $state = !empty($location['region']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['region']) : null;
$lat = !empty($location['latitude']) ? $location['latitude'] : null; $lat = !empty($location['latitude']) ? $location['latitude'] : null;
$long = !empty($location['longitude']) ? $location['longitude'] : null; $long = !empty($location['longitude']) ? $location['longitude'] : null;
// If lat and long are just a generic country with no detail lets make some tail up
// else openmap just stacks them all on top of each other.
strlen($lat) < 4 && $lat += (random_int(1, 15) / 10);
strlen($long) < 4 && $long += (random_int(1, 15) / 10);
} }
echo '<br>'; echo $newline;
$statslastdate = date('Y-m-d H:i:s'); $statslastdate = date('Y-m-d H:i:s');
$ping = curl_init(); $ping = curl_init();
curl_setopt($ping, CURLOPT_URL, 'https://api.uptimerobot.com/getMonitors?format=json&noJsonCallback=1&customUptimeRatio=7-30-60-90&responseTimes=1&responseTimesAverage=86400&apiKey=' . $row[$i]['stats_apikey']); curl_setopt($ping, CURLOPT_URL, 'https://api.uptimerobot.com/getMonitors?format=json&noJsonCallback=1&customUptimeRatio=7-30-60-90&responseTimes=1&responseTimesAverage=86400&apiKey=' . $row['stats_apikey']);
curl_setopt($ping, CURLOPT_POST, 0); curl_setopt($ping, CURLOPT_POST, 0);
curl_setopt($ping, CURLOPT_HEADER, 0); curl_setopt($ping, CURLOPT_HEADER, 0);
curl_setopt($ping, CURLOPT_RETURNTRANSFER, 1); curl_setopt($ping, CURLOPT_RETURNTRANSFER, 1);
...@@ -230,10 +182,7 @@ while ($row = pg_fetch_all($result)) { ...@@ -230,10 +182,7 @@ while ($row = pg_fetch_all($result)) {
curl_setopt($ping, CURLOPT_FOLLOWLOCATION, true); curl_setopt($ping, CURLOPT_FOLLOWLOCATION, true);
$uptr = json_decode(curl_exec($ping)); $uptr = json_decode(curl_exec($ping));
curl_close($ping); curl_close($ping);
if ($debug) { _debug('Uptime Robot', $uptr, true);
print_r($uptr);
echo '<br>';
}
$uptr || $score -= 2; $uptr || $score -= 2;
$responsetime = $uptr->monitors->monitor{'0'}->responsetime{'0'}->value; $responsetime = $uptr->monitors->monitor{'0'}->responsetime{'0'}->value;
...@@ -267,10 +216,8 @@ while ($row = pg_fetch_all($result)) { ...@@ -267,10 +216,8 @@ while ($row = pg_fetch_all($result)) {
} elseif ($softwarename === 'friendica') { } elseif ($softwarename === 'friendica') {
$masterversion = $fmasterversion; $masterversion = $fmasterversion;
} }
$hidden = $score > 70 ? 'no' : 'yes'; $hidden = $score <= 70;
if ($debug) { _debug('Hidden', $hidden ? 'yes' : 'no');
echo 'Hidden: ' . $hidden . '<br>';
}
// lets cap the scores or you can go too high or too low to never be effected by them // lets cap the scores or you can go too high or too low to never be effected by them
if ($score > 100) { if ($score > 100) {
$score = 100; $score = 100;
...@@ -281,18 +228,39 @@ while ($row = pg_fetch_all($result)) { ...@@ -281,18 +228,39 @@ while ($row = pg_fetch_all($result)) {
//sql it //sql it
$timenow = date('Y-m-d H:i:s'); $timenow = date('Y-m-d H:i:s');
$sql = 'UPDATE pods SET secure = $2, hidden = $3, ip = $4, ipv6 = $5, monthsmonitored = $6, uptime_alltime = $7, status = $8, date_laststats = $9, date_updated = $10, responsetime = $11, score = $12, adminrating = $13, country = $14, city = $15, state = $16, lat = $17, long = $18, userrating = $19, shortversion = $20, masterversion = $21, signup = $22, total_users = $23, active_users_halfyear = $24, active_users_monthly = $25, local_posts = $26, name = $27, comment_counts = $28, service_facebook = $29, service_tumblr = $30, service_twitter = $31, service_wordpress = $32, weightedscore = $33, service_xmpp = $34, softwarename = $35, sslvalid = $36, uptime_custom = $37, dnssec = $38, sslexpire = $39 $sql_set = 'UPDATE pods SET secure = $2, hidden = $3, ip = $4, ipv6 = $5, monthsmonitored = $6, uptime_alltime = $7, status = $8, date_laststats = $9, date_updated = $10, responsetime = $11, score = $12, adminrating = $13, country = $14, city = $15, state = $16, lat = $17, long = $18, userrating = $19, shortversion = $20, masterversion = $21, signup = $22, total_users = $23, active_users_halfyear = $24, active_users_monthly = $25, local_posts = $26, name = $27, comment_counts = $28, service_facebook = $29, service_tumblr = $30, service_twitter = $31, service_wordpress = $32, weightedscore = $33, service_xmpp = $34, softwarename = $35, sslvalid = $36, uptime_custom = $37, dnssec = $38, sslexpire = $39 WHERE domain = $1';
WHERE domain = $1'; $result_set = pg_query_params($dbh, $sql_set, [$domain, 1, (int) $hidden, $ip, (int) $ipv6, $months, $uptime, $status, $statslastdate, $timenow, $responsetime, $score, $admin_rating, $country, $city, $state, $lat, $long, $user_rating, $shortversion, $masterversion, (int) $signup, $total_users, $active_users_halfyear, $active_users_monthly, $local_posts, $name, $comment_counts, (int) $service_facebook, (int) $service_tumblr, (int) $service_twitter, (int) $service_wordpress, $weightedscore, (int) $service_xmpp, $softwarename, $outputsslerror, $uptime_custom, (int) $dnssec, $sslexpire]);
$result_set || die('Error in SQL query3: ' . pg_last_error());
$result = pg_query_params($dbh, $sql, [$domain, $secure, $hidden, $ip, $ipv6, $months, $uptime, $status, $statslastdate, $timenow, $responsetime, $score, $admin_rating, $country, $city, $state, $lat, $long, $user_rating, $shortversion, $masterversion, $signup, $total_users, $active_users_halfyear, $active_users_monthly, $local_posts, $name, $comment_counts, $service_facebook, $service_tumblr, $service_twitter, $service_wordpress, $weightedscore, $service_xmpp, $softwarename, $outputsslerror, $uptime_custom, $dnssec, $sslexpire]);
$result || die('Error in SQL query3: ' . pg_last_error()); _debug('Score out of 100', $score);
if ($debug) {
echo '<br>Score out of 100: ' . $score . '<br>';
} else {
echo 'Success'; echo 'Success';
echo $newline;
echo $newline;
}
/**
* Output a debug message and variable value
*
* @param string $label
* @param mixed $var
* @param bool $dump
*/
function _debug($label, $var = null, $dump = false) {
global $debug, $newline;
if (!$debug) {
return;
} }
//end foreach
if ($dump || is_array($var)) {
$output = print_r($var, true);
} elseif (is_bool($var)) {
$output = $var ? 'true' : 'false';
} else {
$output = (string) $var;
} }
printf('%s: %s%s', $label, $output, $newline);
} }
WGET="/usr/bin/wget" SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
if [ -s /tmp/index.google ];then if [ -s /tmp/index.google ];then
echo "already running die" echo "already running die"
exit; exit;
...@@ -6,8 +7,7 @@ else ...@@ -6,8 +7,7 @@ else
echo "Checking for internet"; echo "Checking for internet";
fi fi
$WGET -q --tries=10 --timeout=15 http://www.google.com -O /tmp/index.google wget -q --tries=2 --timeout=15 http://www.google.com -O /tmp/index.google
# &> /dev/null
sleep 2 sleep 2
if [ ! -s /tmp/index.google ];then if [ ! -s /tmp/index.google ];then
...@@ -16,8 +16,8 @@ if [ ! -s /tmp/index.google ];then ...@@ -16,8 +16,8 @@ if [ ! -s /tmp/index.google ];then
exit; exit;
else else
echo "Pulling in new pod data"; echo "Pulling in new pod data";
cd /var/www/poduptime/db cd "$SCRIPT_DIR"
php pull.php debug=1 php pull.php
touch last.data touch last.data
php backup.php php backup.php
rm /tmp/index.google rm /tmp/index.google
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment