Skip to content

GitLab

Commit 8ae5a7c3 authored by David Morley's avatar David Morley Committed by GitHub
Browse files

Merge pull request #100 from diasporg/pullfix 

make this more logical and work for pods
parents ca94de08 d5440771
Hide whitespace changes
Inline Side-by-side
Showing with 76 additions and 68 deletions
cleanup.php
View file @ 8ae5a7c3
......@@ -4,7 +4,7 @@ require_once __DIR__ . '/config.php';
$dbh = pg_connect("dbname=$pgdb user=$pguser password=$pgpass");
$dbh || die('Error in connection: ' . pg_last_error());
$sql = "SELECT domain,masterversion,shortversion,softwarename,monthsmonitored,score,signup,secure,name,country,city,state,lat,long,uptime_alltime,active_users_halfyear,active_users_monthly,service_facebook,service_twitter,service_tumblr,service_wordpress,service_xmpp,responsetime,date_updated,ipv6,total_users,local_posts,comment_counts,stats_apikey,userrating,sslvalid FROM pods WHERE score < 50 ORDER BY weightedscore";
$sql = "SELECT domain,masterversion,shortversion,softwarename,monthsmonitored,score,signup,secure,name,country,city,state,lat,long,uptime_alltime,active_users_halfyear,active_users_monthly,service_facebook,service_twitter,service_tumblr,service_wordpress,service_xmpp,latency,date_updated,ipv6,total_users,local_posts,comment_counts,stats_apikey,userrating,sslvalid FROM pods WHERE score < 50 ORDER BY weightedscore";
$result = pg_query($dbh, $sql);
$result || die('Error in SQL query: ' . pg_last_error());
$numrows = pg_num_rows($result);
......@@ -44,7 +44,7 @@ $numrows = pg_num_rows($result);
'This pod %1$s has been watched for %2$s months and its average ping time is %3$s with uptime of %4$s%% this month and was last checked on %5$s. On a score of -20 to +20 this pod is a %6$s right now',
$pod_name,
$row['monthsmonitored'],
$row['responsetime'],
$row['latency'],
$row['uptime_alltime'],
$row['date_updated'],
$row['score']
......@@ -71,7 +71,7 @@ $numrows = pg_num_rows($result);
}
echo '<td class="' . $classver . '"><div title="' . $pre . ' codename: ' . $row['shortversion'] . ' master version is: ' . $row['masterversion'] . '" class="tipsy">' . $version . '</div></td>';
echo '<td>' . $row['uptime_alltime'] . '</td>';
echo '<td>' . $row['responsetime'] . '</td>';
echo '<td>' . $row['latency'] . '</td>';
echo '<td>' . ($row['signup'] === 't' ? 'Open' : 'Closed') . '</td>';
echo '<td>' . $row['total_users'] . '</td>';
echo '<td>' . $row['active_users_halfyear'] . '</td>';
......
config.php.example
View file @ 8ae5a7c3
......@@ -18,3 +18,5 @@ $adminemail = '';
$adminkey = '';
//DNS server for dnssec testing
$dnsserver = '';
//CA for curl to use - suggest wget http://curl.haxx.se/ca/cacert.pem as they pull from mozilla if you use a system CA then certs like wosign and startssl will be valid but users will be unable to connect to them
$cafullpath = '';
db/migration00001.sql
View file @ 8ae5a7c3
......@@ -4,7 +4,7 @@ ALTER TABLE pods DROP Hgitdate, DROP Hgitref, DROP Hruntime, DROP Hencoding, DRO
ALTER TABLE pods RENAME COLUMN pingdomurl TO stats_apikey;
ALTER TABLE pods RENAME COLUMN xmpp TO service_xmpp;
ALTER TABLE pods RENAME COLUMN uptimelast7 TO uptime_alltime;
ALTER TABLE pods RENAME COLUMN responsetimelast7 TO responsetime;
ALTER TABLE pods RENAME COLUMN responsetimelast7 TO latency;
ALTER TABLE pods RENAME COLUMN dateUpdated TO date_updated;
ALTER TABLE pods RENAME COLUMN dateLaststats TO date_laststats;
......@@ -17,6 +17,7 @@ ALTER TABLE pods ALTER hidden DROP DEFAULT;
ALTER TABLE pods ALTER COLUMN hidden TYPE boolean USING hidden::boolean;
ALTER TABLE pods ALTER COLUMN secure TYPE boolean USING secure::boolean;
ALTER TABLE pods ALTER COLUMN signup TYPE boolean USING signup::boolean;
ALTER TABLE pods ALTER COLUMN latency TYPE smallint USING latency::int;
ALTER TABLE pods ALTER weight SET DEFAULT 10;
ALTER TABLE pods ALTER score SET DEFAULT 50;
......@@ -44,7 +45,7 @@ CREATE TABLE checks (
domain text,
online boolean,
error text,
ttl numeric(8,6),
latency numeric(8,6),
total_users int,
local_posts int,
comment_counts int,
......
db/pods_apiv1.sql
View file @ 8ae5a7c3
......@@ -5,7 +5,7 @@ CREATE TABLE pods_apiv1 AS SELECT * FROM pods;
ALTER TABLE pods_apiv1 RENAME COLUMN stats_apikey TO pingdomurl;
ALTER TABLE pods_apiv1 RENAME COLUMN service_xmpp TO xmpp;
ALTER TABLE pods_apiv1 RENAME COLUMN uptime_alltime TO uptimelast7;
ALTER TABLE pods_apiv1 RENAME COLUMN responsetime TO responsetimelast7;
ALTER TABLE pods_apiv1 RENAME COLUMN latency TO responsetimelast7;
ALTER TABLE pods_apiv1 RENAME COLUMN date_updated TO dateUpdated;
ALTER TABLE pods_apiv1 RENAME COLUMN date_laststats TO dateLaststats;
......@@ -18,6 +18,7 @@ ALTER TABLE pods_apiv1 ALTER hidden DROP DEFAULT;
ALTER TABLE pods_apiv1 ALTER COLUMN hidden TYPE text USING hidden::text;
ALTER TABLE pods_apiv1 ALTER COLUMN secure TYPE text USING secure::text;
ALTER TABLE pods_apiv1 ALTER COLUMN signup TYPE text USING signup::text;
ALTER TABLE pods_apiv1 ALTER COLUMN responsetimelast7 TYPE text USING responsetimelast7::smallint;
ALTER TABLE pods_apiv1 ADD Hgitdate text, ADD Hgitref text, ADD Hruntime text, ADD Hencoding text, ADD longversion text, ADD ptr text, ADD whois text, ADD postalcode text, ADD connection text, ADD pingdomlast text;
......
db/pull.php
View file @ 8ae5a7c3
......@@ -11,7 +11,6 @@ require_once __DIR__ . '/../config.php';
$dbh = pg_connect("dbname=$pgdb user=$pguser password=$pgpass");
$dbh || die('Error in connection: ' . pg_last_error());
//foreach pod check it and update db
if ($_domain) {
$sql = 'SELECT domain,score,date_created,adminrating,weight,hidden,podmin_notify,email FROM pods WHERE domain = $1';
$result = pg_query_params($dbh, $sql, [$_domain]);
......@@ -60,76 +59,77 @@ while ($row = pg_fetch_assoc($result)) {
curl_setopt($chss, CURLOPT_TIMEOUT, 9);
curl_setopt($chss, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($chss, CURLOPT_CERTINFO, 1);
curl_setopt($chss, CURLOPT_CAINFO, $cafullpath);
$outputssl = curl_exec($chss);
$outputsslerror = curl_error($chss);
$info = curl_getinfo($chss, CURLINFO_CERTINFO);
$ttl = curl_getinfo($chss, CURLINFO_CONNECT_TIME);
$latency = curl_getinfo($chss, CURLINFO_CONNECT_TIME);
$sslexpire = $info[0]['Expire date'] ?? null;
curl_close($chss);
_debug('Nodeinfo output', $outputssl, true);
_debug('Nodeinfo output error', $outputsslerror, true);
_debug('Cert expire date', $sslexpire);
_debug('TTL', $ttl);
_debug('Latency', $latency);
$jsonssl = json_decode($outputssl);
if (!$jsonssl) {
_debug('Connection', 'Can not connect to pod');
$sql_errors = 'INSERT INTO checks (domain, online, error, ttl) VALUES ($1, $2, $3, $4)';
$result_errors = pg_query_params($dbh, $sql_errors, [$domain, 0, $outputsslerror, $ttl]);
$result_errors || die('Error in SQL query: ' . pg_last_error());
continue;
}
if ($jsonssl !== null) {
$xdver = $jsonssl->software->version ?? 0;
$dverr = explode('-', trim($xdver));
$shortversion = $dverr[0];
$signup = ($jsonssl->openRegistrations ?? false) === true;
$softwarename = $jsonssl->software->name ?? 'unknown';
$name = $jsonssl->metadata->nodeName ?? 'null';
$total_users = $jsonssl->usage->users->total ?? 0;
$active_users_halfyear = $jsonssl->usage->users->activeHalfyear ?? 0;
$active_users_monthly = $jsonssl->usage->users->activeMonth ?? 0;
$local_posts = $jsonssl->usage->localPosts ?? 0;
$comment_counts = $jsonssl->usage->localComments ?? 0;
$service_xmpp = ($jsonssl->metadata->xmppChat ?? false) === true;
$service_facebook = false;
$service_twitter = false;
$service_tumblr = false;
$service_wordpress = false;
if (json_last_error() === 0) {
(!$jsonssl->software->version) || $score += 1;
$xdver = $jsonssl->software->version ?? 0;
$dverr = explode('-', trim($xdver));
$shortversion = $dverr[0];
_debug('Version code', $shortversion);
$signup = ($jsonssl->openRegistrations === true);
$softwarename = $jsonssl->software->name ?? 'null';
$name = $jsonssl->metadata->nodeName ?? 'null';
$total_users = $jsonssl->usage->users->total ?? 0;
$active_users_halfyear = $jsonssl->usage->users->activeHalfyear ?? 0;
$active_users_monthly = $jsonssl->usage->users->activeMonth ?? 0;
$local_posts = $jsonssl->usage->localPosts ?? 0;
$comment_counts = $jsonssl->usage->localComments ?? 0;
$service_facebook = in_array('facebook', $jsonssl->services->outbound, true);
$service_twitter = in_array('twitter', $jsonssl->services->outbound, true);
$service_tumblr = in_array('tumblr', $jsonssl->services->outbound, true);
$service_wordpress = in_array('wordpress', $jsonssl->services->outbound, true);
$service_xmpp = ($jsonssl->metadata->xmppChat ?? false) === true;
$status = 'Up';
}
$sql_checks = 'INSERT INTO checks (domain, online, ttl, total_users, local_posts, comment_counts, shortversion) VALUES ($1, $2, $3, $4, $5, $6, $7)';
$result_checks = pg_query_params($dbh, $sql_checks, [$domain, 1, $ttl, $total_users, $local_posts, $comment_counts, $shortversion]);
if ($jsonssl !== null) {
$status = 'Up';
$sql_checks = 'INSERT INTO checks (domain, online, latency, total_users, local_posts, comment_counts, shortversion) VALUES ($1, $2, $3, $4, $5, $6, $7)';
$result_checks = pg_query_params($dbh, $sql_checks, [$domain, 1, $latency, $total_users, $local_posts, $comment_counts, $shortversion]);
$result_checks || die('Error in SQL query: ' . pg_last_error());
} else {
$score -= 1;
$dver = '.connect error';
$shortversion = 0;
}
if (!$jsonssl) {
_debug('Connection', 'Can not connect to pod');
$sql_errors = 'INSERT INTO checks (domain, online, error, latency) VALUES ($1, $2, $3, $4)';
$result_errors = pg_query_params($dbh, $sql_errors, [$domain, 0, $outputsslerror, $latency]);
$result_errors || die('Error in SQL query: ' . pg_last_error());
$score -= 1;
$shortversion = '0.error';
$status = 'Down';
}
_debug('Version code', $shortversion);
_debug('Signup Open', $signup);
$iplookupv4 = [];
$ip = '';
$ip = '';
exec(escapeshellcmd('delv @' . $dnsserver . ' ' . $domain), $iplookupv4);
_debug('Iplookupv4', $iplookupv4, true);
$dnssec = in_array('; fully validated', $iplookupv4, true) ?? false;
$dnssec = in_array('; fully validated', $iplookupv4, true) ?? false;
$getaonly = array_values(preg_grep('/\s+IN\s+A\s+.*/', $iplookupv4));
if ($iplookupv4) {
if ($getaonly) {
preg_match('/A\s(.*)/', $getaonly[0], $aversion);
$ip = trim($aversion[1]) ?? '';
$ip = trim($aversion[1]) ?? '';
}
$iplookupv6 = [];
$ipv6 = null;
exec(escapeshellcmd('delv @' . $dnsserver . ' ' . $domain . ' AAAA '), $iplookupv6);
_debug('Iplookupv6', $iplookupv6, true);
$getaaaaonly = array_values(preg_grep('/\s+IN\s+AAAA\s+.*/', $iplookupv6));
if ($getaaaaonly) {
preg_match('/AAAA\s(.*)/', $getaaaaonly[0], $aaaaversion);
......@@ -138,27 +138,29 @@ while ($row = pg_fetch_assoc($result)) {
$ip || $score -= 2;
_debug('IPv4', $ip);
_debug('Iplookupv4', $iplookupv4, true);
_debug('IPv6', $ipv6);
_debug('Iplookupv6', $iplookupv6, true);
$location = geoip_record_by_name($ip);
_debug('Location', $location, true);
$country = !empty($location['country_code']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['country_code']) : null;
$city = !empty($location['city']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['city']) : null;
$state = !empty($location['region']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['region']) : null;
$lat = !empty($location['latitude']) ? $location['latitude'] : 0;
$long = !empty($location['longitude']) ? $location['longitude'] : 0;
$country = !empty($location['country_code']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['country_code']) : null;
$city = !empty($location['city']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['city']) : null;
$state = !empty($location['region']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['region']) : null;
$lat = !empty($location['latitude']) ? $location['latitude'] : 0;
$long = !empty($location['longitude']) ? $location['longitude'] : 0;
echo $newline;
$statslastdate = date('Y-m-d H:i:s');
$diff = (new DateTime())->diff(new DateTime($dateadded));
$months = $diff->m + ($diff->y * 12);
$diff = (new DateTime())->diff(new DateTime($dateadded));
$months = $diff->m + ($diff->y * 12);
$responsetime = 0;
$sqlttl = 'SELECT round(avg(ttl) * 1000) AS ttl FROM checks WHERE domain = $1';
$resultttl = pg_query_params($dbh, $sqlttl, [$domain]);
$resultttl || die('Error in SQL query resultchecks: ' . pg_last_error());
$responsetime = pg_fetch_result($resultttl, 0);
$avglatency = 0;
$sqllatency = 'SELECT round(avg(latency) * 1000) AS latency FROM checks WHERE domain = $1';
$resultlatency = pg_query_params($dbh, $sqllatency, [$domain]);
$resultlatency || die('Error in SQL query resultchecks: ' . pg_last_error());
$avglatency = pg_fetch_result($resultlatency, 0);
$uptime = 0;
$sqlonline = 'SELECT avg(online::int) * 100 AS online FROM checks WHERE domain = $1';
......@@ -194,13 +196,13 @@ while ($row = pg_fetch_assoc($result)) {
$weightedscore = ($uptime + $score + ($active_users_monthly / 19999) - ((10 - $weight) * .12));
$timenow = date('Y-m-d H:i:s');
$sql_set = 'UPDATE pods SET secure = $2, hidden = $3, ip = $4, ipv6 = $5, monthsmonitored = $6, uptime_alltime = $7, status = $8, date_laststats = $9, date_updated = $10, responsetime = $11, score = $12, adminrating = $13, country = $14, city = $15, state = $16, lat = $17, long = $18, userrating = $19, shortversion = $20, masterversion = $21, signup = $22, total_users = $23, active_users_halfyear = $24, active_users_monthly = $25, local_posts = $26, name = $27, comment_counts = $28, service_facebook = $29, service_tumblr = $30, service_twitter = $31, service_wordpress = $32, weightedscore = $33, service_xmpp = $34, softwarename = $35, sslvalid = $36, dnssec = $37, sslexpire = $38 WHERE domain = $1';
$result_set = pg_query_params($dbh, $sql_set, [$domain, 1, (int) $hidden, $ip, (int) ($ipv6 !== null), $months, $uptime, $status, $statslastdate, $timenow, $responsetime, $score, $admin_rating, $country, $city, $state, $lat, $long, $user_rating, $shortversion, $masterversion, (int) $signup, $total_users, $active_users_halfyear, $active_users_monthly, $local_posts, $name, $comment_counts, (int) $service_facebook, (int) $service_tumblr, (int) $service_twitter, (int) $service_wordpress, $weightedscore, (int) $service_xmpp, $softwarename, $outputsslerror, (int) $dnssec, $sslexpire]);
$sql_set = 'UPDATE pods SET secure = $2, hidden = $3, ip = $4, ipv6 = $5, monthsmonitored = $6, uptime_alltime = $7, status = $8, date_laststats = $9, date_updated = $10, latency = $11, score = $12, adminrating = $13, country = $14, city = $15, state = $16, lat = $17, long = $18, userrating = $19, shortversion = $20, masterversion = $21, signup = $22, total_users = $23, active_users_halfyear = $24, active_users_monthly = $25, local_posts = $26, name = $27, comment_counts = $28, service_facebook = $29, service_tumblr = $30, service_twitter = $31, service_wordpress = $32, weightedscore = $33, service_xmpp = $34, softwarename = $35, sslvalid = $36, dnssec = $37, sslexpire = $38 WHERE domain = $1';
$result_set = pg_query_params($dbh, $sql_set, [$domain, 1, (int) $hidden, $ip, (int) ($ipv6 !== null), $months, $uptime, $status, $statslastdate, $timenow, $avglatency, $score, $admin_rating, $country, $city, $state, $lat, $long, $user_rating, $shortversion, $masterversion, (int) $signup, $total_users, $active_users_halfyear, $active_users_monthly, $local_posts, $name, $comment_counts, (int) $service_facebook, (int) $service_tumblr, (int) $service_twitter, (int) $service_wordpress, $weightedscore, (int) $service_xmpp, $softwarename, $outputsslerror, (int) $dnssec, $sslexpire]);
$result_set || die('Error in SQL query3: ' . pg_last_error());
_debug('Score out of 100', $score);
echo 'Success';
echo 'Success '.$domain;
echo $newline;
echo $newline;
......
db/pull.sh
View file @ 8ae5a7c3
......@@ -20,6 +20,8 @@ else
if [ $TIME = 6 ];then
echo "Pulling in master versions";
php pull-masterversions.php
echo "Update CA"
wget https://curl.haxx.se/ca/cacert.pem -O ../cacert.pem
fi
echo "Pulling in new pod data";
php pull.php
......
db/tables.sql
View file @ 8ae5a7c3
......@@ -28,7 +28,7 @@ CREATE TABLE pods (
local_posts int,
uptime_alltime numeric(5,2),
status text,
responsetime text,
latency smallint,
service_facebook boolean,
service_twitter boolean,
service_tumblr boolean,
......@@ -76,7 +76,7 @@ CREATE TABLE checks (
domain text,
online boolean,
error text,
ttl numeric(8,6),
latency numeric(8,6),
total_users int,
local_posts int,
comment_counts int,
......
showfull.php
View file @ 8ae5a7c3
......@@ -8,7 +8,7 @@ $country_code = $_SERVER['HTTP_CF_IPCOUNTRY'] ?? '';
$dbh = pg_connect("dbname=$pgdb user=$pguser password=$pgpass");
$dbh || die('Error in connection: ' . pg_last_error());
$sql = 'SELECT domain,dnssec,podmin_statement,sslexpire,masterversion,shortversion,softwarename,monthsmonitored,score,signup,name,country,city,state,lat,long,uptime_alltime,active_users_halfyear,active_users_monthly,service_facebook,service_twitter,service_tumblr,service_wordpress,service_xmpp,responsetime,date_updated,ipv6,total_users,local_posts,comment_counts,stats_apikey,userrating FROM pods pods ORDER BY uptime_alltime DESC';
$sql = 'SELECT domain,dnssec,podmin_statement,sslexpire,masterversion,shortversion,softwarename,monthsmonitored,score,signup,name,country,city,state,lat,long,uptime_alltime,active_users_halfyear,active_users_monthly,service_facebook,service_twitter,service_tumblr,service_wordpress,service_xmpp,latency,date_updated,ipv6,total_users,local_posts,comment_counts,stats_apikey,userrating FROM pods pods ORDER BY uptime_alltime DESC';
$result = pg_query($dbh, $sql);
$result || die('Error in SQL query: ' . pg_last_error());
......@@ -26,7 +26,7 @@ $numrows = pg_num_rows($result);
<th><a data-toggle="tooltip" data-placement="bottom" title="Type of software this pod runs">Software</a></th>
<th><a data-toggle="tooltip" data-placement="bottom" title="Percent of the time the pod is online.">Uptime</a></th>
<th><a data-toggle="tooltip" data-placement="bottom" title="Does this pod offer ipv6 connection.">IPv6</a></th>
<th><a data-toggle="tooltip" data-placement="bottom" title="Average response/ping time in ms.">Ping</a></th>
<th><a data-toggle="tooltip" data-placement="bottom" title="Average connection latency time in ms.">Latency</a></th>
<th><a data-toggle="tooltip" data-placement="bottom" title="Does this pod allow new users.">Signups</a></th>
<th><a data-toggle="tooltip" data-placement="bottom" title="Number of total users on this pod.">Users</a></th>
<th><a data-toggle="tooltip" data-placement="bottom" title="Number of users active last 6 months on this pod.">Active 6m</a></th>
......@@ -46,7 +46,7 @@ $numrows = pg_num_rows($result);
<?php
while ($row = pg_fetch_array($result)) {
$pod_name = htmlentities($row['name'], ENT_QUOTES);
$tip = "\n This {$row['softwarename']} pod {$pod_name} has been watched for {$row['monthsmonitored']} months with an overall uptime of {$row['uptime_alltime']}% and a response time average today of {$row['responsetime']}ms was last checked on {$row['date_updated']}. ";
$tip = "\n This {$row['softwarename']} pod {$pod_name} has been watched for {$row['monthsmonitored']} months with an overall uptime of {$row['uptime_alltime']}% and a response time average today of {$row['latency']}ms was last checked on {$row['date_updated']}. ";
$tip .= "On a scale of 100 this pod is a {$row['score']} right now";
echo '<tr><td><a title="' . $tip . '" data-toggle="tooltip" data-placement="bottom" target="_self" href="/go.php?domain=' . $row['domain'] . '">' . $row['domain'] . '</a><span class="text-success" " data-toggle="tooltip" title="This site is SSL/TLS encrypted with a cert that expires: ' . $row['sslexpire'] . '"> &#128274;</span></td>';
......@@ -72,7 +72,7 @@ $numrows = pg_num_rows($result);
echo '<td>' . $row['softwarename'] . '</td>';
echo '<td>' . ($row['uptime_alltime'] > 0 ? $row['uptime_alltime'].'%' : '') . '</td>';
echo '<td>' . ($row['ipv6'] === 't' ? '&#10003;' : '') . '</td>';
echo '<td>' . ($row['responsetime'] > 0 ? $row['responsetime'] : '') . '</td>';
echo '<td>' . ($row['latency'] > 0 ? $row['latency'] : '') . '</td>';
echo '<td>' . ($row['signup'] === 't' ? '&#10003;' : '') . '</td>';
echo '<td>' . ($row['total_users'] > 0 ? $row['total_users'] : '') . '</td>';
echo '<td>' . ($row['active_users_halfyear'] > 0 ? $row['active_users_halfyear'] : '') . '</td>';
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment