pull.php 15.8 KB
Newer Older
MatrixCrawler's avatar
MatrixCrawler committed
1
<?php
2 3 4 5 6 7

/**
 * Pull pod info.
 */

declare(strict_types=1);
8

9
if ($_SERVER['SERVER_ADDR'] !== $_SERVER['REMOTE_ADDR']) {
10 11
    header('HTTP/1.0 403 Forbidden');
    exit;
12 13
}

14
use LanguageDetection\Language;
15 16
use Poduptime\PodStatus;
use RedBeanPHP\R;
dmorley's avatar
dmorley committed
17
use GeoIp2\Database\Reader;
noplanman's avatar
noplanman committed
18

19 20 21 22
$debug    = isset($_GET['debug']) || (isset($argv) && in_array('debug', $argv, true));
$sqldebug = isset($_GET['sqldebug']) || (isset($argv) && in_array('sqldebug', $argv, true));
$write    = !(isset($_GET['nowrite']) || (isset($argv) && in_array('nowrite', $argv, true)));
$newline  = PHP_SAPI === 'cli' ? "\n\n" : '<br><br>';
David Morley's avatar
David Morley committed
23

noplanman's avatar
noplanman committed
24 25 26 27
$_domain = $_GET['domain'] ?? null;

// Must have a domain, except if called from CLI.
$_domain || PHP_SAPI === 'cli' || die('No valid input');
28

David Morley's avatar
David Morley committed
29
require_once __DIR__ . '/../vendor/autoload.php';
noplanman's avatar
noplanman committed
30 31 32
require_once __DIR__ . '/../config.php';

define('PODUPTIME', microtime(true));
dmorley's avatar
cleanup  
dmorley committed
33

noplanman's avatar
noplanman committed
34 35
// Set up global DB connection.
R::setup("pgsql:host={$pghost};dbname={$pgdb}", $pguser, $pgpass, true);
36
$sqldebug && R::fancyDebug(true);
noplanman's avatar
noplanman committed
37
R::testConnection() || die('Error in DB connection');
dmorley's avatar
dmorley committed
38
R::usePartialBeans(true);
noplanman's avatar
noplanman committed
39

dmorley's avatar
dmorley committed
40 41 42
// Setup GeoIP Database
$reader = new Reader($geoip2db);

noplanman's avatar
noplanman committed
43
try {
44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59
    $sql = '
        SELECT domain, score, date_created, adminrating, weight, hidden, podmin_notify, email, masterversion, shortversion, status
        FROM pods
    ';

    $pods = [];
    if ($_domain) {
        $sql  .= ' WHERE domain = ?';
        $pods = R::getAll($sql, [$_domain]);
    } elseif (PHP_SAPI === 'cli' && (isset($argv) && in_array('Check_System_Deleted', $argv, true))) {
        $sql  .= ' WHERE status = ?';
        $pods = R::getAll($sql, [PodStatus::SYSTEM_DELETED]);
    } elseif (PHP_SAPI === 'cli') {
        $sql  .= ' WHERE status < ?';
        $pods = R::getAll($sql, [PodStatus::PAUSED]);
    }
noplanman's avatar
noplanman committed
60
} catch (\RedBeanPHP\RedException $e) {
61
    die('Error in SQL query: ' . $e->getMessage());
dmorley's avatar
cleanup  
dmorley committed
62
}
noplanman's avatar
noplanman committed
63 64

foreach ($pods as $pod) {
65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85
    $domain    = $pod['domain'];
    $score     = (int) $pod['score'];
    $dateadded = $pod['date_created'];
    $admindb   = (int) $pod['adminrating'];
    $weight    = $pod['weight'];
    $hiddennow = $pod['hidden'];
    $email     = $pod['email'];
    $notify    = $pod['podmin_notify'];
    $masterv   = $pod['masterversion'];
    $shortv    = $pod['shortversion'];
    $dbstatus  = $pod['status'];

    try {
        $ratings = R::getAll('
            SELECT admin, rating
            FROM rating_comments
            WHERE domain = ?
        ', [$domain]);
    } catch (\RedBeanPHP\RedException $e) {
        die('Error in SQL query: ' . $e->getMessage());
    }
86

87
    _debug('Domain', $domain);
noplanman's avatar
noplanman committed
88

89 90 91 92 93 94 95 96
    $user_ratings  = [];
    $admin_ratings = [];
    foreach ($ratings as $rating) {
        if ($rating['admin'] == 0) {
            $user_ratings[] = $rating['rating'];
        } elseif ($rating['admin'] == 1) {
            $admin_ratings[] = $rating['rating'];
        }
97
    }
98 99 100 101 102
    $user_rating  = empty($user_ratings) ? 0 : max(10, round(array_sum($user_ratings) / count($user_ratings), 2));
    $admin_rating = empty($admin_ratings) ? 0 : max(10, round(array_sum($admin_ratings) / count($admin_ratings), 2));

    if ($admindb == -1) {
        $admin_rating = -1;
103
    }
noplanman's avatar
noplanman committed
104

105 106 107 108 109 110 111 112 113
    $d = new DOMDocument;
    libxml_use_internal_errors(true);
    $d->loadHTMLFile('https://' . $domain);
    $body = $d->getElementsByTagName('body')->item(0);
    if ($body->nodeValue) {
        $ld               = new Language;
        $detectedlanguage = strtoupper(key($ld->detect($body->nodeValue)->bestResults()->close()));
        _debug('Detected Language', $detectedlanguage);
    }
noplanman's avatar
noplanman committed
114

115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230
    if ($infos = json_decode(file_get_contents('https://' . $domain . '/.well-known/nodeinfo'), true)) {
        $link = max($infos['links'])['href'];
    } else {
        $link = 'https://' . $domain . '/nodeinfo/1.0';
    }

    $chss = curl_init();
    curl_setopt($chss, CURLOPT_URL, $link);
    curl_setopt($chss, CURLOPT_CONNECTTIMEOUT, 10);
    curl_setopt($chss, CURLOPT_TIMEOUT, 30);
    curl_setopt($chss, CURLOPT_RETURNTRANSFER, 1);
    curl_setopt($chss, CURLOPT_CERTINFO, 1);
    curl_setopt($chss, CURLOPT_CAINFO, $cafullpath);
    $outputssl      = curl_exec($chss);
    $outputsslerror = curl_error($chss);
    $info           = curl_getinfo($chss, CURLINFO_CERTINFO);
    $conntime       = curl_getinfo($chss, CURLINFO_CONNECT_TIME);
    $nstime         = curl_getinfo($chss, CURLINFO_NAMELOOKUP_TIME);
    $latency        = $conntime - $nstime;
    $sslexpire      = $info[0]['Expire date'] ?? null;
    curl_close($chss);

    _debug('Nodeinfo output', $outputssl, true);
    _debug('Nodeinfo output error', $outputsslerror, true);
    _debug('Cert expire date', $sslexpire);
    _debug('Conntime', $conntime);
    _debug('NStime', $nstime);
    _debug('Latency', $latency);

    $jsonssl = json_decode($outputssl);

    $xdver = $jsonssl->software->version ?? 0;
    preg_match_all('((?:\d(.|-)?)+(\.|-)\d+\.*)', $xdver, $dverr);
    $shortversion          = $dverr[0][0];
    $signup                = ($jsonssl->openRegistrations ?? false) === true;
    $softwarename          = $jsonssl->software->name ?? 'unknown';
    $name                  = $jsonssl->metadata->nodeName ?? $softwarename;
    $total_users           = $jsonssl->usage->users->total ?? 0;
    $active_users_halfyear = $jsonssl->usage->users->activeHalfyear ?? 0;
    $active_users_monthly  = $jsonssl->usage->users->activeMonth ?? 0;
    $local_posts           = $jsonssl->usage->localPosts ?? 0;
    $comment_counts        = $jsonssl->usage->localComments ?? 0;
    $service_xmpp          = ($jsonssl->metadata->xmppChat ?? false) === true;
    $service_facebook      = false;
    $service_twitter       = false;
    $service_tumblr        = false;
    $service_wordpress     = false;
    if (json_last_error() === 0) {
        (!$jsonssl->software->version) || $score += 1;
        if (is_array($jsonssl->services->outbound)) {
            $service_facebook  = in_array('facebook', $jsonssl->services->outbound, true);
            $service_twitter   = in_array('twitter', $jsonssl->services->outbound, true);
            $service_tumblr    = in_array('tumblr', $jsonssl->services->outbound, true);
            $service_wordpress = in_array('wordpress', $jsonssl->services->outbound, true);
        }
    }

    if ($jsonssl !== null) {

        try {
            $c                   = R::dispense('checks');
            $c['domain']         = $domain;
            $c['online']         = true;
            $c['latency']        = $latency;
            $c['total_users']    = $total_users;
            $c['local_posts']    = $local_posts;
            $c['comment_counts'] = $comment_counts;
            $c['shortversion']   = $shortversion;
            if ($write) {
                R::store($c);
            } else {
                echo $c;
            }
        } catch (\RedBeanPHP\RedException $e) {
            die('Error in SQL query: ' . $e->getMessage());
        }

        $status = PodStatus::UP;
    }

    if (!$jsonssl) {
        _debug('Connection', 'Can not connect to pod');

        try {
            $c            = R::dispense('checks');
            $c['domain']  = $domain;
            $c['online']  = false;
            $c['error']   = $outputsslerror;
            $c['latency'] = $latency;
            if ($write) {
                R::store($c);
            } else {
                echo $c;
            }
        } catch (\RedBeanPHP\RedException $e) {
            die('Error in SQL query: ' . $e->getMessage());
        }

        $score  -= 1;
        $status = PodStatus::DOWN;
    }

    _debug('Version code', $shortversion);
    _debug('Signup Open', $signup);

    $dnsserver = !empty($dnsserver) ? $dnsserver : '1.1.1.1';
    $delv      = new NPM\Xec\Command("delv @{$dnsserver} {$domain}");
    $delv->throwExceptionOnError(false);

    $ip         = '';
    $iplookupv4 = explode(PHP_EOL, trim($delv->execute([], null, 15)->stdout));
    $dnssec     = in_array('; fully validated', $iplookupv4, true) ?? false;
    $getaonly   = array_values(preg_grep('/\s+IN\s+A\s+.*/', $iplookupv4));
    if ($getaonly) {
        preg_match('/A\s(.*)/', $getaonly[0], $aversion);
        $ip = trim($aversion[1]) ?? '';
noplanman's avatar
noplanman committed
231
    }
232 233 234 235
    $ip || $score -= 2;

    $iplookupv6 = explode(PHP_EOL, trim($delv->execute(['AAAA'], null, 15)->stdout));
    $ipv6       = (bool) preg_grep('/\s+IN\s+AAAA\s+.*/', $iplookupv6);
236

237 238 239 240
    _debug('IPv4', $ip);
    _debug('Iplookupv4', $iplookupv4, true);
    _debug('IPv6', $ipv6);
    _debug('Iplookupv6', $iplookupv6, true);
noplanman's avatar
noplanman committed
241

dmorley's avatar
dmorley committed
242
    $geo = $reader->city($ip);
dmorley's avatar
dmorley committed
243 244 245 246
    $countryname = $geo->country->name ?? null ?: null;
    $country     = $geo->country->isoCode ?? null ?: null;
    $city        = $geo->city->name ?? null ?: null;
    $state       = $geo->mostSpecificSubdivision->name ?? null ?: null;
247 248
    $lat         = $geo->location->latitude ?? null ?: 0;
    $long        = $geo->location->longitude ?? null ?: 0;
dmorley's avatar
dmorley committed
249 250

    _debug('Location', json_encode($geo->raw), true);
251 252 253 254 255 256 257

    echo $newline;
    $statslastdate = date('Y-m-d H:i:s');

    $diff   = (new DateTime())->diff(new DateTime($dateadded));
    $months = $diff->m + ($diff->y * 12);
    $days   = $diff->days;
noplanman's avatar
noplanman committed
258 259

    try {
260 261 262 263 264 265 266 267 268 269
        $checks = R::getRow('
            SELECT
                round(avg(latency) * 1000) AS latency,
                round(avg(online::INT) * 100, 2) AS online
            FROM checks
            WHERE domain = ?
        ', [$domain]);

        $avglatency = $checks['latency'] ?? 0;
        $uptime     = $checks['online'] ?? 0;
noplanman's avatar
noplanman committed
270
    } catch (\RedBeanPHP\RedException $e) {
271
        die('Error in SQL query: ' . $e->getMessage());
noplanman's avatar
noplanman committed
272 273
    }

274
    _debug('Uptime', $uptime);
noplanman's avatar
noplanman committed
275

276 277 278 279 280
    try {
        $masterdata = R::getRow('SELECT version, devlastcommit, releasedate FROM masterversions WHERE software = ? ORDER BY id DESC LIMIT 1', [$softwarename]);
    } catch (\RedBeanPHP\RedException $e) {
        die('Error in SQL query: ' . $e->getMessage());
    }
David Morley's avatar
David Morley committed
281

282 283 284 285
    $masterversion = $masterdata['version'];
    _debug('Masterversion', $masterversion);
    $masterversioncheck = explode('.', $masterversion);
    $shortversioncheck  = explode('.', $shortversion);
286

287
    _debug('Days since master code release', date_diff((new DateTime($masterdata['releasedate'])), (new DateTime()))->format('%d'));
288 289 290


    try {
291
        $lastpodupdates = R::getRow('SELECT DISTINCT ON (shortversion) shortversion, date_checked FROM checks WHERE domain = ? AND shortversion IS NOT NULL ORDER BY shortversion DESC LIMIT 1', [$domain]);
292
    } catch (\RedBeanPHP\RedException $e) {
293
        die('Error in SQL query: ' . $e->getMessage());
294
    }
295 296 297 298

    $devlastdays = $masterdata['devlastcommit'] ? date_diff((new DateTime($masterdata['devlastcommit'])), (new DateTime()))->format('%a') : 30;//tmp//if no dev branch then what?


299
    _debug('Dev git last commit was  ', $devlastdays);
300
    if (strpos($xdver, 'dev') !== false || strpos($xdver, 'rc') !== false || $shortversioncheck > $masterversioncheck) {
301
//tmp//if pod is on the development branch - see when you last updated your pod and when the last commit was made to dev branch - if the repo is active and your not updating every 120 days why are you on dev branch?
302 303 304 305 306 307
        $updategap = date_diff((new DateTime($lastpodupdates['date_checked'])), (new DateTime($masterdata['devlastcommit'])))->format('%a');
        if ($updategap + $devlastdays > 130) {
            _debug('Outdated', 'Yes');
            $score -= 2;
        }
    } elseif (($masterversioncheck[1] - $shortversioncheck[1]) > 1) {
308
///tmp/If pod is two versions off AND it's been more than 60 days since that release came out AND your on the master production branch
309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330
        _debug('Outdated', 'Yes');
        $score     -= 2;
        $updategap = date_diff((new DateTime($lastpodupdates['date_checked'])), (new DateTime($masterdata['releasedate'])))->format('%a');
    } elseif ($updategap - date_diff((new DateTime($masterdata['releasedate'])), (new DateTime()))->format('%a') > 90) {
        _debug('Outdated', 'Yes');
        $score     -= 2;
        $updategap = date_diff((new DateTime($lastpodupdates['date_checked'])), (new DateTime($masterdata['releasedate'])))->format('%a');
    } else {
        $updategap = date_diff((new DateTime($lastpodupdates['date_checked'])), (new DateTime($masterdata['releasedate'])))->format('%a');
    }
    _debug('Pod code was updated after ', $updategap);

    $hidden = $score <= 70;
    _debug('Hidden', $hidden ? 'yes' : 'no');

    if (!$hiddennow && $hidden && $notify && !(isset($argv) && in_array('develop', $argv, true))) {
        $to      = $email;
        $headers = ['From: ' . $adminemail, 'Bcc: ' . $adminemail];
        $subject = 'Monitoring notice from poduptime';
        $message = 'Notice for ' . $domain . '. Your score fell to ' . $score . ' and your pod is now marked as hidden.';
        @mail($to, $subject, $message, implode("\r\n", $headers));
        _debug('Mail Notice', 'sent to ' . $email);
dmorley's avatar
dmorley committed
331
    }
332 333 334 335 336 337 338
    if ($score > 100) {
        $score = 100;
    } elseif ($score < 0) {
        $score = 0;
        if ($masterv <> $shortv) {
            $status = PodStatus::SYSTEM_DELETED;
        }
dmorley's avatar
dmorley committed
339
    }
340 341 342
    _debug('Score', $score);
    $weightedscore = ($uptime + $score - (10 - $weight)) / 2;
    _debug('Weighted Score', $weightedscore);
noplanman's avatar
noplanman committed
343

344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395
    try {
        $p                     = R::findOne('pods', 'domain = ?', [$domain]);
        $p['secure']           = true;
        $p['hidden']           = $hidden;
        $p['ip']               = $ip;
        $p['ipv6']             = $ipv6;
        $p['daysmonitored']    = $days;
        $p['monthsmonitored']  = $months;
        $p['uptime_alltime']   = $uptime;
        $p['status']           = $status;
        $p['date_laststats']   = $statslastdate;
        $p['date_updated']     = date('Y-m-d H:i:s');
        $p['latency']          = $avglatency;
        $p['score']            = $score;
        $p['adminrating']      = $admin_rating;
        $p['country']          = $country;
        $p['countryname']      = $countryname;
        $p['city']             = $city;
        $p['state']            = $state;
        $p['lat']              = $lat;
        $p['long']             = $long;
        $p['detectedlanguage'] = $detectedlanguage;
        $p['userrating']       = $user_rating;
        $p['masterversion']    = $masterversion;
        $p['weightedscore']    = $weightedscore;
        $p['sslvalid']         = $outputsslerror;
        $p['dnssec']           = $dnssec;
        $p['sslexpire']        = $sslexpire;
        if ($dbstatus == PodStatus::UP && $status == PodStatus::UP) {
            $p['shortversion']          = $shortversion;
            $p['signup']                = $signup;
            $p['total_users']           = $total_users;
            $p['active_users_halfyear'] = $active_users_halfyear;
            $p['active_users_monthly']  = $active_users_monthly;
            $p['local_posts']           = $local_posts;
            $p['name']                  = $name;
            $p['comment_counts']        = $comment_counts;
            $p['service_facebook']      = $service_facebook;
            $p['service_tumblr']        = $service_tumblr;
            $p['service_twitter']       = $service_twitter;
            $p['service_wordpress']     = $service_wordpress;
            $p['service_xmpp']          = $service_xmpp;
            $p['softwarename']          = $softwarename;
        }

        if ($write) {
            R::store($p);
        } else {
            echo $p;
        }
    } catch (\RedBeanPHP\RedException $e) {
        die('Error in SQL query: ' . $e->getMessage());
396
    }
David Morley's avatar
David Morley committed
397

398
    echo 'Success ' . $domain;
noplanman's avatar
noplanman committed
399

400 401
    echo $newline;
    echo $newline;
David Morley's avatar
David Morley committed
402
}
403

David Morley's avatar
David Morley committed
404 405
/**
 * Output a debug message and variable value
noplanman's avatar
noplanman committed
406
 *
David Morley's avatar
David Morley committed
407 408 409 410
 * @param string $label
 * @param mixed  $var
 * @param bool   $dump
 */
noplanman's avatar
noplanman committed
411 412
function _debug($label, $var = null, $dump = false)
{
413
    global $debug, $newline;
414

415 416 417
    if (!$debug) {
        return;
    }
noplanman's avatar
noplanman committed
418

419 420 421 422 423 424 425
    if ($dump || is_array($var)) {
        $output = print_r($var, true);
    } elseif (is_bool($var)) {
        $output = $var ? 'true' : 'false';
    } else {
        $output = (string) $var;
    }
noplanman's avatar
noplanman committed
426

427
    printf('%s: %s%s', $label, $output, $newline);
428
}