pull.php 15.9 KB
Newer Older
MatrixCrawler's avatar
MatrixCrawler committed
1
<?php
2 3 4 5 6 7

/**
 * Pull pod info.
 */

declare(strict_types=1);
8

dmorley's avatar
dmorley committed
9 10 11
require_once __DIR__ . '/../boot.php';

if (!is_cli()) {
dmorley's avatar
dmorley committed
12 13 14 15 16
    $referer = ($_SERVER['HTTP_REFERER'] ? parse_url($_SERVER['HTTP_REFERER'])['host'] : '');
    if ($referer !== $_SERVER['SERVER_NAME']) {
        header('HTTP/1.0 403 Forbidden');
        exit;
    }
17 18
}

19
use Carbon\Carbon;
dmorley's avatar
dmorley committed
20
use GeoIp2\Database\Reader;
dmorley's avatar
dmorley committed
21
use Longman\IPTools\Ip;
22 23
use Poduptime\PodStatus;
use RedBeanPHP\R;
noplanman's avatar
noplanman committed
24

25 26
$debug    = isset($_GET['debug']) || (isset($argv) && in_array('debug', $argv, true));
$sqldebug = isset($_GET['sqldebug']) || (isset($argv) && in_array('sqldebug', $argv, true));
dmorley's avatar
dmorley committed
27
$develop  = isset($_GET['develop']) || (isset($argv) && in_array('develop', $argv, true));
28
$write    = !(isset($_GET['nowrite']) || (isset($argv) && in_array('nowrite', $argv, true)));
dmorley's avatar
dmorley committed
29
$newline  = is_cli() ? "\n\n" : '<br><br>';
David Morley's avatar
David Morley committed
30

noplanman's avatar
noplanman committed
31 32 33
$_domain = $_GET['domain'] ?? null;

// Must have a domain, except if called from CLI.
dmorley's avatar
dmorley committed
34
$_domain || is_cli() || die('No valid input');
noplanman's avatar
noplanman committed
35

noplanman's avatar
noplanman committed
36
$sqldebug && R::fancyDebug(true);
noplanman's avatar
noplanman committed
37 38

try {
dmorley's avatar
dmorley committed
39
    // Setup GeoIP Database
noplanman's avatar
noplanman committed
40
    $reader = new Reader(c('geoip2db'));
dmorley's avatar
dmorley committed
41

42
    $sql = '
dmorley's avatar
dmorley committed
43
        SELECT domain, score, date_created, weight, podmin_notify, podmin_notify_level, email, masterversion, shortversion, status, detectedlanguage
44 45 46 47
        FROM pods
    ';

    $pods = [];
dmorley's avatar
dmorley committed
48

49 50 51
    if ($_domain) {
        $sql  .= ' WHERE domain = ?';
        $pods = R::getAll($sql, [$_domain]);
dmorley's avatar
dmorley committed
52 53
    } elseif (is_cli() && (isset($argv) && in_array('Check_System_Deleted', $argv, true))) {
        $sql  .= ' WHERE status = ? ORDER BY id';
54
        $pods = R::getAll($sql, [PodStatus::SYSTEM_DELETED]);
dmorley's avatar
dmorley committed
55 56
    } elseif (is_cli()) {
        $sql  .= ' WHERE status < ? ORDER BY id';
57 58
        $pods = R::getAll($sql, [PodStatus::PAUSED]);
    }
noplanman's avatar
noplanman committed
59
} catch (\RedBeanPHP\RedException $e) {
60
    die('Error in SQL query: ' . $e->getMessage());
dmorley's avatar
dmorley committed
61 62
} catch (\MaxMind\Db\Reader\InvalidDatabaseException $e) {
    die('Invalid GeoIP database: ' . $e->getMessage());
dmorley's avatar
cleanup  
dmorley committed
63
}
noplanman's avatar
noplanman committed
64 65

foreach ($pods as $pod) {
dmorley's avatar
dmorley committed
66 67 68 69 70 71 72 73 74 75 76 77
    $domain          = $pod['domain'];
    $score           = (int) $pod['score'];
    $dbscore         = $score;
    $dateadded       = $pod['date_created'];
    $weight          = $pod['weight'];
    $email           = $pod['email'];
    $notify          = $pod['podmin_notify'];
    $notify_level    = $pod['podmin_notify_level'];
    $masterv         = $pod['masterversion'];
    $shortv          = $pod['shortversion'];
    $dbstatus        = $pod['status'];
    $language        = $pod['detectedlanguage'];
78 79 80

    try {
        $ratings = R::getAll('
dmorley's avatar
dmorley committed
81 82
            SELECT rating
            FROM ratingcomments
83 84 85 86 87
            WHERE domain = ?
        ', [$domain]);
    } catch (\RedBeanPHP\RedException $e) {
        die('Error in SQL query: ' . $e->getMessage());
    }
88

dmorley's avatar
dmorley committed
89
    debug('Domain', $domain);
noplanman's avatar
noplanman committed
90

dmorley's avatar
dmorley committed
91
    $user_rating = 0;
dmorley's avatar
dmorley committed
92

dmorley's avatar
dmorley committed
93 94
    if ($user_ratings = array_column($ratings, 'rating')) {
        $user_rating = round(array_sum($user_ratings) / count($user_ratings), 2);
95
    }
96

dmorley's avatar
dmorley committed
97
    $nodeinfo_meta = curl("https://{$domain}/.well-known/nodeinfo");
noplanman's avatar
noplanman committed
98

99 100
    // Default link to fetch node info.
    $nodeinfo_url = "https://{$domain}/nodeinfo/1.0";
101

dmorley's avatar
dmorley committed
102
    if ($info = json_decode($nodeinfo_meta['body'] ?: '', true)) {
103
        $nodeinfo_url = max($info['links'])['href'];
104 105
    }

dmorley's avatar
dmorley committed
106
    debug('Nodeinfo link', $nodeinfo_url);
dmorley's avatar
dmorley committed
107

dmorley's avatar
dmorley committed
108
    $nodeinfo       = curl($nodeinfo_url);
109 110 111
    $outputssl      = $nodeinfo['body'];
    $outputsslerror = $nodeinfo['error'];
    $info           = $nodeinfo['info'];
dmorley's avatar
dmorley committed
112
    $httpcode       = $nodeinfo['code'];
113 114
    $conntime       = $nodeinfo['conntime'];
    $nstime         = $nodeinfo['nstime'];
115 116 117
    $latency        = $conntime - $nstime;
    $sslexpire      = $info[0]['Expire date'] ?? null;

dmorley's avatar
dmorley committed
118 119 120 121 122 123 124
    debug('Nodeinfo output', $outputssl, true);
    debug('Nodeinfo output error', $outputsslerror, true);
    debug('Nodeinfo service http response code', $httpcode);
    debug('Cert expire date', $sslexpire);
    debug('Conntime', $conntime);
    debug('NStime', $nstime);
    debug('Latency', $latency);
125

dmorley's avatar
dmorley committed
126
    $jsonssl = json_decode($outputssl ?: '');
127 128

    if ($jsonssl !== null) {
dmorley's avatar
dmorley committed
129
        $version               = $jsonssl->software->version ?? 0;
dmorley's avatar
dmorley committed
130 131
        preg_match_all('((?:\d(.|-)?)+(\.)\d+\.*)', $version, $sversion);
        $shortversion          = $sversion[0][0] ?? '0.0.0.0';
dmorley's avatar
dmorley committed
132 133 134 135 136 137 138 139 140 141 142 143
        $signup                = ($jsonssl->openRegistrations ?? false) === true;
        $softwarename          = $jsonssl->software->name ?? 'unknown';
        $name                  = $jsonssl->metadata->nodeName ?? $softwarename;
        $total_users           = $jsonssl->usage->users->total ?? 0;
        $active_users_halfyear = $jsonssl->usage->users->activeHalfyear ?? 0;
        $active_users_monthly  = $jsonssl->usage->users->activeMonth ?? 0;
        $local_posts           = $jsonssl->usage->localPosts ?? 0;
        $comment_counts        = $jsonssl->usage->localComments ?? 0;
        $service_xmpp          = ($jsonssl->metadata->xmppChat ?? false) === true;
        if (json_last_error() === 0) {
            (!$jsonssl->software->version) || $score += 1;
            if (is_array($jsonssl->services->outbound)) {
144
                $services = json_encode($jsonssl->services->outbound);
dmorley's avatar
dmorley committed
145 146
            }
        }
147 148 149 150 151 152 153 154 155 156

        try {
            $c                   = R::dispense('checks');
            $c['domain']         = $domain;
            $c['online']         = true;
            $c['latency']        = $latency;
            $c['total_users']    = $total_users;
            $c['local_posts']    = $local_posts;
            $c['comment_counts'] = $comment_counts;
            $c['shortversion']   = $shortversion;
dmorley's avatar
dmorley committed
157
            $c['version']        = $version;
dmorley's avatar
dmorley committed
158

159 160 161 162 163 164 165 166 167
            if ($write) {
                R::store($c);
            } else {
                echo $c;
            }
        } catch (\RedBeanPHP\RedException $e) {
            die('Error in SQL query: ' . $e->getMessage());
        }

dmorley's avatar
dmorley committed
168
        debug('Version code', $shortversion);
dmorley's avatar
dmorley committed
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184

        try {
            $masterdata = R::getRow('
                SELECT version, devlastcommit, releasedate
                FROM masterversions
                WHERE software = ?
                ORDER BY id
                DESC LIMIT 1
            ', [$softwarename]);
        } catch (\RedBeanPHP\RedException $e) {
            die('Error in SQL query: ' . $e->getMessage());
        }

        $masterversion = ($masterdata['version'] ?? '0.0.0.0');
        $devlastcommit = ($masterdata['devlastcommit'] ?? date('Y-m-d H:i:s'));
        $releasedate   = ($masterdata['releasedate'] ?? date('Y-m-d H:i:s'));
dmorley's avatar
dmorley committed
185

dmorley's avatar
dmorley committed
186
        debug('Masterversion', $masterversion);
dmorley's avatar
dmorley committed
187

dmorley's avatar
dmorley committed
188 189 190 191
        $masterversioncheck = explode('.', $masterversion);
        $shortversioncheck  = (strpos($shortversion, '.') ? explode('.', $shortversion) : implode('.', ['0', preg_replace('/\D/', '', $shortversion), '0']));
        //this is still off with a pod with v1 as total version. cant explode that, won't have a [0] or [1] later to use either

dmorley's avatar
dmorley committed
192
        debug('Days since master code release', date_diff(new DateTime($releasedate), new DateTime())->format('%d'));
dmorley's avatar
dmorley committed
193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209

        try {
            $lastpodupdates = R::getRow('
                SELECT DISTINCT ON (shortversion, date_checked) shortversion, date_checked
                FROM checks
                WHERE domain = ?
                    AND shortversion IS NOT NULL
                ORDER BY shortversion DESC, date_checked
                LIMIT 1
            ', [$domain]);
        } catch (\RedBeanPHP\RedException $e) {
            die('Error in SQL query: ' . $e->getMessage());
        }

        $lastdatechecked = ($lastpodupdates['date_checked'] ?? date('Y-m-d H:i:s'));
        $devlastdays     = $devlastcommit ? date_diff(new DateTime($devlastcommit), new DateTime())->format('%a') : 30;//tmp//if no dev branch then what?

dmorley's avatar
dmorley committed
210
        debug('Dev last commit was  ', $devlastdays);
dmorley's avatar
dmorley committed
211

dmorley's avatar
dmorley committed
212 213 214 215 216 217
        $updategap = date_diff(new DateTime($lastdatechecked), new DateTime($devlastcommit))->format('%a');

        if (strpos($version, 'dev') !== false || strpos($version, 'rc') !== false || $shortversioncheck > $masterversioncheck) {
            //tmp//if pod is on the development branch - see when you last updated your pod and when the last commit was made to dev branch - if the repo is active and your not updating every 120 days why are you on dev branch?

            if ($updategap + $devlastdays > 400) {
dmorley's avatar
dmorley committed
218

dmorley's avatar
dmorley committed
219
                debug('Outdated More than 400 days', 'Yes');
dmorley's avatar
dmorley committed
220

dmorley's avatar
dmorley committed
221
                $podminhelp = 'Your code base seems too out of date to be used. Last time you updated was ' . $updategap;
dmorley's avatar
dmorley committed
222 223 224 225
                $score -= 2;
            }
        } elseif (($masterversioncheck[1] - $shortversioncheck[1]) > 1) {
            ///tmp/If pod is two versions off AND it's been more than 60 days since that release came out AND your on the master production branch
dmorley's avatar
dmorley committed
226

dmorley's avatar
dmorley committed
227
            debug('Outdated second decimal > 1', 'Yes');
dmorley's avatar
dmorley committed
228

dmorley's avatar
dmorley committed
229 230
            $score     -= 2;
            $updategap = date_diff(new DateTime($lastdatechecked), new DateTime($releasedate))->format('%a');
dmorley's avatar
dmorley committed
231 232
            $podminhelp = 'Your code base seems too out of date to be used. Current version is ' . $masterversion . ' and you are running ' . $shortversion;

dmorley's avatar
dmorley committed
233 234
        } elseif ($updategap - date_diff(new DateTime($releasedate), new DateTime())->format('%a') > 400) {

dmorley's avatar
dmorley committed
235
            debug('Outdated more than 400 days since x ', 'Yes');
dmorley's avatar
dmorley committed
236

dmorley's avatar
dmorley committed
237 238
            $score     -= 2;
            $updategap = date_diff(new DateTime($lastdatechecked), new DateTime($releasedate))->format('%a');
dmorley's avatar
dmorley committed
239 240
            $podminhelp = 'Your code base seems too out of date to be used. Last time you updated was ' . $updategap;

dmorley's avatar
dmorley committed
241 242 243 244
        } else {
            $updategap = date_diff(new DateTime($lastdatechecked), new DateTime($releasedate))->format('%a');
        }

dmorley's avatar
dmorley committed
245
        debug('Pod code was updated after ', $updategap);
dmorley's avatar
dmorley committed
246

247 248 249
        $status = PodStatus::UP;
    }

250 251
    // Default to the already saved language.
    $detectedlanguage = $language;
dmorley's avatar
dmorley committed
252

253 254
    $language_snippet = getWebsiteLanguageSnippetFromUrl("https://{$domain}/");
    if (!$language_snippet) {
dmorley's avatar
dmorley committed
255
        $detectedlanguage = null;
256
        --$score;
dmorley's avatar
dmorley committed
257
        $podminhelp = 'Unable to render the html on your main page https://' . $domain;
noplanman's avatar
noplanman committed
258
    } elseif ($debug || Carbon::now()->hour === 12) {
259
        $detectedlanguage = detectWebsiteLanguageFromSnippet($language_snippet);
dmorley's avatar
dmorley committed
260 261
    }

dmorley's avatar
dmorley committed
262
    debug('Detected Language', $detectedlanguage);
dmorley's avatar
dmorley committed
263

264
    if (!$jsonssl || !$language_snippet) {
dmorley's avatar
dmorley committed
265
        debug('Connection', 'Can not connect to pod');
266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281

        try {
            $c            = R::dispense('checks');
            $c['domain']  = $domain;
            $c['online']  = false;
            $c['error']   = $outputsslerror;
            $c['latency'] = $latency;
            if ($write) {
                R::store($c);
            } else {
                echo $c;
            }
        } catch (\RedBeanPHP\RedException $e) {
            die('Error in SQL query: ' . $e->getMessage());
        }

282
        --$score;
283 284 285
        $status = PodStatus::DOWN;
    }

dmorley's avatar
dmorley committed
286
    debug('Signup Open', $signup);
287

noplanman's avatar
noplanman committed
288
    $dnsserver = c('dnsserver') ?: '1.1.1.1';
289 290 291 292 293 294 295
    $delv      = new NPM\Xec\Command("delv @{$dnsserver} {$domain}");
    $delv->throwExceptionOnError(false);

    $ip         = '';
    $iplookupv4 = explode(PHP_EOL, trim($delv->execute([], null, 15)->stdout));
    $dnssec     = in_array('; fully validated', $iplookupv4, true) ?? false;
    $getaonly   = array_values(preg_grep('/\s+IN\s+A\s+.*/', $iplookupv4));
dmorley's avatar
dmorley committed
296

297 298 299
    if ($getaonly) {
        preg_match('/A\s(.*)/', $getaonly[0], $aversion);
        $ip = trim($aversion[1]) ?? '';
noplanman's avatar
noplanman committed
300
    }
301 302 303 304
    $ip || $score -= 2;

    $iplookupv6 = explode(PHP_EOL, trim($delv->execute(['AAAA'], null, 15)->stdout));
    $ipv6       = (bool) preg_grep('/\s+IN\s+AAAA\s+.*/', $iplookupv6);
305

dmorley's avatar
dmorley committed
306 307
    debug('IPv4', $ip);
    debug('IPv6', $ipv6);
noplanman's avatar
noplanman committed
308

dmorley's avatar
dmorley committed
309 310
    // todo: Temporary workaround (see https://github.com/akalongman/php-ip-tools/issues/8)
    if (Ip::isValid($ip) && Ip::isRemote($ip)) {
dmorley's avatar
dmorley committed
311 312 313 314 315 316 317
        $geo         = $reader->city($ip);
        $countryname = ($geo->country->name ?? null) ?: null;
        $country     = ($geo->country->isoCode ?? null) ?: null;
        $city        = ($geo->city->name ?? null) ?: null;
        $state       = ($geo->mostSpecificSubdivision->name ?? null) ?: null;
        $lat         = ($geo->location->latitude ?? null) ?: 0;
        $long        = ($geo->location->longitude ?? null) ?: 0;
318

dmorley's avatar
dmorley committed
319
        debug('Location', json_encode($geo->raw), true);
dmorley's avatar
dmorley committed
320
    }
321 322 323 324 325 326
    echo $newline;
    $statslastdate = date('Y-m-d H:i:s');

    $diff   = (new DateTime())->diff(new DateTime($dateadded));
    $months = $diff->m + ($diff->y * 12);
    $days   = $diff->days;
noplanman's avatar
noplanman committed
327 328

    try {
329 330 331 332 333 334 335 336 337 338
        $checks = R::getRow('
            SELECT
                round(avg(latency) * 1000) AS latency,
                round(avg(online::INT) * 100, 2) AS online
            FROM checks
            WHERE domain = ?
        ', [$domain]);

        $avglatency = $checks['latency'] ?? 0;
        $uptime     = $checks['online'] ?? 0;
noplanman's avatar
noplanman committed
339
    } catch (\RedBeanPHP\RedException $e) {
340
        die('Error in SQL query: ' . $e->getMessage());
noplanman's avatar
noplanman committed
341 342
    }

dmorley's avatar
dmorley committed
343
    debug('Uptime', $uptime);
noplanman's avatar
noplanman committed
344

dmorley's avatar
dmorley committed
345
    if ($score == ($notify_level - 1) && $notify && !$develop && $dbscore == $notify_level) {
346
        $to      = $email;
noplanman's avatar
noplanman committed
347
        $headers = ['From: ' . c('adminemail'), 'Bcc: ' . c('adminemail')];
348
        $subject = 'Monitoring notice from poduptime';
dmorley's avatar
dmorley committed
349 350 351 352 353 354 355 356 357
        $message = 'Notice for ' . $domain . '. Your score is ' . $score . ' and your pod will fall off the list soon. HTTP response of ' . $httpcode . '. ';

        if ($outputsslerror) {
            $message .= 'SSL Error ' . $outputsslerror;
        }

        if ($podminhelp) {
            $message .= ' ' . $podminhelp;
        }
358
        @mail($to, $subject, $message, implode("\r\n", $headers));
dmorley's avatar
dmorley committed
359

dmorley's avatar
dmorley committed
360 361
        debug('Mail Message body', $message);
        debug('Mail Notice', 'sent to ' . $email);
dmorley's avatar
dmorley committed
362
    }
dmorley's avatar
dmorley committed
363

364 365
    if ($score > 100) {
        $score = 100;
dmorley's avatar
dmorley committed
366
    } elseif ($score < -6000) {
dmorley's avatar
dmorley committed
367
        $status = PodStatus::SYSTEM_DELETED;
dmorley's avatar
dmorley committed
368
    }
369
    $weightedscore = ($uptime + $score - (10 - $weight)) / 2;
dmorley's avatar
dmorley committed
370

dmorley's avatar
dmorley committed
371 372
    debug('Score', $score);
    debug('Weighted Score', $weightedscore);
noplanman's avatar
noplanman committed
373

374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398
    try {
        $p                     = R::findOne('pods', 'domain = ?', [$domain]);
        $p['ip']               = $ip;
        $p['ipv6']             = $ipv6;
        $p['daysmonitored']    = $days;
        $p['monthsmonitored']  = $months;
        $p['uptime_alltime']   = $uptime;
        $p['status']           = $status;
        $p['date_laststats']   = $statslastdate;
        $p['date_updated']     = date('Y-m-d H:i:s');
        $p['latency']          = $avglatency;
        $p['score']            = $score;
        $p['country']          = $country;
        $p['countryname']      = $countryname;
        $p['city']             = $city;
        $p['state']            = $state;
        $p['lat']              = $lat;
        $p['long']             = $long;
        $p['detectedlanguage'] = $detectedlanguage;
        $p['userrating']       = $user_rating;
        $p['masterversion']    = $masterversion;
        $p['weightedscore']    = $weightedscore;
        $p['sslvalid']         = $outputsslerror;
        $p['dnssec']           = $dnssec;
        $p['sslexpire']        = $sslexpire;
dmorley's avatar
dmorley committed
399
        if ($dbstatus == PodStatus::UP && $status == PodStatus::UP) {
400 401 402 403 404 405 406 407 408
            $p['shortversion']          = $shortversion;
            $p['signup']                = $signup;
            $p['total_users']           = $total_users;
            $p['active_users_halfyear'] = $active_users_halfyear;
            $p['active_users_monthly']  = $active_users_monthly;
            $p['local_posts']           = $local_posts;
            $p['name']                  = $name;
            $p['comment_counts']        = $comment_counts;
            $p['service_xmpp']          = $service_xmpp;
409
            $p['services']              = $services;
410 411 412 413 414 415
            $p['softwarename']          = $softwarename;
        }

        if ($write) {
            R::store($p);
        } else {
dmorley's avatar
dmorley committed
416 417
            echo 'Data not saved, testing only';
            echo $newline;
418 419 420
        }
    } catch (\RedBeanPHP\RedException $e) {
        die('Error in SQL query: ' . $e->getMessage());
421
    }
David Morley's avatar
David Morley committed
422

423
    echo 'Success ' . $domain;
noplanman's avatar
noplanman committed
424

425
    echo $newline;
426
}