pull.php 15.9 KB
Newer Older
MatrixCrawler's avatar
MatrixCrawler committed
1
<?php
2 3 4 5 6 7

/**
 * Pull pod info.
 */

declare(strict_types=1);
8

David Morley's avatar
David Morley committed
9 10 11
require_once __DIR__ . '/../boot.php';

if (!is_cli()) {
David Morley's avatar
David Morley committed
12 13 14 15 16
    $referer = ($_SERVER['HTTP_REFERER'] ? parse_url($_SERVER['HTTP_REFERER'])['host'] : '');
    if ($referer !== $_SERVER['SERVER_NAME']) {
        header('HTTP/1.0 403 Forbidden');
        exit;
    }
17 18
}

19
use Carbon\Carbon;
20
use GeoIp2\Database\Reader;
David Morley's avatar
David Morley committed
21
use Longman\IPTools\Ip;
22 23
use Poduptime\PodStatus;
use RedBeanPHP\R;
24

25 26
$debug    = isset($_GET['debug']) || (isset($argv) && in_array('debug', $argv, true));
$sqldebug = isset($_GET['sqldebug']) || (isset($argv) && in_array('sqldebug', $argv, true));
David Morley's avatar
David Morley committed
27
$develop  = isset($_GET['develop']) || (isset($argv) && in_array('develop', $argv, true));
28
$write    = !(isset($_GET['nowrite']) || (isset($argv) && in_array('nowrite', $argv, true)));
David Morley's avatar
David Morley committed
29
$newline  = is_cli() ? "\n\n" : '<br><br>';
David Morley's avatar
David Morley committed
30

31 32 33
$_domain = $_GET['domain'] ?? null;

// Must have a domain, except if called from CLI.
David Morley's avatar
David Morley committed
34
$_domain || is_cli() || die('No valid input');
35

36
$sqldebug && R::fancyDebug(true);
37 38

try {
David Morley's avatar
David Morley committed
39
    // Setup GeoIP Database
40
    $reader = new Reader(c('geoip2db'));
David Morley's avatar
David Morley committed
41

42
    $sql = '
43
        SELECT domain, score, date_created, weight, podmin_notify, podmin_notify_level, email, masterversion, shortversion, status, detectedlanguage
44 45 46 47
        FROM pods
    ';

    $pods = [];
David Morley's avatar
David Morley committed
48

49 50 51
    if ($_domain) {
        $sql  .= ' WHERE domain = ?';
        $pods = R::getAll($sql, [$_domain]);
David Morley's avatar
David Morley committed
52 53
    } elseif (is_cli() && (isset($argv) && in_array('Check_System_Deleted', $argv, true))) {
        $sql  .= ' WHERE status = ? ORDER BY id';
54
        $pods = R::getAll($sql, [PodStatus::SYSTEM_DELETED]);
David Morley's avatar
David Morley committed
55 56
    } elseif (is_cli()) {
        $sql  .= ' WHERE status < ? ORDER BY id';
57 58
        $pods = R::getAll($sql, [PodStatus::PAUSED]);
    }
59
} catch (\RedBeanPHP\RedException $e) {
60
    die('Error in SQL query: ' . $e->getMessage());
David Morley's avatar
David Morley committed
61 62
} catch (\MaxMind\Db\Reader\InvalidDatabaseException $e) {
    die('Invalid GeoIP database: ' . $e->getMessage());
David Morley's avatar
David Morley committed
63
}
64 65

foreach ($pods as $pod) {
66 67 68 69 70 71 72 73 74 75 76 77
    $domain          = $pod['domain'];
    $score           = (int) $pod['score'];
    $dbscore         = $score;
    $dateadded       = $pod['date_created'];
    $weight          = $pod['weight'];
    $email           = $pod['email'];
    $notify          = $pod['podmin_notify'];
    $notify_level    = $pod['podmin_notify_level'];
    $masterv         = $pod['masterversion'];
    $shortv          = $pod['shortversion'];
    $dbstatus        = $pod['status'];
    $language        = $pod['detectedlanguage'];
78 79 80

    try {
        $ratings = R::getAll('
81 82
            SELECT rating
            FROM ratingcomments
83 84 85 86 87
            WHERE domain = ?
        ', [$domain]);
    } catch (\RedBeanPHP\RedException $e) {
        die('Error in SQL query: ' . $e->getMessage());
    }
88

David Morley's avatar
David Morley committed
89
    debug('Domain', $domain);
90

David Morley's avatar
David Morley committed
91
    $user_rating = 0;
David Morley's avatar
David Morley committed
92

David Morley's avatar
David Morley committed
93 94
    if ($user_ratings = array_column($ratings, 'rating')) {
        $user_rating = round(array_sum($user_ratings) / count($user_ratings), 2);
95
    }
96

David Morley's avatar
David Morley committed
97
    $nodeinfo_meta = curl("https://{$domain}/.well-known/nodeinfo");
98

99 100
    // Default link to fetch node info.
    $nodeinfo_url = "https://{$domain}/nodeinfo/1.0";
101

David Morley's avatar
David Morley committed
102
    if ($info = json_decode($nodeinfo_meta['body'] ?: '', true)) {
103
        $nodeinfo_url = max($info['links'])['href'];
104 105
    }

David Morley's avatar
David Morley committed
106
    debug('Nodeinfo link', $nodeinfo_url);
107

David Morley's avatar
David Morley committed
108
    $nodeinfo       = curl($nodeinfo_url);
109 110 111
    $outputssl      = $nodeinfo['body'];
    $outputsslerror = $nodeinfo['error'];
    $info           = $nodeinfo['info'];
David Morley's avatar
David Morley committed
112
    $httpcode       = $nodeinfo['code'];
113 114
    $conntime       = $nodeinfo['conntime'];
    $nstime         = $nodeinfo['nstime'];
115 116 117
    $latency        = $conntime - $nstime;
    $sslexpire      = $info[0]['Expire date'] ?? null;

David Morley's avatar
David Morley committed
118 119 120 121 122 123 124
    debug('Nodeinfo output', $outputssl, true);
    debug('Nodeinfo output error', $outputsslerror, true);
    debug('Nodeinfo service http response code', $httpcode);
    debug('Cert expire date', $sslexpire);
    debug('Conntime', $conntime);
    debug('NStime', $nstime);
    debug('Latency', $latency);
125

David Morley's avatar
David Morley committed
126
    $jsonssl = json_decode($outputssl ?: '');
127 128

    if ($jsonssl !== null) {
David Morley's avatar
David Morley committed
129
        $version               = $jsonssl->software->version ?? 0;
David Morley's avatar
David Morley committed
130 131
        preg_match_all('((?:\d(.|-)?)+(\.)\d+\.*)', $version, $sversion);
        $shortversion          = $sversion[0][0] ?? '0.0.0.0';
132 133 134 135 136 137 138 139 140 141 142 143
        $signup                = ($jsonssl->openRegistrations ?? false) === true;
        $softwarename          = $jsonssl->software->name ?? 'unknown';
        $name                  = $jsonssl->metadata->nodeName ?? $softwarename;
        $total_users           = $jsonssl->usage->users->total ?? 0;
        $active_users_halfyear = $jsonssl->usage->users->activeHalfyear ?? 0;
        $active_users_monthly  = $jsonssl->usage->users->activeMonth ?? 0;
        $local_posts           = $jsonssl->usage->localPosts ?? 0;
        $comment_counts        = $jsonssl->usage->localComments ?? 0;
        $service_xmpp          = ($jsonssl->metadata->xmppChat ?? false) === true;
        if (json_last_error() === 0) {
            (!$jsonssl->software->version) || $score += 1;
            if (is_array($jsonssl->services->outbound)) {
144
                $services = json_encode($jsonssl->services->outbound);
145 146
            }
        }
147 148 149 150 151 152 153 154 155 156

        try {
            $c                   = R::dispense('checks');
            $c['domain']         = $domain;
            $c['online']         = true;
            $c['latency']        = $latency;
            $c['total_users']    = $total_users;
            $c['local_posts']    = $local_posts;
            $c['comment_counts'] = $comment_counts;
            $c['shortversion']   = $shortversion;
David Morley's avatar
David Morley committed
157
            $c['version']        = $version;
David Morley's avatar
David Morley committed
158

159 160 161 162 163 164 165 166 167
            if ($write) {
                R::store($c);
            } else {
                echo $c;
            }
        } catch (\RedBeanPHP\RedException $e) {
            die('Error in SQL query: ' . $e->getMessage());
        }

David Morley's avatar
David Morley committed
168
        debug('Version code', $shortversion);
David Morley's avatar
David Morley committed
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184

        try {
            $masterdata = R::getRow('
                SELECT version, devlastcommit, releasedate
                FROM masterversions
                WHERE software = ?
                ORDER BY id
                DESC LIMIT 1
            ', [$softwarename]);
        } catch (\RedBeanPHP\RedException $e) {
            die('Error in SQL query: ' . $e->getMessage());
        }

        $masterversion = ($masterdata['version'] ?? '0.0.0.0');
        $devlastcommit = ($masterdata['devlastcommit'] ?? date('Y-m-d H:i:s'));
        $releasedate   = ($masterdata['releasedate'] ?? date('Y-m-d H:i:s'));
David Morley's avatar
David Morley committed
185

David Morley's avatar
David Morley committed
186
        debug('Masterversion', $masterversion);
David Morley's avatar
David Morley committed
187

David Morley's avatar
David Morley committed
188 189 190 191
        $masterversioncheck = explode('.', $masterversion);
        $shortversioncheck  = (strpos($shortversion, '.') ? explode('.', $shortversion) : implode('.', ['0', preg_replace('/\D/', '', $shortversion), '0']));
        //this is still off with a pod with v1 as total version. cant explode that, won't have a [0] or [1] later to use either

David Morley's avatar
David Morley committed
192
        debug('Days since master code release', date_diff(new DateTime($releasedate), new DateTime())->format('%d'));
David Morley's avatar
David Morley committed
193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209

        try {
            $lastpodupdates = R::getRow('
                SELECT DISTINCT ON (shortversion, date_checked) shortversion, date_checked
                FROM checks
                WHERE domain = ?
                    AND shortversion IS NOT NULL
                ORDER BY shortversion DESC, date_checked
                LIMIT 1
            ', [$domain]);
        } catch (\RedBeanPHP\RedException $e) {
            die('Error in SQL query: ' . $e->getMessage());
        }

        $lastdatechecked = ($lastpodupdates['date_checked'] ?? date('Y-m-d H:i:s'));
        $devlastdays     = $devlastcommit ? date_diff(new DateTime($devlastcommit), new DateTime())->format('%a') : 30;//tmp//if no dev branch then what?

David Morley's avatar
David Morley committed
210
        debug('Dev last commit was  ', $devlastdays);
David Morley's avatar
David Morley committed
211

David Morley's avatar
David Morley committed
212 213 214 215 216 217
        $updategap = date_diff(new DateTime($lastdatechecked), new DateTime($devlastcommit))->format('%a');

        if (strpos($version, 'dev') !== false || strpos($version, 'rc') !== false || $shortversioncheck > $masterversioncheck) {
            //tmp//if pod is on the development branch - see when you last updated your pod and when the last commit was made to dev branch - if the repo is active and your not updating every 120 days why are you on dev branch?

            if ($updategap + $devlastdays > 400) {
David Morley's avatar
David Morley committed
218

David Morley's avatar
David Morley committed
219
                debug('Outdated More than 400 days', 'Yes');
David Morley's avatar
David Morley committed
220

David Morley's avatar
David Morley committed
221
                $podminhelp = 'Your code base seems too out of date to be used. Last time you updated was ' . $updategap;
David Morley's avatar
David Morley committed
222 223 224 225
                $score -= 2;
            }
        } elseif (($masterversioncheck[1] - $shortversioncheck[1]) > 1) {
            ///tmp/If pod is two versions off AND it's been more than 60 days since that release came out AND your on the master production branch
David Morley's avatar
David Morley committed
226

David Morley's avatar
David Morley committed
227
            debug('Outdated second decimal > 1', 'Yes');
David Morley's avatar
David Morley committed
228

David Morley's avatar
David Morley committed
229 230
            $score     -= 2;
            $updategap = date_diff(new DateTime($lastdatechecked), new DateTime($releasedate))->format('%a');
David Morley's avatar
David Morley committed
231 232
            $podminhelp = 'Your code base seems too out of date to be used. Current version is ' . $masterversion . ' and you are running ' . $shortversion;

David Morley's avatar
David Morley committed
233 234
        } elseif ($updategap - date_diff(new DateTime($releasedate), new DateTime())->format('%a') > 400) {

David Morley's avatar
David Morley committed
235
            debug('Outdated more than 400 days since x ', 'Yes');
David Morley's avatar
David Morley committed
236

David Morley's avatar
David Morley committed
237 238
            $score     -= 2;
            $updategap = date_diff(new DateTime($lastdatechecked), new DateTime($releasedate))->format('%a');
David Morley's avatar
David Morley committed
239 240
            $podminhelp = 'Your code base seems too out of date to be used. Last time you updated was ' . $updategap;

David Morley's avatar
David Morley committed
241 242 243 244
        } else {
            $updategap = date_diff(new DateTime($lastdatechecked), new DateTime($releasedate))->format('%a');
        }

David Morley's avatar
David Morley committed
245
        debug('Pod code was updated after ', $updategap);
David Morley's avatar
David Morley committed
246

247 248 249
        $status = PodStatus::UP;
    }

250 251
    // Default to the already saved language.
    $detectedlanguage = $language;
David Morley's avatar
David Morley committed
252

253 254
    $language_snippet = getWebsiteLanguageSnippetFromUrl("https://{$domain}/");
    if (!$language_snippet) {
David Morley's avatar
David Morley committed
255
        $detectedlanguage = null;
256
        --$score;
David Morley's avatar
David Morley committed
257
        $podminhelp = 'Unable to render the html on your main page https://' . $domain;
Armando Lüscher's avatar
Armando Lüscher committed
258
    } elseif ($debug || Carbon::now()->hour === 12) {
259
        $detectedlanguage = detectWebsiteLanguageFromSnippet($language_snippet);
David Morley's avatar
David Morley committed
260 261
    }

David Morley's avatar
David Morley committed
262
    debug('Detected Language', $detectedlanguage);
David Morley's avatar
David Morley committed
263

264
    if (!$jsonssl || !$language_snippet) {
David Morley's avatar
David Morley committed
265
        debug('Connection', 'Can not connect to pod');
266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281

        try {
            $c            = R::dispense('checks');
            $c['domain']  = $domain;
            $c['online']  = false;
            $c['error']   = $outputsslerror;
            $c['latency'] = $latency;
            if ($write) {
                R::store($c);
            } else {
                echo $c;
            }
        } catch (\RedBeanPHP\RedException $e) {
            die('Error in SQL query: ' . $e->getMessage());
        }

282
        --$score;
283 284 285
        $status = PodStatus::DOWN;
    }

David Morley's avatar
David Morley committed
286
    debug('Signup Open', $signup);
287

288
    $dnsserver = c('dnsserver') ?: '1.1.1.1';
289 290 291 292 293 294 295
    $delv      = new NPM\Xec\Command("delv @{$dnsserver} {$domain}");
    $delv->throwExceptionOnError(false);

    $ip         = '';
    $iplookupv4 = explode(PHP_EOL, trim($delv->execute([], null, 15)->stdout));
    $dnssec     = in_array('; fully validated', $iplookupv4, true) ?? false;
    $getaonly   = array_values(preg_grep('/\s+IN\s+A\s+.*/', $iplookupv4));
David Morley's avatar
David Morley committed
296

297 298 299
    if ($getaonly) {
        preg_match('/A\s(.*)/', $getaonly[0], $aversion);
        $ip = trim($aversion[1]) ?? '';
300
    }
301 302 303 304
    $ip || $score -= 2;

    $iplookupv6 = explode(PHP_EOL, trim($delv->execute(['AAAA'], null, 15)->stdout));
    $ipv6       = (bool) preg_grep('/\s+IN\s+AAAA\s+.*/', $iplookupv6);
305

David Morley's avatar
David Morley committed
306 307
    debug('IPv4', $ip);
    debug('IPv6', $ipv6);
308

David Morley's avatar
David Morley committed
309 310
    // todo: Temporary workaround (see https://github.com/akalongman/php-ip-tools/issues/8)
    if (Ip::isValid($ip) && Ip::isRemote($ip)) {
David Morley's avatar
David Morley committed
311 312 313 314 315 316 317
        $geo         = $reader->city($ip);
        $countryname = ($geo->country->name ?? null) ?: null;
        $country     = ($geo->country->isoCode ?? null) ?: null;
        $city        = ($geo->city->name ?? null) ?: null;
        $state       = ($geo->mostSpecificSubdivision->name ?? null) ?: null;
        $lat         = ($geo->location->latitude ?? null) ?: 0;
        $long        = ($geo->location->longitude ?? null) ?: 0;
318

David Morley's avatar
David Morley committed
319
        debug('Location', json_encode($geo->raw), true);
David Morley's avatar
David Morley committed
320
    }
321 322 323 324 325 326
    echo $newline;
    $statslastdate = date('Y-m-d H:i:s');

    $diff   = (new DateTime())->diff(new DateTime($dateadded));
    $months = $diff->m + ($diff->y * 12);
    $days   = $diff->days;
327 328

    try {
329 330 331 332 333 334 335 336 337 338
        $checks = R::getRow('
            SELECT
                round(avg(latency) * 1000) AS latency,
                round(avg(online::INT) * 100, 2) AS online
            FROM checks
            WHERE domain = ?
        ', [$domain]);

        $avglatency = $checks['latency'] ?? 0;
        $uptime     = $checks['online'] ?? 0;
339
    } catch (\RedBeanPHP\RedException $e) {
340
        die('Error in SQL query: ' . $e->getMessage());
341 342
    }

David Morley's avatar
David Morley committed
343
    debug('Uptime', $uptime);
344

345
    if ($score == ($notify_level - 1) && $notify && !$develop && $dbscore == $notify_level) {
346
        $to      = $email;
347
        $headers = ['From: ' . c('adminemail'), 'Bcc: ' . c('adminemail')];
348
        $subject = 'Monitoring notice from poduptime';
David Morley's avatar
David Morley committed
349 350 351 352 353 354 355 356 357
        $message = 'Notice for ' . $domain . '. Your score is ' . $score . ' and your pod will fall off the list soon. HTTP response of ' . $httpcode . '. ';

        if ($outputsslerror) {
            $message .= 'SSL Error ' . $outputsslerror;
        }

        if ($podminhelp) {
            $message .= ' ' . $podminhelp;
        }
358
        @mail($to, $subject, $message, implode("\r\n", $headers));
David Morley's avatar
David Morley committed
359

David Morley's avatar
David Morley committed
360 361
        debug('Mail Message body', $message);
        debug('Mail Notice', 'sent to ' . $email);
David Morley's avatar
David Morley committed
362
    }
David Morley's avatar
David Morley committed
363

364 365
    if ($score > 100) {
        $score = 100;
David Morley's avatar
David Morley committed
366
    } elseif ($score < -6000) {
David Morley's avatar
David Morley committed
367
        $status = PodStatus::SYSTEM_DELETED;
David Morley's avatar
David Morley committed
368
    }
369
    $weightedscore = ($uptime + $score - (10 - $weight)) / 2;
David Morley's avatar
David Morley committed
370

David Morley's avatar
David Morley committed
371 372
    debug('Score', $score);
    debug('Weighted Score', $weightedscore);
373

374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397
    try {
        $p                     = R::findOne('pods', 'domain = ?', [$domain]);
        $p['ip']               = $ip;
        $p['ipv6']             = $ipv6;
        $p['daysmonitored']    = $days;
        $p['monthsmonitored']  = $months;
        $p['uptime_alltime']   = $uptime;
        $p['status']           = $status;
        $p['date_laststats']   = $statslastdate;
        $p['date_updated']     = date('Y-m-d H:i:s');
        $p['latency']          = $avglatency;
        $p['score']            = $score;
        $p['country']          = $country;
        $p['countryname']      = $countryname;
        $p['city']             = $city;
        $p['state']            = $state;
        $p['lat']              = $lat;
        $p['long']             = $long;
        $p['userrating']       = $user_rating;
        $p['masterversion']    = $masterversion;
        $p['weightedscore']    = $weightedscore;
        $p['sslvalid']         = $outputsslerror;
        $p['dnssec']           = $dnssec;
        $p['sslexpire']        = $sslexpire;
David Morley's avatar
David Morley committed
398
        if ($dbstatus == PodStatus::UP && $status == PodStatus::UP) {
399 400 401 402 403 404 405
            $p['shortversion']          = $shortversion;
            $p['signup']                = $signup;
            $p['total_users']           = $total_users;
            $p['active_users_halfyear'] = $active_users_halfyear;
            $p['active_users_monthly']  = $active_users_monthly;
            $p['local_posts']           = $local_posts;
            $p['name']                  = $name;
David Morley's avatar
David Morley committed
406
            $p['detectedlanguage'] = $detectedlanguage;
407 408
            $p['comment_counts']        = $comment_counts;
            $p['service_xmpp']          = $service_xmpp;
409
            $p['services']              = $services;
410 411 412 413 414 415
            $p['softwarename']          = $softwarename;
        }

        if ($write) {
            R::store($p);
        } else {
416 417
            echo 'Data not saved, testing only';
            echo $newline;
418 419 420
        }
    } catch (\RedBeanPHP\RedException $e) {
        die('Error in SQL query: ' . $e->getMessage());
421
    }
David Morley's avatar
David Morley committed
422

423
    echo 'Success ' . $domain;
424

425
    echo $newline;
426
}