update.php 18.7 KB
Newer Older
MatrixCrawler's avatar
MatrixCrawler committed
1
<?php
2 3 4 5 6 7

/**
 * Pull pod info.
 */

declare(strict_types=1);
8

dmorley's avatar
dmorley committed
9 10 11
require_once __DIR__ . '/../boot.php';

if (!is_cli()) {
dmorley's avatar
dmorley committed
12 13 14 15 16
    $referer = ($_SERVER['HTTP_REFERER'] ? parse_url($_SERVER['HTTP_REFERER'])['host'] : '');
    if ($referer !== $_SERVER['SERVER_NAME']) {
        header('HTTP/1.0 403 Forbidden');
        exit;
    }
17 18
}

19
use Carbon\Carbon;
dmorley's avatar
dmorley committed
20
use GeoIp2\Database\Reader;
dmorley's avatar
dmorley committed
21
use Longman\IPTools\Ip;
22 23
use Poduptime\PodStatus;
use RedBeanPHP\R;
noplanman's avatar
noplanman committed
24

dmorley's avatar
dmorley committed
25 26 27 28 29 30
if (!is_connected()) {
    die('no internet');
}

$time_start = microtime(true);

31
$debug    = isset($_GET['debug']) || (isset($argv) && in_array('debug', $argv, true));
32
$init = isset($_GET['init']) || (isset($argv) && in_array('init', $argv, true));
33
$sqldebug = isset($_GET['sqldebug']) || (isset($argv) && in_array('sqldebug', $argv, true));
dmorley's avatar
dmorley committed
34
$develop  = isset($_GET['develop']) || (isset($argv) && in_array('develop', $argv, true));
35
$write    = !(isset($_GET['nowrite']) || (isset($argv) && in_array('nowrite', $argv, true)));
dmorley's avatar
dmorley committed
36
$newline  = is_cli() ? "\n\n" : '<br><br>';
David Morley's avatar
David Morley committed
37

noplanman's avatar
noplanman committed
38 39 40
$_domain = $_GET['domain'] ?? null;

// Must have a domain, except if called from CLI.
dmorley's avatar
dmorley committed
41
$_domain || is_cli() || die('No valid input');
noplanman's avatar
noplanman committed
42

noplanman's avatar
noplanman committed
43
$sqldebug && R::fancyDebug(true);
noplanman's avatar
noplanman committed
44 45

try {
dmorley's avatar
dmorley committed
46
    // Setup GeoIP Database
noplanman's avatar
noplanman committed
47
    $reader = new Reader(c('geoip2db'));
dmorley's avatar
dmorley committed
48

49
    $sql = '
dmorley's avatar
dmorley committed
50
        SELECT domain, score, date_created, weight, podmin_notify, podmin_notify_level, email, masterversion, shortversion, status, detectedlanguage
51 52 53 54
        FROM pods
    ';

    $pods = [];
dmorley's avatar
dmorley committed
55

56 57 58
    if ($_domain) {
        $sql  .= ' WHERE domain = ?';
        $pods = R::getAll($sql, [$_domain]);
dmorley's avatar
dmorley committed
59
    } elseif (is_cli()) {
dmorley's avatar
dmorley committed
60 61 62
        if (getMeta('pods_updating')) {
            die('already running');
        }
dmorley's avatar
dmorley committed
63
        updateMeta('pods_updating', true);
dmorley's avatar
dmorley committed
64 65 66 67 68 69 70 71

        if (isset($argv) && in_array('Check_System_Deleted', $argv, true)) {
            $sql  .= ' WHERE status = ? ORDER BY id';
            $pods = R::getAll($sql, [PodStatus::SYSTEM_DELETED]);
        } else {
            $sql  .= ' WHERE status < ? ORDER BY id';
            $pods = R::getAll($sql, [PodStatus::PAUSED]);
        }
72
    }
noplanman's avatar
noplanman committed
73
} catch (\RedBeanPHP\RedException $e) {
dmorley's avatar
dmorley committed
74
    debug('Error in SQL query: ' . sprintf($e->getMessage()));
dmorley's avatar
dmorley committed
75
} catch (\MaxMind\Db\Reader\InvalidDatabaseException $e) {
dmorley's avatar
dmorley committed
76
    debug('Invalid GeoIP database: ' . sprintf($e->getMessage()));
dmorley's avatar
cleanup  
dmorley committed
77
}
noplanman's avatar
noplanman committed
78 79

foreach ($pods as $pod) {
dmorley's avatar
dmorley committed
80 81 82 83 84 85 86 87 88 89 90 91
    $domain          = $pod['domain'];
    $score           = (int) $pod['score'];
    $dbscore         = $score;
    $dateadded       = $pod['date_created'];
    $weight          = $pod['weight'];
    $email           = $pod['email'];
    $notify          = $pod['podmin_notify'];
    $notify_level    = $pod['podmin_notify_level'];
    $masterv         = $pod['masterversion'];
    $shortv          = $pod['shortversion'];
    $dbstatus        = $pod['status'];
    $language        = $pod['detectedlanguage'];
92 93 94

    try {
        $ratings = R::getAll('
dmorley's avatar
dmorley committed
95 96
            SELECT rating
            FROM ratingcomments
97 98 99
            WHERE domain = ?
        ', [$domain]);
    } catch (\RedBeanPHP\RedException $e) {
dmorley's avatar
dmorley committed
100
        debug('Error in SQL query: ' . sprintf($e->getMessage()));
101
    }
102

dmorley's avatar
dmorley committed
103
    debug('Domain', $domain);
noplanman's avatar
noplanman committed
104

dmorley's avatar
dmorley committed
105
    $user_rating = 0;
dmorley's avatar
dmorley committed
106

dmorley's avatar
dmorley committed
107 108
    if ($user_ratings = array_column($ratings, 'rating')) {
        $user_rating = round(array_sum($user_ratings) / count($user_ratings), 2);
109
    }
110

dmorley's avatar
dmorley committed
111
    $nodeinfo_meta = curl("https://{$domain}/.well-known/nodeinfo");
noplanman's avatar
noplanman committed
112

113 114
    // Default link to fetch node info.
    $nodeinfo_url = "https://{$domain}/nodeinfo/1.0";
115

dmorley's avatar
dmorley committed
116
    if ($info = json_decode($nodeinfo_meta['body'] ?: '', true)) {
dmorley's avatar
dmorley committed
117 118 119 120 121
        if (count($info['links'], COUNT_RECURSIVE) === 2) {
            $nodeinfo_url = $info['links']['href'];
        } else {
            $nodeinfo_url = max($info['links'])['href'];
        }
122 123
    }

dmorley's avatar
dmorley committed
124
    debug('Nodeinfo link', $nodeinfo_url);
dmorley's avatar
dmorley committed
125

dmorley's avatar
dmorley committed
126
    $nodeinfo       = curl($nodeinfo_url);
127 128 129
    $outputssl      = $nodeinfo['body'];
    $outputsslerror = $nodeinfo['error'];
    $info           = $nodeinfo['info'];
dmorley's avatar
dmorley committed
130
    $httpcode       = $nodeinfo['code'];
131 132
    $conntime       = $nodeinfo['conntime'];
    $nstime         = $nodeinfo['nstime'];
133 134 135
    $latency        = $conntime - $nstime;
    $sslexpire      = $info[0]['Expire date'] ?? null;

dmorley's avatar
dmorley committed
136 137 138 139 140 141 142
    debug('Nodeinfo output', $outputssl, true);
    debug('Nodeinfo output error', $outputsslerror, true);
    debug('Nodeinfo service http response code', $httpcode);
    debug('Cert expire date', $sslexpire);
    debug('Conntime', $conntime);
    debug('NStime', $nstime);
    debug('Latency', $latency);
143

dmorley's avatar
dmorley committed
144
    $jsonssl = json_decode($outputssl ?: '');
145 146

    if ($jsonssl !== null) {
dmorley's avatar
dmorley committed
147
        $version               = $jsonssl->software->version ?? 0;
dmorley's avatar
dmorley committed
148 149
        preg_match_all('((?:\d(.|-)?)+(\.)\d+\.*)', $version, $sversion);
        $shortversion          = $sversion[0][0] ?? '0.0.0.0';
dmorley's avatar
dmorley committed
150
        $signup                = ($jsonssl->openRegistrations ?? false) === true;
dmorley's avatar
dmorley committed
151
        $softwarename          = strtolower($jsonssl->software->name) ?? 'unknown';
dmorley's avatar
dmorley committed
152 153 154 155 156 157 158 159 160 161
        $name                  = $jsonssl->metadata->nodeName ?? $softwarename;
        $total_users           = $jsonssl->usage->users->total ?? 0;
        $active_users_halfyear = $jsonssl->usage->users->activeHalfyear ?? 0;
        $active_users_monthly  = $jsonssl->usage->users->activeMonth ?? 0;
        $local_posts           = $jsonssl->usage->localPosts ?? 0;
        $comment_counts        = $jsonssl->usage->localComments ?? 0;
        $service_xmpp          = ($jsonssl->metadata->xmppChat ?? false) === true;
        if (json_last_error() === 0) {
            (!$jsonssl->software->version) || $score += 1;
            if (is_array($jsonssl->services->outbound)) {
162
                $services = json_encode($jsonssl->services->outbound);
dmorley's avatar
dmorley committed
163
            }
dmorley's avatar
dmorley committed
164 165 166
            if (is_array($jsonssl->protocols)) {
                $protocols = json_encode($jsonssl->protocols);
            }
dmorley's avatar
dmorley committed
167
        }
168 169 170 171 172 173 174 175 176 177

        try {
            $c                   = R::dispense('checks');
            $c['domain']         = $domain;
            $c['online']         = true;
            $c['latency']        = $latency;
            $c['total_users']    = $total_users;
            $c['local_posts']    = $local_posts;
            $c['comment_counts'] = $comment_counts;
            $c['shortversion']   = $shortversion;
dmorley's avatar
dmorley committed
178
            $c['version']        = $version;
dmorley's avatar
dmorley committed
179

180 181 182
            if ($write) {
                R::store($c);
            } else {
dmorley's avatar
dmorley committed
183 184
                echo 'Data not saved, testing only';
                echo $newline;
185 186
            }
        } catch (\RedBeanPHP\RedException $e) {
dmorley's avatar
dmorley committed
187
            debug('Error in SQL query: ' . sprintf($e->getMessage()));
188 189
        }

dmorley's avatar
dmorley committed
190
        debug('Version code', $shortversion);
dmorley's avatar
dmorley committed
191 192 193

        try {
            $masterdata = R::getRow('
dmorley's avatar
dmorley committed
194
                SELECT version, devlastcommit, releasedate, date_checked
dmorley's avatar
dmorley committed
195 196 197 198 199 200
                FROM masterversions
                WHERE software = ?
                ORDER BY id
                DESC LIMIT 1
            ', [$softwarename]);
        } catch (\RedBeanPHP\RedException $e) {
dmorley's avatar
dmorley committed
201
            debug('Error in SQL query: ' . sprintf($e->getMessage()));
dmorley's avatar
dmorley committed
202 203 204 205 206
        }

        $masterversion = ($masterdata['version'] ?? '0.0.0.0');
        $devlastcommit = ($masterdata['devlastcommit'] ?? date('Y-m-d H:i:s'));
        $releasedate   = ($masterdata['releasedate'] ?? date('Y-m-d H:i:s'));
dmorley's avatar
dmorley committed
207

dmorley's avatar
dmorley committed
208
        debug('Masterversion', $masterversion);
dmorley's avatar
dmorley committed
209

dmorley's avatar
dmorley committed
210 211 212 213
        $masterversioncheck = explode('.', $masterversion);
        $shortversioncheck  = (strpos($shortversion, '.') ? explode('.', $shortversion) : implode('.', ['0', preg_replace('/\D/', '', $shortversion), '0']));
        //this is still off with a pod with v1 as total version. cant explode that, won't have a [0] or [1] later to use either

dmorley's avatar
dmorley committed
214
        debug('Days since master code release', date_diff(new DateTime($releasedate), new DateTime())->format('%d'));
dmorley's avatar
dmorley committed
215 216 217

        try {
            $lastpodupdates = R::getRow('
dmorley's avatar
dmorley committed
218
                SELECT DISTINCT ON (version, date_checked) version, date_checked
dmorley's avatar
dmorley committed
219 220
                FROM checks
                WHERE domain = ?
dmorley's avatar
dmorley committed
221 222
                    AND version IS NOT NULL
                ORDER BY version DESC, date_checked
dmorley's avatar
dmorley committed
223 224 225
                LIMIT 1
            ', [$domain]);
        } catch (\RedBeanPHP\RedException $e) {
dmorley's avatar
dmorley committed
226
            debug('Error in SQL query: ' . sprintf($e->getMessage()));
dmorley's avatar
dmorley committed
227 228
        }

dmorley's avatar
dmorley committed
229
        $lastdatechecked = ($lastpodupdates['date_checked'] ?? date('Y-m-d H:i:s.u'));
dmorley's avatar
dmorley committed
230 231
        $devlastdays     = $devlastcommit ? date_diff(new DateTime($devlastcommit), new DateTime())->format('%a') : 30;//tmp//if no dev branch then what?

dmorley's avatar
dmorley committed
232
        debug('Dev last commit was  ', $devlastdays);
dmorley's avatar
dmorley committed
233

dmorley's avatar
dmorley committed
234 235 236 237 238
        $updategap = date_diff(new DateTime($lastdatechecked), new DateTime($devlastcommit))->format('%a');

        if (strpos($version, 'dev') !== false || strpos($version, 'rc') !== false || $shortversioncheck > $masterversioncheck) {
            //tmp//if pod is on the development branch - see when you last updated your pod and when the last commit was made to dev branch - if the repo is active and your not updating every 120 days why are you on dev branch?

dmorley's avatar
dmorley committed
239 240
            if ($updategap + $devlastdays > 200) {
                debug('Outdated More than 200 days', 'Yes');
dmorley's avatar
dmorley committed
241

dmorley's avatar
dmorley committed
242
                $podminhelp = 'Your code base seems too out of date to be used. Last time you updated was ' . $updategap;
dmorley's avatar
dmorley committed
243 244 245 246
                $score -= 2;
            }
        } elseif (($masterversioncheck[1] - $shortversioncheck[1]) > 1) {
            ///tmp/If pod is two versions off AND it's been more than 60 days since that release came out AND your on the master production branch
dmorley's avatar
dmorley committed
247

dmorley's avatar
dmorley committed
248
            debug('Outdated second decimal > 1', 'Yes');
dmorley's avatar
dmorley committed
249

dmorley's avatar
dmorley committed
250 251
            $score     -= 2;
            $updategap = date_diff(new DateTime($lastdatechecked), new DateTime($releasedate))->format('%a');
dmorley's avatar
dmorley committed
252
            $podminhelp = 'Your code base seems too out of date to be used. Current version is ' . $masterversion . ' and you are running ' . $shortversion;
dmorley's avatar
dmorley committed
253 254
        } elseif ($updategap - date_diff(new DateTime($releasedate), new DateTime())->format('%a') > 200) {
            debug('Outdated more than 200 days since x ', 'Yes');
dmorley's avatar
dmorley committed
255

dmorley's avatar
dmorley committed
256 257
            $score     -= 2;
            $updategap = date_diff(new DateTime($lastdatechecked), new DateTime($releasedate))->format('%a');
dmorley's avatar
dmorley committed
258
            $podminhelp = 'Your code base seems too out of date to be used. Last time you updated was ' . $updategap;
dmorley's avatar
dmorley committed
259 260 261 262
        } else {
            $updategap = date_diff(new DateTime($lastdatechecked), new DateTime($releasedate))->format('%a');
        }

dmorley's avatar
dmorley committed
263
        debug('Pod code was updated after ', $updategap);
dmorley's avatar
dmorley committed
264

265 266 267
        $status = PodStatus::UP;
    }

dmorley's avatar
dmorley committed
268 269
    $langhours  = Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('languages_updated', 'date_created'))->diffInHours();

270 271
    // Default to the already saved language.
    $detectedlanguage = $language;
dmorley's avatar
dmorley committed
272
    $languagesupdated = false;
dmorley's avatar
dmorley committed
273

274 275
    $language_snippet = getWebsiteLanguageSnippetFromUrl("https://{$domain}/");
    if (!$language_snippet) {
dmorley's avatar
dmorley committed
276
        $detectedlanguage = null;
277
        --$score;
dmorley's avatar
dmorley committed
278
        $podminhelp = 'Unable to render the html on your main page https://' . $domain;
dmorley's avatar
dmorley committed
279
    } elseif ($debug || $langhours > 24) {
280
        $detectedlanguage = detectWebsiteLanguageFromSnippet($language_snippet);
dmorley's avatar
dmorley committed
281
        $languagesupdated = true;
dmorley's avatar
dmorley committed
282 283
    }

dmorley's avatar
dmorley committed
284
    debug('Detected Language', $detectedlanguage);
dmorley's avatar
dmorley committed
285

286
    if (!$jsonssl || !$language_snippet) {
dmorley's avatar
dmorley committed
287
        debug('Connection', 'Can not connect to pod');
288 289 290 291 292 293 294 295 296 297

        try {
            $c            = R::dispense('checks');
            $c['domain']  = $domain;
            $c['online']  = false;
            $c['error']   = $outputsslerror;
            $c['latency'] = $latency;
            if ($write) {
                R::store($c);
            } else {
dmorley's avatar
dmorley committed
298 299
                echo 'Data not saved, testing only';
                echo $newline;
300 301
            }
        } catch (\RedBeanPHP\RedException $e) {
dmorley's avatar
dmorley committed
302
            debug('Error in SQL query: ' . sprintf($e->getMessage()));
303 304
        }

305
        --$score;
306 307 308
        $status = PodStatus::DOWN;
    }

dmorley's avatar
dmorley committed
309
    debug('Signup Open', $signup);
310

noplanman's avatar
noplanman committed
311
    $dnsserver = c('dnsserver') ?: '1.1.1.1';
312 313
    $dig       = new NPM\Xec\Command("dig @{$dnsserver} {$domain} +dnssec");
    $dig->throwExceptionOnError(false);
314 315

    $ip         = '';
316 317 318
    $iplookupv4 = explode(PHP_EOL, trim($dig->execute(['A'], null, 15)->stdout));
    $flags      = preg_grep('/;; flags:(.*?);/', $iplookupv4);
    $dnssec     = (bool) preg_grep('/ad/', $flags);
319
    $getaonly   = array_values(preg_grep('/\s+IN\s+A\s+.*/', $iplookupv4));
dmorley's avatar
dmorley committed
320

321 322 323
    if ($getaonly) {
        preg_match('/A\s(.*)/', $getaonly[0], $aversion);
        $ip = trim($aversion[1]) ?? '';
noplanman's avatar
noplanman committed
324
    }
325 326
    $ip || $score -= 2;

327
    $iplookupv6 = explode(PHP_EOL, trim($dig->execute(['AAAA'], null, 15)->stdout));
328
    $ipv6       = (bool) preg_grep('/\s+IN\s+AAAA\s+.*/', $iplookupv6);
329

dmorley's avatar
dmorley committed
330 331
    debug('IPv4', $ip);
    debug('IPv6', $ipv6);
332
    debug('DNSSEC', $dnssec);
noplanman's avatar
noplanman committed
333

dmorley's avatar
dmorley committed
334
    // todo: Temporary workaround (see https://github.com/akalongman/php-ip-tools/issues/8)
dmorley's avatar
dmorley committed
335
    if (Ip::isValid($ip) && Ip::isRemote($ip) && $ip != '0.0.0.0') {
dmorley's avatar
dmorley committed
336 337 338 339 340 341 342
        $geo         = $reader->city($ip);
        $countryname = ($geo->country->name ?? null) ?: null;
        $country     = ($geo->country->isoCode ?? null) ?: null;
        $city        = ($geo->city->name ?? null) ?: null;
        $state       = ($geo->mostSpecificSubdivision->name ?? null) ?: null;
        $lat         = ($geo->location->latitude ?? null) ?: 0;
        $long        = ($geo->location->longitude ?? null) ?: 0;
343

dmorley's avatar
dmorley committed
344
        debug('Location', json_encode($geo->raw), true);
dmorley's avatar
dmorley committed
345
    }
dmorley's avatar
dmorley committed
346

347 348 349 350 351
    $statslastdate = date('Y-m-d H:i:s');

    $diff   = (new DateTime())->diff(new DateTime($dateadded));
    $months = $diff->m + ($diff->y * 12);
    $days   = $diff->days;
noplanman's avatar
noplanman committed
352 353

    try {
354 355 356 357 358 359 360 361 362 363
        $checks = R::getRow('
            SELECT
                round(avg(latency) * 1000) AS latency,
                round(avg(online::INT) * 100, 2) AS online
            FROM checks
            WHERE domain = ?
        ', [$domain]);

        $avglatency = $checks['latency'] ?? 0;
        $uptime     = $checks['online'] ?? 0;
noplanman's avatar
noplanman committed
364
    } catch (\RedBeanPHP\RedException $e) {
dmorley's avatar
dmorley committed
365
        debug('Error in SQL query: ' . sprintf($e->getMessage()));
noplanman's avatar
noplanman committed
366 367
    }

dmorley's avatar
dmorley committed
368
    debug('Uptime', $uptime);
noplanman's avatar
noplanman committed
369

dmorley's avatar
dmorley committed
370
    if ($score == ($notify_level - 1) && $notify && !$develop && $dbscore == $notify_level) {
371
        $to      = $email;
noplanman's avatar
noplanman committed
372
        $headers = ['From: ' . c('adminemail'), 'Bcc: ' . c('adminemail')];
373
        $subject = 'Monitoring notice from poduptime';
dmorley's avatar
dmorley committed
374 375 376 377 378 379 380 381 382
        $message = 'Notice for ' . $domain . '. Your score is ' . $score . ' and your pod will fall off the list soon. HTTP response of ' . $httpcode . '. ';

        if ($outputsslerror) {
            $message .= 'SSL Error ' . $outputsslerror;
        }

        if ($podminhelp) {
            $message .= ' ' . $podminhelp;
        }
383
        @mail($to, $subject, $message, implode("\r\n", $headers));
dmorley's avatar
dmorley committed
384

dmorley's avatar
dmorley committed
385 386
        debug('Mail Message body', $message);
        debug('Mail Notice', 'sent to ' . $email);
dmorley's avatar
dmorley committed
387
    }
dmorley's avatar
dmorley committed
388

dmorley's avatar
dmorley committed
389 390
    $weightedscore = ($uptime + $score - (10 - $weight)) / 2;

391 392
    if ($score > 100) {
        $score = 100;
dmorley's avatar
dmorley committed
393 394
    }
    if ($score < 1) {
dmorley's avatar
dmorley committed
395
        $weightedscore = 0;
dmorley's avatar
dmorley committed
396 397
    }
    if ($score < -6000) {
dmorley's avatar
dmorley committed
398
        $status = PodStatus::SYSTEM_DELETED;
dmorley's avatar
dmorley committed
399
    }
dmorley's avatar
dmorley committed
400

dmorley's avatar
dmorley committed
401
    debug('Score', $score);
dmorley's avatar
dmorley committed
402
    debug('Status', $status);
dmorley's avatar
dmorley committed
403
    debug('Weighted Score', $weightedscore);
noplanman's avatar
noplanman committed
404

405 406 407 408 409 410 411 412
    try {
        $p                     = R::findOne('pods', 'domain = ?', [$domain]);
        $p['ip']               = $ip;
        $p['ipv6']             = $ipv6;
        $p['daysmonitored']    = $days;
        $p['monthsmonitored']  = $months;
        $p['uptime_alltime']   = $uptime;
        $p['status']           = $status;
dmorley's avatar
dmorley committed
413 414
        $p['date_laststats']   = date('Y-m-d H:i:s.u');
        $p['date_updated']     = $lastpodupdates['date_checked'];
415 416 417 418 419 420 421 422 423 424 425 426 427 428
        $p['latency']          = $avglatency;
        $p['score']            = $score;
        $p['country']          = $country;
        $p['countryname']      = $countryname;
        $p['city']             = $city;
        $p['state']            = $state;
        $p['lat']              = $lat;
        $p['long']             = $long;
        $p['userrating']       = $user_rating;
        $p['masterversion']    = $masterversion;
        $p['weightedscore']    = $weightedscore;
        $p['sslvalid']         = $outputsslerror;
        $p['dnssec']           = $dnssec;
        $p['sslexpire']        = $sslexpire;
dmorley's avatar
dmorley committed
429
        if ($dbstatus == PodStatus::UP && $status == PodStatus::UP) {
430 431 432 433 434 435 436
            $p['shortversion']          = $shortversion;
            $p['signup']                = $signup;
            $p['total_users']           = $total_users;
            $p['active_users_halfyear'] = $active_users_halfyear;
            $p['active_users_monthly']  = $active_users_monthly;
            $p['local_posts']           = $local_posts;
            $p['name']                  = $name;
dmorley's avatar
dmorley committed
437
            $p['detectedlanguage']      = $detectedlanguage;
438 439
            $p['comment_counts']        = $comment_counts;
            $p['service_xmpp']          = $service_xmpp;
440
            $p['services']              = $services;
dmorley's avatar
dmorley committed
441
            $p['protocols']             = $protocols;
442 443 444 445 446 447
            $p['softwarename']          = $softwarename;
        }

        if ($write) {
            R::store($p);
        } else {
dmorley's avatar
dmorley committed
448 449
            echo 'Data not saved, testing only';
            echo $newline;
450 451
        }
    } catch (\RedBeanPHP\RedException $e) {
dmorley's avatar
dmorley committed
452 453 454 455 456 457
        debug('Error in SQL query: ' . sprintf($e->getMessage()));
    }

    debug('Success', $domain);
}

dmorley's avatar
dmorley committed
458
//if a full run
dmorley's avatar
dmorley committed
459
if (!$_domain) {
dmorley's avatar
dmorley committed
460 461 462 463
    updateMeta('pods_updated');
    $time_end = microtime(true);
    $execution_time = ($time_end - $time_start) / 60;
    updateMeta('pods_update_runtime', round($execution_time));
464
    if ($languagesupdated || $init) {
dmorley's avatar
dmorley committed
465 466 467
        updateMeta('languages_updated');
    }

468
    if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('masterversions_updated', 'date_created'))->diffInHours() > 12 || $init) {
dmorley's avatar
dmorley committed
469 470 471
        require __DIR__ . '/update-masterversions.php';
    }

472
    if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('statstable_updated', 'date_created'))->diffInHours() > 23 || $init) {
dmorley's avatar
dmorley committed
473 474 475
        require __DIR__ . '/update-monthly-stats.php';
    }

476
    if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('cacert_updated', 'date_created'))->diffInHours() > 40 || $init) {
dmorley's avatar
dmorley committed
477
        copy('https://curl.haxx.se/ca/cacert.pem', c('cafullpath'));
dmorley's avatar
dmorley committed
478 479 480
        updateMeta('cacert_updated');
    }

481
    if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('sitemap_updated', 'date_created'))->diffInHours() > 2 || $init) {
dmorley's avatar
dmorley committed
482 483 484
        require __DIR__ . '/update-sitemap.php';
    }

485
    if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('geoip_updated', 'date_created'))->diffInHours() > 45 || $init) {
dmorley's avatar
dmorley committed
486
        copy('compress.zlib://http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz', c('geoip2db'));
dmorley's avatar
dmorley committed
487 488 489
        updateMeta('geoip_updated');
    }

490
    if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('federation_updated', 'date_created'))->diffInHours() > 9 || $init) {
dmorley's avatar
dmorley committed
491
        require __DIR__ . '/update-remote-data.php';
492
    }
David Morley's avatar
David Morley committed
493

dmorley's avatar
dmorley committed
494
    require __DIR__ . '/backup.php';
noplanman's avatar
noplanman committed
495

dmorley's avatar
dmorley committed
496
    updateMeta('pods_updating', false);
497
}