pull.php 14.7 KB
Newer Older
MatrixCrawler's avatar
MatrixCrawler committed
1
<?php
2
//* Copyright (c) 2011, David Morley. This file is licensed under the Affero General Public License version 3 or later. See the COPYRIGHT file. */
3

4 5 6 7 8
if ($_SERVER['SERVER_ADDR'] !== $_SERVER['REMOTE_ADDR']) {
  header('HTTP/1.0 403 Forbidden');
  exit;
}

9 10
use RedBeanPHP\R;

11 12 13 14
$debug     = isset($_GET['debug']) || (isset($argv) && in_array('debug', $argv, true));
$sqldebug  = isset($_GET['sqldebug']) || (isset($argv) && in_array('sqldebug', $argv, true));
$write     = !(isset($_GET['nowrite']) || (isset($argv) && in_array('nowrite', $argv, true)));
$newline   = PHP_SAPI === 'cli' ? "\n\n" : '<br><br>';
David Morley's avatar
David Morley committed
15

16 17 18 19
$_domain = $_GET['domain'] ?? null;

// Must have a domain, except if called from CLI.
$_domain || PHP_SAPI === 'cli' || die('No valid input');
20

David Morley's avatar
David Morley committed
21
require_once __DIR__ . '/../vendor/autoload.php';
22 23 24
require_once __DIR__ . '/../config.php';

define('PODUPTIME', microtime(true));
David Morley's avatar
David Morley committed
25

26 27
// Set up global DB connection.
R::setup("pgsql:host={$pghost};dbname={$pgdb}", $pguser, $pgpass, true);
28
$sqldebug && R::fancyDebug(true);
29
R::testConnection() || die('Error in DB connection');
David Morley's avatar
David Morley committed
30
R::usePartialBeans(true);
31 32 33

try {
  $sql = '
David Morley's avatar
David Morley committed
34
    SELECT domain, score, date_created, adminrating, weight, hidden, podmin_notify, email, masterversion, shortversion, status
35 36 37 38 39 40 41
    FROM pods
  ';

  $pods = [];
  if ($_domain) {
    $sql .= ' WHERE domain = ?';
    $pods = R::getAll($sql, [$_domain]);
42 43 44
  } elseif (PHP_SAPI === 'cli' && (isset($argv) && in_array('Check_System_Deleted', $argv, true))) {
    $sql .= ' WHERE status = ?';
    $pods = R::getAll($sql, [PodStatus::System_Deleted]);
45
  } elseif (PHP_SAPI === 'cli') {
David Morley's avatar
David Morley committed
46 47
    $sql .= ' WHERE status < ?';
    $pods = R::getAll($sql, [PodStatus::Paused]);
48 49 50
  }
} catch (\RedBeanPHP\RedException $e) {
  die('Error in SQL query: ' . $e->getMessage());
David Morley's avatar
David Morley committed
51
}
52 53 54 55 56 57 58 59 60 61

foreach ($pods as $pod) {
  $domain    = $pod['domain'];
  $score     = (int) $pod['score'];
  $dateadded = $pod['date_created'];
  $admindb   = (int) $pod['adminrating'];
  $weight    = $pod['weight'];
  $hiddennow = $pod['hidden'];
  $email     = $pod['email'];
  $notify    = $pod['podmin_notify'];
David Morley's avatar
David Morley committed
62 63
  $masterv   = $pod['masterversion'];
  $shortv    = $pod['shortversion'];
David Morley's avatar
David Morley committed
64
  $dbstatus  = $pod['status'];
65 66 67 68 69 70 71 72 73 74

  try {
    $ratings = R::getAll('
      SELECT admin, rating
      FROM rating_comments
      WHERE domain = ?
    ', [$domain]);
  } catch (\RedBeanPHP\RedException $e) {
    die('Error in SQL query: ' . $e->getMessage());
  }
75

David Morley's avatar
David Morley committed
76
  _debug('Domain', $domain);
77

David Morley's avatar
David Morley committed
78 79
  $user_ratings  = [];
  $admin_ratings = [];
80
  foreach ($ratings as $rating) {
David Morley's avatar
David Morley committed
81 82 83 84
    if ($rating['admin'] == 0) {
      $user_ratings[] = $rating['rating'];
    } elseif ($rating['admin'] == 1) {
      $admin_ratings[] = $rating['rating'];
85
    }
David Morley's avatar
David Morley committed
86 87 88
  }
  $user_rating  = empty($user_ratings) ? 0 : max(10, round(array_sum($user_ratings) / count($user_ratings), 2));
  $admin_rating = empty($admin_ratings) ? 0 : max(10, round(array_sum($admin_ratings) / count($admin_ratings), 2));
89

David Morley's avatar
David Morley committed
90 91 92
  if ($admindb == -1) {
    $admin_rating = -1;
  }
93

94 95 96 97 98 99
  if ($infos = json_decode(file_get_contents('https://' . $domain . '/.well-known/nodeinfo'), true)) {
    $link = max($infos['links'])['href'];
  } else {
    $link = 'https://' . $domain . '/nodeinfo/1.0';
  }

David Morley's avatar
David Morley committed
100
  $chss = curl_init();
101
  curl_setopt($chss, CURLOPT_URL, $link);
102 103
  curl_setopt($chss, CURLOPT_CONNECTTIMEOUT, 10);
  curl_setopt($chss, CURLOPT_TIMEOUT, 30);
David Morley's avatar
David Morley committed
104 105
  curl_setopt($chss, CURLOPT_RETURNTRANSFER, 1);
  curl_setopt($chss, CURLOPT_CERTINFO, 1);
dmorley's avatar
dmorley committed
106
  curl_setopt($chss, CURLOPT_CAINFO, $cafullpath);
David Morley's avatar
David Morley committed
107 108 109
  $outputssl      = curl_exec($chss);
  $outputsslerror = curl_error($chss);
  $info           = curl_getinfo($chss, CURLINFO_CERTINFO);
110 111 112
  $conntime       = curl_getinfo($chss, CURLINFO_CONNECT_TIME);
  $nstime         = curl_getinfo($chss, CURLINFO_NAMELOOKUP_TIME);
  $latency        = $conntime - $nstime;
dmorley's avatar
dmorley committed
113
  $sslexpire      = $info[0]['Expire date'] ?? null;
David Morley's avatar
David Morley committed
114
  curl_close($chss);
115

David Morley's avatar
David Morley committed
116 117
  _debug('Nodeinfo output', $outputssl, true);
  _debug('Nodeinfo output error', $outputsslerror, true);
118
  _debug('Cert expire date', $sslexpire);
119 120
  _debug('Conntime', $conntime);
  _debug('NStime', $nstime);
dmorley's avatar
dmorley committed
121
  _debug('Latency', $latency);
122

David Morley's avatar
David Morley committed
123
  $jsonssl = json_decode($outputssl);
124

125
  $xdver                 = $jsonssl->software->version ?? 0;
126 127
  preg_match_all('((?:\d(.|-)?)+(\.|-)\d+\.*)', $xdver, $dverr);
  $shortversion          = $dverr[0][0];
128 129
  $signup                = ($jsonssl->openRegistrations ?? false) === true;
  $softwarename          = $jsonssl->software->name ?? 'unknown';
David Morley's avatar
David Morley committed
130
  $name                  = $jsonssl->metadata->nodeName ?? $softwarename;
131 132 133 134 135 136
  $total_users           = $jsonssl->usage->users->total ?? 0;
  $active_users_halfyear = $jsonssl->usage->users->activeHalfyear ?? 0;
  $active_users_monthly  = $jsonssl->usage->users->activeMonth ?? 0;
  $local_posts           = $jsonssl->usage->localPosts ?? 0;
  $comment_counts        = $jsonssl->usage->localComments ?? 0;
  $service_xmpp          = ($jsonssl->metadata->xmppChat ?? false) === true;
dmorley's avatar
dmorley committed
137 138 139 140
  $service_facebook      = false;
  $service_twitter       = false;
  $service_tumblr        = false;
  $service_wordpress     = false;
141
  if (json_last_error() === 0) {
dmorley's avatar
dmorley committed
142
    (!$jsonssl->software->version) || $score += 1;
David Morley's avatar
David Morley committed
143 144 145 146 147
    if (is_array($jsonssl->services->outbound)) {
      $service_facebook  = in_array('facebook', $jsonssl->services->outbound, true);
      $service_twitter   = in_array('twitter', $jsonssl->services->outbound, true);
      $service_tumblr    = in_array('tumblr', $jsonssl->services->outbound, true);
      $service_wordpress = in_array('wordpress', $jsonssl->services->outbound, true);
148
    }
149
  }
150

151
  if ($jsonssl !== null) {
152 153 154 155 156 157 158 159 160 161

    try {
      $c                   = R::dispense('checks');
      $c['domain']         = $domain;
      $c['online']         = true;
      $c['latency']        = $latency;
      $c['total_users']    = $total_users;
      $c['local_posts']    = $local_posts;
      $c['comment_counts'] = $comment_counts;
      $c['shortversion']   = $shortversion;
162 163 164 165 166
      if ($write) {
        R::store($c);
      } else {
        echo $c;
      }
167 168 169
    } catch (\RedBeanPHP\RedException $e) {
      die('Error in SQL query: ' . $e->getMessage());
    }
170

David Morley's avatar
David Morley committed
171
    $status = PodStatus::Up;
David Morley's avatar
David Morley committed
172
  }
173 174

  if (!$jsonssl) {
175
    _debug('Connection', 'Can not connect to pod');
176 177 178 179 180 181 182

    try {
      $c            = R::dispense('checks');
      $c['domain']  = $domain;
      $c['online']  = false;
      $c['error']   = $outputsslerror;
      $c['latency'] = $latency;
183 184 185 186 187
      if ($write) {
        R::store($c);
      } else {
        echo $c;
      }
188 189 190 191 192
    } catch (\RedBeanPHP\RedException $e) {
      die('Error in SQL query: ' . $e->getMessage());
    }

    $score        -= 1;
David Morley's avatar
David Morley committed
193
    $status       = PodStatus::Down;
194 195
  }

196
  _debug('Version code', $shortversion);
David Morley's avatar
David Morley committed
197
  _debug('Signup Open', $signup);
David Morley's avatar
David Morley committed
198

199
  $dnsserver = !empty($dnsserver) ? $dnsserver : '1.1.1.1';
David Morley's avatar
David Morley committed
200 201 202
  $delv = new NPM\Xec\Command("delv @{$dnsserver} {$domain}");
  $delv->throwExceptionOnError(false);

203
  $ip         = '';
David Morley's avatar
David Morley committed
204 205 206
  $iplookupv4 = explode(PHP_EOL, trim($delv->execute([], null, 15)->stdout));
  $dnssec     = in_array('; fully validated', $iplookupv4, true) ?? false;
  $getaonly   = array_values(preg_grep('/\s+IN\s+A\s+.*/', $iplookupv4));
207
  if ($getaonly) {
208
    preg_match('/A\s(.*)/', $getaonly[0], $aversion);
209
    $ip = trim($aversion[1]) ?? '';
210
  }
dmorley's avatar
dmorley committed
211
  $ip || $score -= 2;
David Morley's avatar
David Morley committed
212

David Morley's avatar
David Morley committed
213 214 215
  $iplookupv6 = explode(PHP_EOL, trim($delv->execute(['AAAA'], null, 15)->stdout));
  $ipv6       = (bool) preg_grep('/\s+IN\s+AAAA\s+.*/', $iplookupv6);

David Morley's avatar
David Morley committed
216
  _debug('IPv4', $ip);
217
  _debug('Iplookupv4', $iplookupv4, true);
David Morley's avatar
David Morley committed
218
  _debug('IPv6', $ipv6);
219
  _debug('Iplookupv6', $iplookupv6, true);
David Morley's avatar
David Morley committed
220 221 222

  $location = geoip_record_by_name($ip);
  _debug('Location', $location, true);
223 224 225 226 227 228
  $countryname  = !empty($location['country_name']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['country_name']) : null;
  $country      = !empty($location['country_code']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['country_code']) : null;
  $city         = !empty($location['city']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['city']) : null;
  $state        = !empty($location['region']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['region']) : null;
  $lat          = !empty($location['latitude']) ? $location['latitude'] : 0;
  $long         = !empty($location['longitude']) ? $location['longitude'] : 0;
229

David Morley's avatar
David Morley committed
230 231
  echo $newline;
  $statslastdate = date('Y-m-d H:i:s');
David Morley's avatar
David Morley committed
232

233 234
  $diff   = (new DateTime())->diff(new DateTime($dateadded));
  $months = $diff->m + ($diff->y * 12);
235
  $days = $diff->days;
236 237 238 239 240 241 242 243

  try {
    $checks = R::getRow('
      SELECT
        round(avg(latency) * 1000) AS latency,
        round(avg(online::INT) * 100, 2) AS online
      FROM checks
      WHERE domain = ?
244
    ', [$domain]);
245 246 247 248 249 250 251

    $avglatency = $checks['latency'] ?? 0;
    $uptime     = $checks['online'] ?? 0;
  } catch (\RedBeanPHP\RedException $e) {
    die('Error in SQL query: ' . $e->getMessage());
  }

252
  _debug('Uptime', $uptime);
David Morley's avatar
David Morley committed
253

254
  try {
255
    $masterdata = R::getRow('SELECT version, devlastcommit, releasedate FROM masterversions WHERE software = ? ORDER BY id DESC LIMIT 1', [$softwarename]);
256 257 258
  } catch (\RedBeanPHP\RedException $e) {
    die('Error in SQL query: ' . $e->getMessage());
  }
David Morley's avatar
David Morley committed
259

260
  $masterversion = $masterdata['version'];
David Morley's avatar
David Morley committed
261
  _debug('Masterversion', $masterversion);
David Morley's avatar
David Morley committed
262 263
  $masterversioncheck = explode('.',$masterversion);
  $shortversioncheck = explode('.',$shortversion);
264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288

  _debug('Days since master code release', date_diff((new DateTime($masterdata['releasedate'])),(new DateTime()))->format('%d'));


    try {
      $lastpodupdates = R::getRow('SELECT DISTINCT ON (shortversion) shortversion, date_checked FROM checks WHERE domain = ? AND shortversion IS NOT NULL ORDER BY shortversion DESC LIMIT 1', [$domain]);
    } catch (\RedBeanPHP\RedException $e) {
      die('Error in SQL query: ' . $e->getMessage());
    }
   
    $devlastdays = $masterdata['devlastcommit'] ? date_diff((new DateTime($masterdata['devlastcommit'])),(new DateTime()))->format('%a') : 30;//tmp//if no dev branch then what?
    
    
    _debug('Dev git last commit was  ', $devlastdays);
      if (strpos($xdver,'dev') !== false || strpos($xdver,'rc') !== false || $shortversioncheck > $masterversioncheck) {
//tmp//if pod is on the development branch - see when you last updated your pod and when the last commit was made to dev branch - if the repo is active and your not updating every 120 days why are you on dev branch?
    $updategap   = date_diff((new DateTime($lastpodupdates['date_checked'])),(new DateTime($masterdata['devlastcommit'])))->format('%a');
    if ($updategap + $devlastdays > 130) {
      _debug('Outdated', 'Yes');$score -= 2;
    }
  } elseif (($masterversioncheck[1] - $shortversioncheck[1]) > 1) {
///tmp/If pod is two versions off AND it's been more than 60 days since that release came out AND your on the master production branch
    _debug('Outdated', 'Yes');$score -= 2;
    $updategap   = date_diff((new DateTime($lastpodupdates['date_checked'])),(new DateTime($masterdata['releasedate'])))->format('%a');
  } elseif ($updategap - date_diff((new DateTime($masterdata['releasedate'])),(new DateTime()))->format('%a') > 90) {
David Morley's avatar
David Morley committed
289
    _debug('Outdated', 'Yes');$score -= 2;
290 291 292
    $updategap   = date_diff((new DateTime($lastpodupdates['date_checked'])),(new DateTime($masterdata['releasedate'])))->format('%a');
  } else {
    $updategap   = date_diff((new DateTime($lastpodupdates['date_checked'])),(new DateTime($masterdata['releasedate'])))->format('%a');
David Morley's avatar
David Morley committed
293
  }
294 295
  _debug('Pod code was updated after ', $updategap);
  
David Morley's avatar
David Morley committed
296 297
  $hidden = $score <= 70;
  _debug('Hidden', $hidden ? 'yes' : 'no');
298

299
  if (!$hiddennow && $hidden && $notify && !(isset($argv) && in_array('develop', $argv, true))) {
300 301 302 303 304
    $to      = $email;
    $headers = ['From: ' . $adminemail, 'Bcc: ' . $adminemail];
    $subject = 'Monitoring notice from poduptime';
    $message = 'Notice for ' . $domain . '. Your score fell to ' . $score . ' and your pod is now marked as hidden.';
    @mail($to, $subject, $message, implode("\r\n", $headers));
305
    _debug('Mail Notice', 'sent to ' . $email);
306
  }
David Morley's avatar
David Morley committed
307 308 309 310
  if ($score > 100) {
    $score = 100;
  } elseif ($score < 0) {
    $score = 0;
David Morley's avatar
David Morley committed
311 312 313
    if ($masterv <> $shortv) {
      $status = PodStatus::System_Deleted;
    }
David Morley's avatar
David Morley committed
314
  }
David Morley's avatar
David Morley committed
315
  _debug('Score', $score);
316 317
  $weightedscore = ($uptime + $score - (10 - $weight)) / 2;
  _debug('Weighted Score', $weightedscore);
318

319 320 321 322 323
  try {
    $p                          = R::findOne('pods', 'domain = ?', [$domain]);
    $p['secure']                = true;
    $p['hidden']                = $hidden;
    $p['ip']                    = $ip;
dmorley's avatar
dmorley committed
324
    $p['ipv6']                  = $ipv6;
325
    $p['daysmonitored']         = $days;
326 327 328 329 330 331 332 333 334
    $p['monthsmonitored']       = $months;
    $p['uptime_alltime']        = $uptime;
    $p['status']                = $status;
    $p['date_laststats']        = $statslastdate;
    $p['date_updated']          = date('Y-m-d H:i:s');
    $p['latency']               = $avglatency;
    $p['score']                 = $score;
    $p['adminrating']           = $admin_rating;
    $p['country']               = $country;
335
    $p['countryname']           = $countryname;
336 337 338 339 340 341 342 343 344 345
    $p['city']                  = $city;
    $p['state']                 = $state;
    $p['lat']                   = $lat;
    $p['long']                  = $long;
    $p['userrating']            = $user_rating;
    $p['masterversion']         = $masterversion;
    $p['weightedscore']         = $weightedscore;
    $p['sslvalid']              = $outputsslerror;
    $p['dnssec']                = $dnssec;
    $p['sslexpire']             = $sslexpire;
David Morley's avatar
David Morley committed
346
    if ($dbstatus == PodStatus::Up && $status == PodStatus::Up) {
David Morley's avatar
David Morley committed
347 348 349 350 351 352 353 354 355 356 357 358 359 360 361
      $p['shortversion']          = $shortversion;
      $p['signup']                = $signup;
      $p['total_users']           = $total_users;
      $p['active_users_halfyear'] = $active_users_halfyear;
      $p['active_users_monthly']  = $active_users_monthly;
      $p['local_posts']           = $local_posts;
      $p['name']                  = $name;
      $p['comment_counts']        = $comment_counts;
      $p['service_facebook']      = $service_facebook;
      $p['service_tumblr']        = $service_tumblr;
      $p['service_twitter']       = $service_twitter;
      $p['service_wordpress']     = $service_wordpress;
      $p['service_xmpp']          = $service_xmpp;
      $p['softwarename']          = $softwarename;
    }
362

363 364 365 366 367
    if ($write) {
      R::store($p);
    } else {
      echo $p;
    }
368 369 370
  } catch (\RedBeanPHP\RedException $e) {
    die('Error in SQL query: ' . $e->getMessage());
  }
David Morley's avatar
David Morley committed
371

372 373
  echo 'Success ' . $domain;

David Morley's avatar
David Morley committed
374 375 376
  echo $newline;
  echo $newline;
}
377

David Morley's avatar
David Morley committed
378 379
/**
 * Output a debug message and variable value
380
 *
David Morley's avatar
David Morley committed
381 382 383 384
 * @param string $label
 * @param mixed  $var
 * @param bool   $dump
 */
385 386
function _debug($label, $var = null, $dump = false)
{
David Morley's avatar
David Morley committed
387
  global $debug, $newline;
388

David Morley's avatar
David Morley committed
389 390
  if (!$debug) {
    return;
David Morley's avatar
David Morley committed
391
  }
392

David Morley's avatar
David Morley committed
393
  if ($dump || is_array($var)) {
394
    $output = print_r($var, true);
David Morley's avatar
David Morley committed
395 396 397 398 399
  } elseif (is_bool($var)) {
    $output = $var ? 'true' : 'false';
  } else {
    $output = (string) $var;
  }
400

David Morley's avatar
David Morley committed
401
  printf('%s: %s%s', $label, $output, $newline);
402
}