pull.php 11.7 KB
Newer Older
MatrixCrawler's avatar
MatrixCrawler committed
1
<?php
2
//* Copyright (c) 2011, David Morley. This file is licensed under the Affero General Public License version 3 or later. See the COPYRIGHT file. */
3

noplanman's avatar
noplanman committed
4 5
use RedBeanPHP\R;

David Morley's avatar
David Morley committed
6 7 8
$debug   = isset($_GET['debug']) || (isset($argv) && in_array('debug', $argv, true));
$newline = PHP_SAPI === 'cli' ? "\n" : '<br>';

noplanman's avatar
noplanman committed
9 10 11 12
$_domain = $_GET['domain'] ?? null;

// Must have a domain, except if called from CLI.
$_domain || PHP_SAPI === 'cli' || die('No valid input');
13

David Morley's avatar
David Morley committed
14
require_once __DIR__ . '/../vendor/autoload.php';
noplanman's avatar
noplanman committed
15 16 17
require_once __DIR__ . '/../config.php';

define('PODUPTIME', microtime(true));
dmorley's avatar
cleanup  
dmorley committed
18

noplanman's avatar
noplanman committed
19 20 21
// Set up global DB connection.
R::setup("pgsql:host={$pghost};dbname={$pgdb}", $pguser, $pgpass, true);
R::testConnection() || die('Error in DB connection');
dmorley's avatar
dmorley committed
22
R::usePartialBeans(true);
noplanman's avatar
noplanman committed
23 24 25

try {
  $sql = '
dmorley's avatar
dmorley committed
26
    SELECT domain, score, date_created, adminrating, weight, hidden, podmin_notify, email, masterversion, shortversion, status
noplanman's avatar
noplanman committed
27 28 29 30 31 32 33 34
    FROM pods
  ';

  $pods = [];
  if ($_domain) {
    $sql .= ' WHERE domain = ?';
    $pods = R::getAll($sql, [$_domain]);
  } elseif (PHP_SAPI === 'cli') {
dmorley's avatar
dmorley committed
35 36
    $sql .= ' WHERE status < ?';
    $pods = R::getAll($sql, [PodStatus::Paused]);
noplanman's avatar
noplanman committed
37 38 39
  }
} catch (\RedBeanPHP\RedException $e) {
  die('Error in SQL query: ' . $e->getMessage());
dmorley's avatar
cleanup  
dmorley committed
40
}
noplanman's avatar
noplanman committed
41 42 43 44 45 46 47 48 49 50

foreach ($pods as $pod) {
  $domain    = $pod['domain'];
  $score     = (int) $pod['score'];
  $dateadded = $pod['date_created'];
  $admindb   = (int) $pod['adminrating'];
  $weight    = $pod['weight'];
  $hiddennow = $pod['hidden'];
  $email     = $pod['email'];
  $notify    = $pod['podmin_notify'];
dmorley's avatar
dmorley committed
51 52
  $masterv   = $pod['masterversion'];
  $shortv    = $pod['shortversion'];
dmorley's avatar
dmorley committed
53
  $dbstatus  = $pod['status'];
noplanman's avatar
noplanman committed
54 55 56 57 58 59 60 61 62 63

  try {
    $ratings = R::getAll('
      SELECT admin, rating
      FROM rating_comments
      WHERE domain = ?
    ', [$domain]);
  } catch (\RedBeanPHP\RedException $e) {
    die('Error in SQL query: ' . $e->getMessage());
  }
64

David Morley's avatar
David Morley committed
65
  _debug('Domain', $domain);
noplanman's avatar
noplanman committed
66

David Morley's avatar
David Morley committed
67 68
  $user_ratings  = [];
  $admin_ratings = [];
noplanman's avatar
noplanman committed
69
  foreach ($ratings as $rating) {
David Morley's avatar
David Morley committed
70 71 72 73
    if ($rating['admin'] == 0) {
      $user_ratings[] = $rating['rating'];
    } elseif ($rating['admin'] == 1) {
      $admin_ratings[] = $rating['rating'];
74
    }
David Morley's avatar
David Morley committed
75 76 77
  }
  $user_rating  = empty($user_ratings) ? 0 : max(10, round(array_sum($user_ratings) / count($user_ratings), 2));
  $admin_rating = empty($admin_ratings) ? 0 : max(10, round(array_sum($admin_ratings) / count($admin_ratings), 2));
noplanman's avatar
noplanman committed
78

David Morley's avatar
David Morley committed
79 80 81
  if ($admindb == -1) {
    $admin_rating = -1;
  }
82

David Morley's avatar
David Morley committed
83 84
  $chss = curl_init();
  curl_setopt($chss, CURLOPT_URL, 'https://' . $domain . '/nodeinfo/1.0');
85 86
  curl_setopt($chss, CURLOPT_CONNECTTIMEOUT, 10);
  curl_setopt($chss, CURLOPT_TIMEOUT, 30);
David Morley's avatar
David Morley committed
87 88
  curl_setopt($chss, CURLOPT_RETURNTRANSFER, 1);
  curl_setopt($chss, CURLOPT_CERTINFO, 1);
dmorley's avatar
latent  
dmorley committed
89
  curl_setopt($chss, CURLOPT_CAINFO, $cafullpath);
David Morley's avatar
David Morley committed
90 91 92
  $outputssl      = curl_exec($chss);
  $outputsslerror = curl_error($chss);
  $info           = curl_getinfo($chss, CURLINFO_CERTINFO);
93 94 95
  $conntime       = curl_getinfo($chss, CURLINFO_CONNECT_TIME);
  $nstime         = curl_getinfo($chss, CURLINFO_NAMELOOKUP_TIME);
  $latency        = $conntime - $nstime;
dmorley's avatar
dmorley committed
96
  $sslexpire      = $info[0]['Expire date'] ?? null;
David Morley's avatar
David Morley committed
97
  curl_close($chss);
98

David Morley's avatar
David Morley committed
99 100
  _debug('Nodeinfo output', $outputssl, true);
  _debug('Nodeinfo output error', $outputsslerror, true);
101
  _debug('Cert expire date', $sslexpire);
102 103
  _debug('Conntime', $conntime);
  _debug('NStime', $nstime);
dmorley's avatar
latent  
dmorley committed
104
  _debug('Latency', $latency);
noplanman's avatar
noplanman committed
105

David Morley's avatar
David Morley committed
106
  $jsonssl = json_decode($outputssl);
noplanman's avatar
noplanman committed
107

108 109 110 111 112
  $xdver                 = $jsonssl->software->version ?? 0;
  $dverr                 = explode('-', trim($xdver));
  $shortversion          = $dverr[0];
  $signup                = ($jsonssl->openRegistrations ?? false) === true;
  $softwarename          = $jsonssl->software->name ?? 'unknown';
David Morley's avatar
David Morley committed
113
  $name                  = $jsonssl->metadata->nodeName ?? $softwarename;
114 115 116 117 118 119
  $total_users           = $jsonssl->usage->users->total ?? 0;
  $active_users_halfyear = $jsonssl->usage->users->activeHalfyear ?? 0;
  $active_users_monthly  = $jsonssl->usage->users->activeMonth ?? 0;
  $local_posts           = $jsonssl->usage->localPosts ?? 0;
  $comment_counts        = $jsonssl->usage->localComments ?? 0;
  $service_xmpp          = ($jsonssl->metadata->xmppChat ?? false) === true;
dmorley's avatar
dum  
dmorley committed
120 121 122 123
  $service_facebook      = false;
  $service_twitter       = false;
  $service_tumblr        = false;
  $service_wordpress     = false;
124
  if (json_last_error() === 0) {
dmorley's avatar
dmorley committed
125
    (!$jsonssl->software->version) || $score += 1;
dmorley's avatar
dmorley committed
126 127 128 129 130 131
    if (is_array($jsonssl->services->outbound)) {
      $service_facebook  = in_array('facebook', $jsonssl->services->outbound, true);
      $service_twitter   = in_array('twitter', $jsonssl->services->outbound, true);
      $service_tumblr    = in_array('tumblr', $jsonssl->services->outbound, true);
      $service_wordpress = in_array('wordpress', $jsonssl->services->outbound, true);
      }
132
  }
noplanman's avatar
noplanman committed
133

134
  if ($jsonssl !== null) {
noplanman's avatar
noplanman committed
135 136 137 138 139 140 141 142 143 144 145 146 147 148

    try {
      $c                   = R::dispense('checks');
      $c['domain']         = $domain;
      $c['online']         = true;
      $c['latency']        = $latency;
      $c['total_users']    = $total_users;
      $c['local_posts']    = $local_posts;
      $c['comment_counts'] = $comment_counts;
      $c['shortversion']   = $shortversion;
      R::store($c);
    } catch (\RedBeanPHP\RedException $e) {
      die('Error in SQL query: ' . $e->getMessage());
    }
dmorley's avatar
dmorley committed
149 150
    
    $status = PodStatus::Up;
David Morley's avatar
David Morley committed
151
  }
noplanman's avatar
noplanman committed
152 153

  if (!$jsonssl) {
154
    _debug('Connection', 'Can not connect to pod');
noplanman's avatar
noplanman committed
155 156 157 158 159 160 161 162 163 164 165 166 167

    try {
      $c            = R::dispense('checks');
      $c['domain']  = $domain;
      $c['online']  = false;
      $c['error']   = $outputsslerror;
      $c['latency'] = $latency;
      R::store($c);
    } catch (\RedBeanPHP\RedException $e) {
      die('Error in SQL query: ' . $e->getMessage());
    }

    $score        -= 1;
dmorley's avatar
dmorley committed
168
    $status       = PodStatus::Down;
noplanman's avatar
noplanman committed
169 170
  }

171
  _debug('Version code', $shortversion);
David Morley's avatar
David Morley committed
172
  _debug('Signup Open', $signup);
David Morley's avatar
David Morley committed
173

dmorley's avatar
dmorley committed
174
  $dnsserver = !empty($dnsserver) ? $dnsserver : '1.1.1.1';
David Morley's avatar
David Morley committed
175 176 177
  $delv = new NPM\Xec\Command("delv @{$dnsserver} {$domain}");
  $delv->throwExceptionOnError(false);

178
  $ip         = '';
David Morley's avatar
David Morley committed
179 180 181
  $iplookupv4 = explode(PHP_EOL, trim($delv->execute([], null, 15)->stdout));
  $dnssec     = in_array('; fully validated', $iplookupv4, true) ?? false;
  $getaonly   = array_values(preg_grep('/\s+IN\s+A\s+.*/', $iplookupv4));
182
  if ($getaonly) {
183
    preg_match('/A\s(.*)/', $getaonly[0], $aversion);
184
    $ip = trim($aversion[1]) ?? '';
185
  }
dmorley's avatar
dmorley committed
186
  $ip || $score -= 2;
David Morley's avatar
David Morley committed
187

dmorley's avatar
dmorley committed
188 189 190
  $iplookupv6 = explode(PHP_EOL, trim($delv->execute(['AAAA'], null, 15)->stdout));
  $ipv6       = (bool) preg_grep('/\s+IN\s+AAAA\s+.*/', $iplookupv6);

David Morley's avatar
David Morley committed
191
  _debug('IPv4', $ip);
192
  _debug('Iplookupv4', $iplookupv4, true);
David Morley's avatar
David Morley committed
193
  _debug('IPv6', $ipv6);
194
  _debug('Iplookupv6', $iplookupv6, true);
David Morley's avatar
David Morley committed
195 196 197

  $location = geoip_record_by_name($ip);
  _debug('Location', $location, true);
198 199 200 201 202
  $country  = !empty($location['country_code']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['country_code']) : null;
  $city     = !empty($location['city']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['city']) : null;
  $state    = !empty($location['region']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['region']) : null;
  $lat      = !empty($location['latitude']) ? $location['latitude'] : 0;
  $long     = !empty($location['longitude']) ? $location['longitude'] : 0;
noplanman's avatar
noplanman committed
203

David Morley's avatar
David Morley committed
204 205
  echo $newline;
  $statslastdate = date('Y-m-d H:i:s');
David Morley's avatar
David Morley committed
206

noplanman's avatar
noplanman committed
207 208 209 210 211 212 213 214 215 216
  $diff   = (new DateTime())->diff(new DateTime($dateadded));
  $months = $diff->m + ($diff->y * 12);

  try {
    $checks = R::getRow('
      SELECT
        round(avg(latency) * 1000) AS latency,
        round(avg(online::INT) * 100, 2) AS online
      FROM checks
      WHERE domain = ?
217
    ', [$domain]);
noplanman's avatar
noplanman committed
218 219 220 221 222 223 224

    $avglatency = $checks['latency'] ?? 0;
    $uptime     = $checks['online'] ?? 0;
  } catch (\RedBeanPHP\RedException $e) {
    die('Error in SQL query: ' . $e->getMessage());
  }

225
  _debug('Uptime', $uptime);
David Morley's avatar
David Morley committed
226

noplanman's avatar
noplanman committed
227
  try {
dmorley's avatar
dmorley committed
228
    $masterversion = R::getCell('SELECT version FROM masterversions WHERE software = ? ORDER BY id DESC LIMIT 1', [$softwarename]);
noplanman's avatar
noplanman committed
229 230 231
  } catch (\RedBeanPHP\RedException $e) {
    die('Error in SQL query: ' . $e->getMessage());
  }
David Morley's avatar
David Morley committed
232 233

  _debug('Masterversion', $masterversion);
dmorley's avatar
dmorley committed
234 235
  $masterversioncheck = explode('.',$masterversion);
  $shortversioncheck = explode('.',$shortversion);
236
  if (($masterversioncheck[1] - $shortversioncheck[1]) > 1 && strpos($xdver,'dev') === false) {
dmorley's avatar
dmorley committed
237
    //dev search added to address frendica using very odd versioning for dev code, we should look to pull dev versions and use them rather than assume dev is always ahead of prod code
dmorley's avatar
dmorley committed
238 239
    _debug('Outdated', 'Yes');$score -= 2;
  }
noplanman's avatar
noplanman committed
240

David Morley's avatar
David Morley committed
241 242
  $hidden = $score <= 70;
  _debug('Hidden', $hidden ? 'yes' : 'no');
243

noplanman's avatar
noplanman committed
244
  if (!$hiddennow && $hidden && $notify) {
245 246 247 248 249
    $to      = $email;
    $headers = ['From: ' . $adminemail, 'Bcc: ' . $adminemail];
    $subject = 'Monitoring notice from poduptime';
    $message = 'Notice for ' . $domain . '. Your score fell to ' . $score . ' and your pod is now marked as hidden.';
    @mail($to, $subject, $message, implode("\r\n", $headers));
noplanman's avatar
noplanman committed
250
    _debug('Mail Notice', 'sent to ' . $email);
251
  }
David Morley's avatar
David Morley committed
252 253 254 255
  if ($score > 100) {
    $score = 100;
  } elseif ($score < 0) {
    $score = 0;
dmorley's avatar
dmorley committed
256 257 258
    if ($masterv <> $shortv) {
      $status = PodStatus::System_Deleted;
    }
David Morley's avatar
David Morley committed
259
  }
David Morley's avatar
David Morley committed
260
  _debug('Score', $score);
261 262
  $weightedscore = ($uptime + $score - (10 - $weight)) / 2;
  _debug('Weighted Score', $weightedscore);
noplanman's avatar
noplanman committed
263

noplanman's avatar
noplanman committed
264 265 266 267 268
  try {
    $p                          = R::findOne('pods', 'domain = ?', [$domain]);
    $p['secure']                = true;
    $p['hidden']                = $hidden;
    $p['ip']                    = $ip;
dmorley's avatar
ipv6  
dmorley committed
269
    $p['ipv6']                  = $ipv6;
noplanman's avatar
noplanman committed
270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288
    $p['monthsmonitored']       = $months;
    $p['uptime_alltime']        = $uptime;
    $p['status']                = $status;
    $p['date_laststats']        = $statslastdate;
    $p['date_updated']          = date('Y-m-d H:i:s');
    $p['latency']               = $avglatency;
    $p['score']                 = $score;
    $p['adminrating']           = $admin_rating;
    $p['country']               = $country;
    $p['city']                  = $city;
    $p['state']                 = $state;
    $p['lat']                   = $lat;
    $p['long']                  = $long;
    $p['userrating']            = $user_rating;
    $p['masterversion']         = $masterversion;
    $p['weightedscore']         = $weightedscore;
    $p['sslvalid']              = $outputsslerror;
    $p['dnssec']                = $dnssec;
    $p['sslexpire']             = $sslexpire;
dmorley's avatar
fix AND  
dmorley committed
289
    if ($dbstatus == PodStatus::Up && $status == PodStatus::Up) {
dmorley's avatar
dmorley committed
290 291 292 293 294 295 296 297 298 299 300 301 302 303 304
      $p['shortversion']          = $shortversion;
      $p['signup']                = $signup;
      $p['total_users']           = $total_users;
      $p['active_users_halfyear'] = $active_users_halfyear;
      $p['active_users_monthly']  = $active_users_monthly;
      $p['local_posts']           = $local_posts;
      $p['name']                  = $name;
      $p['comment_counts']        = $comment_counts;
      $p['service_facebook']      = $service_facebook;
      $p['service_tumblr']        = $service_tumblr;
      $p['service_twitter']       = $service_twitter;
      $p['service_wordpress']     = $service_wordpress;
      $p['service_xmpp']          = $service_xmpp;
      $p['softwarename']          = $softwarename;
    }
noplanman's avatar
noplanman committed
305 306 307 308 309

    R::store($p);
  } catch (\RedBeanPHP\RedException $e) {
    die('Error in SQL query: ' . $e->getMessage());
  }
David Morley's avatar
David Morley committed
310

noplanman's avatar
noplanman committed
311 312
  echo 'Success ' . $domain;

David Morley's avatar
David Morley committed
313 314 315
  echo $newline;
  echo $newline;
}
316

David Morley's avatar
David Morley committed
317 318
/**
 * Output a debug message and variable value
noplanman's avatar
noplanman committed
319
 *
David Morley's avatar
David Morley committed
320 321 322 323
 * @param string $label
 * @param mixed  $var
 * @param bool   $dump
 */
noplanman's avatar
noplanman committed
324 325
function _debug($label, $var = null, $dump = false)
{
David Morley's avatar
David Morley committed
326
  global $debug, $newline;
327

David Morley's avatar
David Morley committed
328 329
  if (!$debug) {
    return;
dmorley's avatar
cleanup  
dmorley committed
330
  }
noplanman's avatar
noplanman committed
331

David Morley's avatar
David Morley committed
332
  if ($dump || is_array($var)) {
noplanman's avatar
noplanman committed
333
    $output = print_r($var, true);
David Morley's avatar
David Morley committed
334 335 336 337 338
  } elseif (is_bool($var)) {
    $output = $var ? 'true' : 'false';
  } else {
    $output = (string) $var;
  }
noplanman's avatar
noplanman committed
339

David Morley's avatar
David Morley committed
340
  printf('%s: %s%s', $label, $output, $newline);
341
}