pull.php 15.5 KB
Newer Older
MatrixCrawler's avatar
MatrixCrawler committed
1
<?php
2 3
//$debug = isset($_GET['debug']);
$debug = true;
4 5
//$debug = isset($argv[1])?1:0;
//* Copyright (c) 2011, David Morley. This file is licensed under the Affero General Public License version 3 or later. See the COPYRIGHT file. */
6 7 8 9

// Other parameters.
$_domain = $_GET['domain'] ?? '';

10
require_once __DIR__ . '/../config.php';
dmorley's avatar
cleanup  
dmorley committed
11

dmorley's avatar
dmorley committed
12
//get master code version for diaspora pods
dmorley's avatar
cleanup  
dmorley committed
13
$mv = curl_init();
14
curl_setopt($mv, CURLOPT_URL, 'https://raw.githubusercontent.com/diaspora/diaspora/master/config/defaults.yml');
dmorley's avatar
cleanup  
dmorley committed
15 16 17 18 19 20 21
curl_setopt($mv, CURLOPT_POST, 0);
curl_setopt($mv, CURLOPT_HEADER, 0);
curl_setopt($mv, CURLOPT_CONNECTTIMEOUT, 5);
curl_setopt($mv, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($mv, CURLOPT_NOBODY, 0);
$outputmv = curl_exec($mv);
curl_close($mv);
22
preg_match('/number: "(.*?)"/', $outputmv, $version);
dmorley's avatar
cleanup  
dmorley committed
23
$dmasterversion = trim($version[1], '"');
24 25 26
if ($debug) {
  echo 'Diaspora Masterversion: ' . $dmasterversion . '<br>';
}
dmorley's avatar
cleanup  
dmorley committed
27

dmorley's avatar
dmorley committed
28
//get master code version for freindica pods
dmorley's avatar
cleanup  
dmorley committed
29
$mv = curl_init();
30
curl_setopt($mv, CURLOPT_URL, 'https://raw.githubusercontent.com/friendica/friendica/master/boot.php');
dmorley's avatar
cleanup  
dmorley committed
31 32 33 34 35 36 37
curl_setopt($mv, CURLOPT_POST, 0);
curl_setopt($mv, CURLOPT_HEADER, 0);
curl_setopt($mv, CURLOPT_CONNECTTIMEOUT, 5);
curl_setopt($mv, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($mv, CURLOPT_NOBODY, 0);
$outputmv = curl_exec($mv);
curl_close($mv);
38
preg_match('/FRIENDICA_VERSION\',      \'(.*?)\'/', $outputmv, $version);
dmorley's avatar
cleanup  
dmorley committed
39
$fmasterversion = trim($version[1], '"');
40 41 42
if ($debug) {
  echo 'Frendica Masterversion: ' . $fmasterversion . '<br>';
}
43

44 45
$dbh = pg_connect("dbname=$pgdb user=$pguser password=$pgpass");
$dbh || die('Error in connection: ' . pg_last_error());
MatrixCrawler's avatar
MatrixCrawler committed
46

dmorley's avatar
cleanup  
dmorley committed
47
//foreach pod check it and update db
48
if ($_domain) {
dmorley's avatar
dmorley committed
49
  $sql    = 'SELECT domain,statsurl,score,datecreated,weight FROM pods WHERE domain = $1';
50
  $sleep  = '0';
51 52
  $result = pg_query_params($dbh, $sql, [$_domain]);
} elseif (PHP_SAPI === 'cli') {
dmorley's avatar
dmorley committed
53
  $sql    = 'SELECT domain,statsurl,score,datecreated,adminrating,weight FROM pods';
54
  $sleep  = '1';
dmorley's avatar
cleanup  
dmorley committed
55
  $result = pg_query($dbh, $sql);
David Morley's avatar
David Morley committed
56 57
} else {
  die('No valid input');
dmorley's avatar
cleanup  
dmorley committed
58
}
59 60
$result || die('Error in SQL query1: ' . pg_last_error());

dmorley's avatar
cleanup  
dmorley committed
61 62
while ($row = pg_fetch_all($result)) {
  $numrows = pg_num_rows($result);
63
  for ($i = 0; $i < $numrows; $i++) {
64
    $domain    = $row[$i]['domain'];
65
    $score     = (int) $row[$i]['score'];
66
    $dateadded = $row[$i]['datecreated'];
67
    $admindb   = (int) $row[$i]['adminrating'];
68 69 70 71 72 73 74 75
    $weight    = $row[$i]['weight'];
    //get ratings
    $userrate       = 0;
    $adminrate      = 0;
    $userratingavg  = [];
    $adminratingavg = [];
    $userrating     = [];
    $adminrating    = [];
dmorley's avatar
dmorley committed
76
    $sqlforr        = 'SELECT admin,rating FROM rating_comments WHERE domain = $1';
77
    $ratings        = pg_query_params($dbh, $sqlforr, [$domain]);
78 79
    $ratings || die('Error in SQL query2: ' . pg_last_error());

80 81 82 83 84 85 86 87 88 89
    $numratings = pg_num_rows($ratings);
    while ($myrow = pg_fetch_assoc($ratings)) {
      if ($myrow['admin'] == 0) {
        $userratingavg[] = $myrow['rating'];
        $userrate ++;
      } elseif ($myrow['admin'] == 1) {
        $adminratingavg[] = $myrow['rating'];
        $adminrate ++;
      }
    }
MatrixCrawler's avatar
MatrixCrawler committed
90

91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130
    if ($userrate > 0) {
      $userrating = round(array_sum($userratingavg) / $userrate, 2);
    }
    if ($adminrate > 0) {
      $adminrating = round(array_sum($adminratingavg) / $adminrate, 2);
    }
    if ($debug) {
      echo 'Domain: ' . $domain . '<br>';
    }
    if (!$userrating) {
      $userrating = 0;
    }
    if ($userrating > 10) {
      $userrating = 10;
    }
    if (!$adminrating) {
      $adminrating = 0;
    }
    if ($adminrating > 10) {
      $adminrating = 10;
    }
    if ($admindb == - 1) {
      $adminrating = - 1;
    }
    pg_free_result($ratings);
    $userrate  = 0;
    $adminrate = 0;
    unset($userratingavg);
    unset($adminratingavg);
    unset($name);
    unset($total_users);
    unset($active_users_halfyear);
    unset($active_users_monthly);
    unset($local_posts);
    unset($registrations_open);
    unset($comment_counts);
    unset($service_facebook);
    unset($service_twitter);
    unset($service_tumblr);
    unset($service_wordpess);
131
    unset($service_xmpp);
132
    unset($shortversion);
133 134 135 136
    unset($dverr);
    unset($xdver);
    unset($softwarename);
    unset($outputsslerror);
137

138 139 140 141 142 143 144 145 146 147 148
    $chss = curl_init();
    curl_setopt($chss, CURLOPT_URL, 'https://' . $domain . '/nodeinfo/1.0');
    curl_setopt($chss, CURLOPT_POST, 0);
    curl_setopt($chss, CURLOPT_HEADER, 0);
    curl_setopt($chss, CURLOPT_CONNECTTIMEOUT, 9);
    curl_setopt($chss, CURLOPT_TIMEOUT, 9);
    curl_setopt($chss, CURLOPT_RETURNTRANSFER, 1);
    curl_setopt($chss, CURLOPT_NOBODY, 0);
    $outputssl      = curl_exec($chss);
    $outputsslerror = curl_error($chss);
    curl_close($chss);
149

150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170
    $ch = curl_init();
    curl_setopt($ch, CURLOPT_URL, 'http://' . $domain . '/nodeinfo/1.0');
    curl_setopt($ch, CURLOPT_POST, 0);
    curl_setopt($ch, CURLOPT_HEADER, 0);
    curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 9);
    curl_setopt($ch, CURLOPT_TIMEOUT, 9);
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
    curl_setopt($ch, CURLOPT_NOBODY, 0);
    $output = curl_exec($ch);
    curl_close($ch);
    if ($debug) {
      echo 'not-e';
      print $output;
    }
    if ($debug) {
      echo 'e';
      var_dump($outputssl);
    }
    if (!$output && !$outpulssl && !$domain) {
      continue;
      echo 'no connection to pod';
171
      
dmorley's avatar
dmorley committed
172
      $sql    = 'INSERT INTO checks (domain, online, error) VALUES ($1, $2, $3)';
173 174 175
      $result = pg_query_params($dbh, $sql, [$domain, false, $outputsslerror]);
      $result || die('Error in SQL query: ' . pg_last_error());
      
176 177 178 179
    }
    if ($outputssl) {
      $secure        = 'true';
      $outputresults = $outputssl;
180 181 182 183 184
      
      $sql    = 'INSERT INTO checks (domain, online) VALUES ($1, $2)';
      $result = pg_query_params($dbh, $sql, [$domain, true]);
      $result || die('Error in SQL query: ' . pg_last_error());
  
185 186 187 188 189 190 191 192 193 194 195 196 197 198 199
    } elseif ($output) {
      $secure        = 'false';
      $outputresults = $output;
    }
    if (stristr($outputresults, 'openRegistrations')) {
      $score = $score + 1;
      if ($debug) {
        echo 'Secure: ' . $secure . '<br>';
      }
      //get new json from nodeinfo
      $jsonssl = json_decode($outputresults);
      var_dump($jsonssl);
      if ($jsonssl->openRegistrations === true) {
        $registrations_open = 1;
      }
200
      $xdver = $jsonssl->software->version ?? 0;
201
      $dverr = explode('-', trim($xdver));
202
      $shortversion  = $dverr[0];
203
      if ($debug) {
204
        echo ' <br> Version code: ' . $shortversion . '<br>';
205
      }
206
      if (!$shortversion) {
207 208
        $score = $score - 2;
      }
209 210 211 212 213 214 215 216 217 218 219 220
      $softwarename          = $jsonssl->software->name ?? 'null';
      $name                  = $jsonssl->metadata->nodeName ?? 'null';
      $total_users           = $jsonssl->usage->users->total ?? 0;
      $active_users_halfyear = $jsonssl->usage->users->activeHalfyear ?? 0;
      $active_users_monthly  = $jsonssl->usage->users->activeMonth ?? 0;
      $local_posts           = $jsonssl->usage->localPosts ?? 0;
      $comment_counts        = $jsonssl->usage->localComments ?? 0;
      $service_facebook      = in_array('facebook', $jsonssl->services->outbound, true) ? 'true' : 'false';
      $service_twitter       = in_array('twitter', $jsonssl->services->outbound, true) ? 'true' : 'false';
      $service_tumblr        = in_array('tumblr', $jsonssl->services->outbound, true) ? 'true' : 'false';
      $service_wordpress     = in_array('wordpress', $jsonssl->services->outbound, true) ? 'true' : 'false';
      $service_xmpp          = $jsonssl->metadata->xmppChat === true ? 'true' : 'false';
221 222 223 224
    } else {
      $secure = 'false';
      $score  = $score - 1;
      $dver   = '.connect error';
225
      $shortversion  = 0;
226 227 228 229 230 231 232
      //no diaspora cookie on either, lets set this one as hidden and notify someone its not really a pod
      //could also be a ssl pod with a bad cert, I think its ok to call that a dead pod now
    }
    $signup = $registrations_open;
    if ($debug) {
      echo '<br>Signup Open: ' . $signup . '<br>';
    }
233 234 235 236 237
    $ip6cmd    = escapeshellcmd('dig +nocmd ' . $domain . ' aaaa +noall +short');
    $ipcmd     = escapeshellcmd('dig +nocmd ' . $domain . ' a +noall +short');
    $ip6 = exec($ip6cmd);
    $ip  = exec($ipcmd);
    $test   = strpos($ip6, ':');
238 239 240 241 242 243
    if ($test === false) {
      $ipv6 = 'no';
    } else {
      $ipv6 = 'yes';
    }
    if ($debug) {
244
      echo 'IP: ' . $ip . '<br>';
245
    }
246
    $location = geoip_record_by_name($ip);
247 248 249 250 251 252
    if ($debug) {
      echo ' Location: ';
      var_dump($location);
      echo '<br>';
    }
    if ($location) {
dmorley's avatar
dmorley committed
253 254 255 256 257
      $country = isset($location['country_code']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['country_code']) : null;
      $city    = isset($location['city']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['city']) : null;
      $state   = isset($location['region']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['region']) : null;
      $lat     = isset($location['latitude']) ? $location['latitude']: null;
      $long    = isset($location['longitude']) ? $location['longitude'] : null;
258 259 260 261 262 263 264 265 266
      //if lat and long are just a generic country with no detail lets make some tail up or openmap just stacks them all on top another
      if (strlen($lat) < 4) {
        $lat = $lat + (rand(1, 15) / 10);
      }
      if (strlen($long) < 4) {
        $long = $long + (rand(1, 15) / 10);
      }
    }
    echo '<br>';
267
    $statslastdate = date('Y-m-d H:i:s');
dmorley's avatar
dmorley committed
268
    if (strpos($row[$i]['statsurl'], 'pingdom.com')) {
269 270 271
      //curl the pingdom page 
      $ping      = curl_init();
      $thismonth = '/' . date('Y') . '/' . date('m');
dmorley's avatar
dmorley committed
272
      curl_setopt($ping, CURLOPT_URL, $row[$i]['statsurl'] . $thismonth);
273
      if ($debug) {
dmorley's avatar
dmorley committed
274
        echo $row[$i]['statsurl'] . $thismonth;
275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301
      }
      curl_setopt($ping, CURLOPT_POST, 0);
      curl_setopt($ping, CURLOPT_HEADER, 1);
      curl_setopt($ping, CURLOPT_RETURNTRANSFER, 1);
      curl_setopt($ping, CURLOPT_CONNECTTIMEOUT, 8);
      curl_setopt($ping, CURLOPT_NOBODY, 0);
      curl_setopt($ping, CURLOPT_MAXCONNECTS, 5);
      curl_setopt($ping, CURLOPT_FOLLOWLOCATION, true);
      $pingdom = curl_exec($ping);
      $info    = curl_getinfo($ping);
      curl_close($ping);
      if ($debug) {
        echo '<br>Pingdom code: ' . $info['http_code'] . '<br>';
      }
      if ($info['http_code'] == 200) {
        //response time
        preg_match_all('/<h3>Avg. resp. time this month<\/h3>
          <p class="large">(.*?)</', $pingdom, $matcheach);
        $responsetime = $matcheach[1][0];
        //months monitored
        preg_match_all('/"historySelect">\s*(.*?)\s*<\/select/is', $pingdom, $matchhistory);
        $implodemonths = implode(' ', $matchhistory[1]);
        preg_match_all('/<option(.*?)/s', $implodemonths, $matchdates);
        $months = isset($matchdates[0]) ? count($matchdates[0]) : 0;
        //uptime %
        preg_match_all('/<h3>Uptime this month<\/h3>\s*<p class="large">(.*?)%</', $pingdom, $matchper);
        $uptime      = isset($matchper[1][0]) ? preg_replace('/,/', '.', $matchper[1][0]) : 0;
302
        $statslastdate = date('Y-m-d H:i:s');
303
        if (strpos($pingdom, "class=\"up\"")) {
dmorley's avatar
dmorley committed
304
          $status = 'up';
305
        } elseif (strpos($pingdom, "class=\"down\"")) {
dmorley's avatar
dmorley committed
306
          $status = 'down';
307
        } elseif (strpos($pingdom, "class=\"paused\"")) {
dmorley's avatar
dmorley committed
308
          $status = 'paused';
dmorley's avatar
cleanup  
dmorley committed
309
        } else {
dmorley's avatar
dmorley committed
310
          $status  = 'error';
311 312 313
          $score = $score - 2;
        }
      } else {
dmorley's avatar
cleanup  
dmorley committed
314
        //pingdom url is <> 200 so stats are gone, lower score
315
        $score = $score - 2;
dmorley's avatar
cleanup  
dmorley committed
316 317
      }
    } else {
318 319
      //do uptimerobot API instead
      $ping = curl_init();
dmorley's avatar
dmorley committed
320
      curl_setopt($ping, CURLOPT_URL, 'https://api.uptimerobot.com/getMonitors?format=json&customUptimeRatio=7-30-60-90&responseTimes=1&responseTimesAverage=86400&apiKey=' . $row[$i]['statsurl']);
321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342
      curl_setopt($ping, CURLOPT_POST, 0);
      curl_setopt($ping, CURLOPT_HEADER, 0);
      curl_setopt($ping, CURLOPT_RETURNTRANSFER, 1);
      curl_setopt($ping, CURLOPT_CONNECTTIMEOUT, 8);
      curl_setopt($ping, CURLOPT_NOBODY, 0);
      curl_setopt($ping, CURLOPT_MAXCONNECTS, 5);
      curl_setopt($ping, CURLOPT_FOLLOWLOCATION, true);
      $uptimerobot = curl_exec($ping);
      curl_close($ping);
      $json_encap = 'jsonUptimeRobotApi()';
      $up2        = substr($uptimerobot, strlen($json_encap) - 1, strlen($uptimerobot) - strlen($json_encap));
      $uptr       = json_decode($up2);
      if ($debug) {
        print_r($uptr);
        echo '<br>';
      }
      if (!$uptr) {
        $score = $score - 2;
      }
      $responsetime    = $uptr->monitors->monitor{'0'}->responsetime{'0'}->value;
      $uptimerobotstat = $uptr->stat;
      $uptime          = $uptr->monitors->monitor{'0'}->alltimeuptimeratio;
dmorley's avatar
dmorley committed
343
      $uptime_custom   = $uptr->monitors->monitor{'0'}->customuptimeratio;
344 345 346
      $diff            = abs(strtotime(date('Y-m-d H:i:s')) - strtotime($dateadded));
      $months          = floor(($diff - $years * 365 * 60 * 60 * 24) / (30 * 60 * 60 * 24));
      if ($uptr->monitors->monitor{'0'}->status == 2) {
dmorley's avatar
dmorley committed
347
        $status = 'Up';
348 349
      }
      if ($uptr->monitors->monitor{'0'}->status == 0) {
dmorley's avatar
dmorley committed
350
        $status = 'Paused';
351 352
      }
      if ($uptr->monitors->monitor{'0'}->status == 1) {
dmorley's avatar
dmorley committed
353
        $status = 'Not Checked Yet';
354 355
      }
      if ($uptr->monitors->monitor{'0'}->status == 8) {
dmorley's avatar
dmorley committed
356
        $status = 'Seems Down';
357 358
      }
      if ($uptr->monitors->monitor{'0'}->status == 9) {
dmorley's avatar
dmorley committed
359
        $status = 'Down';
360 361
      }
      $pingdomdate = date('Y-m-d H:i:s');
dmorley's avatar
dmorley committed
362
      if ($uptimerobotstat == 'fail' || $status <> 'Up') {
363 364
        $score = $score - 2;
      }
dmorley's avatar
cleanup  
dmorley committed
365
    }
366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386
    if ($softwarename == 'diaspora') {
      $masterversion = $dmasterversion;
    } elseif ($softwarename == 'friendica') {
      $masterversion = $fmasterversion;
    }
    if ($score > 70) {
      $hidden = 'no';
    } else {
      $hidden = 'yes';
    }
    if ($debug) {
      echo 'Hidden: ' . $hidden . '<br>';
    }
    // lets cap the scores or you can go too high or too low to never be effected by them
    if ($score > 100) {
      $score = 100;
    } elseif ($score < 0) {
      $score = 0;
    }
    $weightedscore = ($uptime + $score + ($active_users_monthly / 19999) - ((10 - $weight) * .12));
    //sql it
MatrixCrawler's avatar
MatrixCrawler committed
387

388
    $timenow = date('Y-m-d H:i:s');
dmorley's avatar
dmorley committed
389
    $sql     = 'UPDATE pods SET secure = $2, hidden = $3, ip = $4, ipv6 = $5, monthsmonitored = $6, uptime_alltime = $7, status = $8, date_laststats = $9, date_updated = $10, responsetime = $11, score = $12, adminrating = $13, country = $14, city = $15, state = $16, lat = $17, long = $18, userrating = $19, shortversion = $20, masterversion = $21, signup = $22, total_users = $23, active_users_halfyear = $24, active_users_monthly = $25, local_posts = $26, name = $27, comment_counts = $28, service_facebook = $29, service_tumblr = $30, service_twitter = $31, service_wordpress = $32, weightedscore = $33, service_xmpp = $34, softwarename = $35, sslvalid = $36, uptime_custom = $37
390
  WHERE domain = $1';
dmorley's avatar
dmorley committed
391
    $result  = pg_query_params($dbh, $sql, [$domain, $secure, $hidden, $ip, $ipv6, $months, $uptime, $status, $statslastdate, $timenow, $responsetime, $score, $adminrating, $country, $city, $state, $lat, $long, $userrating, $shortversion, $masterversion, $signup, $total_users, $active_users_halfyear, $active_users_monthly, $local_posts, $name, $comment_counts, $service_facebook, $service_tumblr, $service_twitter, $service_wordpress, $weightedscore, $service_xmpp, $softwarename, $outputsslerror, $uptime_custom]);
392 393
    $result || die('Error in SQL query3: ' . pg_last_error());

394 395 396 397 398 399 400
    if ($debug) {
      echo '<br>Score out of 100: ' . $score . '<br>';
    }
    if (!$debug) {
      echo 'Success';
    }
    //end foreach
dmorley's avatar
cleanup  
dmorley committed
401
  }
402
}
dmorley's avatar
cleanup  
dmorley committed
403 404
pg_free_result($result);
pg_close($dbh);