pull.php 15 KB
Newer Older
MatrixCrawler's avatar
MatrixCrawler committed
1
<?php
2 3
//$debug = isset($_GET['debug']);
$debug = true;
4 5
//$debug = isset($argv[1])?1:0;
//* Copyright (c) 2011, David Morley. This file is licensed under the Affero General Public License version 3 or later. See the COPYRIGHT file. */
6 7 8 9

// Other parameters.
$_domain = $_GET['domain'] ?? '';

10
require_once __DIR__ . '/../config.php';
dmorley's avatar
dmorley committed
11

dmorley's avatar
dmorley committed
12
//get master code version for diaspora pods
dmorley's avatar
dmorley committed
13
$mv = curl_init();
14
curl_setopt($mv, CURLOPT_URL, 'https://raw.githubusercontent.com/diaspora/diaspora/master/config/defaults.yml');
dmorley's avatar
dmorley committed
15 16 17 18 19 20 21
curl_setopt($mv, CURLOPT_POST, 0);
curl_setopt($mv, CURLOPT_HEADER, 0);
curl_setopt($mv, CURLOPT_CONNECTTIMEOUT, 5);
curl_setopt($mv, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($mv, CURLOPT_NOBODY, 0);
$outputmv = curl_exec($mv);
curl_close($mv);
22
preg_match('/number: "(.*?)"/', $outputmv, $version);
dmorley's avatar
dmorley committed
23
$dmasterversion = trim($version[1], '"');
24 25 26
if ($debug) {
  echo 'Diaspora Masterversion: ' . $dmasterversion . '<br>';
}
dmorley's avatar
dmorley committed
27

dmorley's avatar
dmorley committed
28
//get master code version for freindica pods
dmorley's avatar
dmorley committed
29
$mv = curl_init();
30
curl_setopt($mv, CURLOPT_URL, 'https://raw.githubusercontent.com/friendica/friendica/master/boot.php');
dmorley's avatar
dmorley committed
31 32 33 34 35 36 37
curl_setopt($mv, CURLOPT_POST, 0);
curl_setopt($mv, CURLOPT_HEADER, 0);
curl_setopt($mv, CURLOPT_CONNECTTIMEOUT, 5);
curl_setopt($mv, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($mv, CURLOPT_NOBODY, 0);
$outputmv = curl_exec($mv);
curl_close($mv);
38
preg_match('/FRIENDICA_VERSION\',      \'(.*?)\'/', $outputmv, $version);
dmorley's avatar
dmorley committed
39
$fmasterversion = trim($version[1], '"');
40 41 42
if ($debug) {
  echo 'Frendica Masterversion: ' . $fmasterversion . '<br>';
}
43

44 45
$dbh = pg_connect("dbname=$pgdb user=$pguser password=$pgpass");
$dbh || die('Error in connection: ' . pg_last_error());
MatrixCrawler's avatar
MatrixCrawler committed
46

dmorley's avatar
dmorley committed
47
//foreach pod check it and update db
48
if ($_domain) {
dmorley's avatar
dmorley committed
49
  $sql    = 'SELECT domain,statsurl,score,datecreated,weight FROM pods WHERE domain = $1';
50
  $sleep  = '0';
51 52
  $result = pg_query_params($dbh, $sql, [$_domain]);
} elseif (PHP_SAPI === 'cli') {
dmorley's avatar
dmorley committed
53
  $sql    = 'SELECT domain,statsurl,score,datecreated,adminrating,weight FROM pods';
54
  $sleep  = '1';
dmorley's avatar
dmorley committed
55
  $result = pg_query($dbh, $sql);
56 57
} else {
  die('No valid input');
dmorley's avatar
dmorley committed
58
}
59 60
$result || die('Error in SQL query1: ' . pg_last_error());

dmorley's avatar
dmorley committed
61 62
while ($row = pg_fetch_all($result)) {
  $numrows = pg_num_rows($result);
63
  for ($i = 0; $i < $numrows; $i++) {
64
    $domain    = $row[$i]['domain'];
65
    $score     = (int) $row[$i]['score'];
66
    $dateadded = $row[$i]['datecreated'];
67
    $admindb   = (int) $row[$i]['adminrating'];
68 69 70 71 72 73 74 75
    $weight    = $row[$i]['weight'];
    //get ratings
    $userrate       = 0;
    $adminrate      = 0;
    $userratingavg  = [];
    $adminratingavg = [];
    $userrating     = [];
    $adminrating    = [];
76
    $sqlforr        = 'SELECT * FROM rating_comments WHERE domain = $1';
77
    $ratings        = pg_query_params($dbh, $sqlforr, [$domain]);
78 79
    $ratings || die('Error in SQL query2: ' . pg_last_error());

80 81 82 83 84 85 86 87 88 89
    $numratings = pg_num_rows($ratings);
    while ($myrow = pg_fetch_assoc($ratings)) {
      if ($myrow['admin'] == 0) {
        $userratingavg[] = $myrow['rating'];
        $userrate ++;
      } elseif ($myrow['admin'] == 1) {
        $adminratingavg[] = $myrow['rating'];
        $adminrate ++;
      }
    }
MatrixCrawler's avatar
MatrixCrawler committed
90

91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130
    if ($userrate > 0) {
      $userrating = round(array_sum($userratingavg) / $userrate, 2);
    }
    if ($adminrate > 0) {
      $adminrating = round(array_sum($adminratingavg) / $adminrate, 2);
    }
    if ($debug) {
      echo 'Domain: ' . $domain . '<br>';
    }
    if (!$userrating) {
      $userrating = 0;
    }
    if ($userrating > 10) {
      $userrating = 10;
    }
    if (!$adminrating) {
      $adminrating = 0;
    }
    if ($adminrating > 10) {
      $adminrating = 10;
    }
    if ($admindb == - 1) {
      $adminrating = - 1;
    }
    pg_free_result($ratings);
    $userrate  = 0;
    $adminrate = 0;
    unset($userratingavg);
    unset($adminratingavg);
    unset($name);
    unset($total_users);
    unset($active_users_halfyear);
    unset($active_users_monthly);
    unset($local_posts);
    unset($registrations_open);
    unset($comment_counts);
    unset($service_facebook);
    unset($service_twitter);
    unset($service_tumblr);
    unset($service_wordpess);
131
    unset($service_xmpp);
132
    unset($shortversion);
133 134 135 136
    unset($dverr);
    unset($xdver);
    unset($softwarename);
    unset($outputsslerror);
137

138 139 140 141 142 143 144 145 146 147 148
    $chss = curl_init();
    curl_setopt($chss, CURLOPT_URL, 'https://' . $domain . '/nodeinfo/1.0');
    curl_setopt($chss, CURLOPT_POST, 0);
    curl_setopt($chss, CURLOPT_HEADER, 0);
    curl_setopt($chss, CURLOPT_CONNECTTIMEOUT, 9);
    curl_setopt($chss, CURLOPT_TIMEOUT, 9);
    curl_setopt($chss, CURLOPT_RETURNTRANSFER, 1);
    curl_setopt($chss, CURLOPT_NOBODY, 0);
    $outputssl      = curl_exec($chss);
    $outputsslerror = curl_error($chss);
    curl_close($chss);
149

150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189
    $ch = curl_init();
    curl_setopt($ch, CURLOPT_URL, 'http://' . $domain . '/nodeinfo/1.0');
    curl_setopt($ch, CURLOPT_POST, 0);
    curl_setopt($ch, CURLOPT_HEADER, 0);
    curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 9);
    curl_setopt($ch, CURLOPT_TIMEOUT, 9);
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
    curl_setopt($ch, CURLOPT_NOBODY, 0);
    $output = curl_exec($ch);
    curl_close($ch);
    if ($debug) {
      echo 'not-e';
      print $output;
    }
    if ($debug) {
      echo 'e';
      var_dump($outputssl);
    }
    if (!$output && !$outpulssl && !$domain) {
      continue;
      echo 'no connection to pod';
    }
    if ($outputssl) {
      $secure        = 'true';
      $outputresults = $outputssl;
    } elseif ($output) {
      $secure        = 'false';
      $outputresults = $output;
    }
    if (stristr($outputresults, 'openRegistrations')) {
      $score = $score + 1;
      if ($debug) {
        echo 'Secure: ' . $secure . '<br>';
      }
      //get new json from nodeinfo
      $jsonssl = json_decode($outputresults);
      var_dump($jsonssl);
      if ($jsonssl->openRegistrations === true) {
        $registrations_open = 1;
      }
190
      $xdver = $jsonssl->software->version ?? 0;
191
      $dverr = explode('-', trim($xdver));
192
      $shortversion  = $dverr[0];
193
      if ($debug) {
194
        echo ' <br> Version code: ' . $shortversion . '<br>';
195
      }
196
      if (!$shortversion) {
197 198
        $score = $score - 2;
      }
199 200 201 202 203 204 205 206 207 208 209 210
      $softwarename          = $jsonssl->software->name ?? 'null';
      $name                  = $jsonssl->metadata->nodeName ?? 'null';
      $total_users           = $jsonssl->usage->users->total ?? 0;
      $active_users_halfyear = $jsonssl->usage->users->activeHalfyear ?? 0;
      $active_users_monthly  = $jsonssl->usage->users->activeMonth ?? 0;
      $local_posts           = $jsonssl->usage->localPosts ?? 0;
      $comment_counts        = $jsonssl->usage->localComments ?? 0;
      $service_facebook      = in_array('facebook', $jsonssl->services->outbound, true) ? 'true' : 'false';
      $service_twitter       = in_array('twitter', $jsonssl->services->outbound, true) ? 'true' : 'false';
      $service_tumblr        = in_array('tumblr', $jsonssl->services->outbound, true) ? 'true' : 'false';
      $service_wordpress     = in_array('wordpress', $jsonssl->services->outbound, true) ? 'true' : 'false';
      $service_xmpp          = $jsonssl->metadata->xmppChat === true ? 'true' : 'false';
211 212 213 214
    } else {
      $secure = 'false';
      $score  = $score - 1;
      $dver   = '.connect error';
215
      $shortversion  = 0;
216 217 218 219 220 221 222
      //no diaspora cookie on either, lets set this one as hidden and notify someone its not really a pod
      //could also be a ssl pod with a bad cert, I think its ok to call that a dead pod now
    }
    $signup = $registrations_open;
    if ($debug) {
      echo '<br>Signup Open: ' . $signup . '<br>';
    }
223 224 225 226 227
    $ip6cmd    = escapeshellcmd('dig +nocmd ' . $domain . ' aaaa +noall +short');
    $ipcmd     = escapeshellcmd('dig +nocmd ' . $domain . ' a +noall +short');
    $ip6 = exec($ip6cmd);
    $ip  = exec($ipcmd);
    $test   = strpos($ip6, ':');
228 229 230 231 232 233
    if ($test === false) {
      $ipv6 = 'no';
    } else {
      $ipv6 = 'yes';
    }
    if ($debug) {
234
      echo 'IP: ' . $ip . '<br>';
235
    }
236
    $location = geoip_record_by_name($ip);
237 238 239 240 241 242
    if ($debug) {
      echo ' Location: ';
      var_dump($location);
      echo '<br>';
    }
    if ($location) {
dmorley's avatar
dmorley committed
243 244 245 246 247
      $country = isset($location['country_code']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['country_code']) : null;
      $city    = isset($location['city']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['city']) : null;
      $state   = isset($location['region']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['region']) : null;
      $lat     = isset($location['latitude']) ? $location['latitude']: null;
      $long    = isset($location['longitude']) ? $location['longitude'] : null;
248 249 250 251 252 253 254 255 256
      //if lat and long are just a generic country with no detail lets make some tail up or openmap just stacks them all on top another
      if (strlen($lat) < 4) {
        $lat = $lat + (rand(1, 15) / 10);
      }
      if (strlen($long) < 4) {
        $long = $long + (rand(1, 15) / 10);
      }
    }
    echo '<br>';
257
    $statslastdate = date('Y-m-d H:i:s');
dmorley's avatar
dmorley committed
258
    if (strpos($row[$i]['statsurl'], 'pingdom.com')) {
259 260 261
      //curl the pingdom page 
      $ping      = curl_init();
      $thismonth = '/' . date('Y') . '/' . date('m');
dmorley's avatar
dmorley committed
262
      curl_setopt($ping, CURLOPT_URL, $row[$i]['statsurl'] . $thismonth);
263
      if ($debug) {
dmorley's avatar
dmorley committed
264
        echo $row[$i]['statsurl'] . $thismonth;
265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291
      }
      curl_setopt($ping, CURLOPT_POST, 0);
      curl_setopt($ping, CURLOPT_HEADER, 1);
      curl_setopt($ping, CURLOPT_RETURNTRANSFER, 1);
      curl_setopt($ping, CURLOPT_CONNECTTIMEOUT, 8);
      curl_setopt($ping, CURLOPT_NOBODY, 0);
      curl_setopt($ping, CURLOPT_MAXCONNECTS, 5);
      curl_setopt($ping, CURLOPT_FOLLOWLOCATION, true);
      $pingdom = curl_exec($ping);
      $info    = curl_getinfo($ping);
      curl_close($ping);
      if ($debug) {
        echo '<br>Pingdom code: ' . $info['http_code'] . '<br>';
      }
      if ($info['http_code'] == 200) {
        //response time
        preg_match_all('/<h3>Avg. resp. time this month<\/h3>
          <p class="large">(.*?)</', $pingdom, $matcheach);
        $responsetime = $matcheach[1][0];
        //months monitored
        preg_match_all('/"historySelect">\s*(.*?)\s*<\/select/is', $pingdom, $matchhistory);
        $implodemonths = implode(' ', $matchhistory[1]);
        preg_match_all('/<option(.*?)/s', $implodemonths, $matchdates);
        $months = isset($matchdates[0]) ? count($matchdates[0]) : 0;
        //uptime %
        preg_match_all('/<h3>Uptime this month<\/h3>\s*<p class="large">(.*?)%</', $pingdom, $matchper);
        $uptime      = isset($matchper[1][0]) ? preg_replace('/,/', '.', $matchper[1][0]) : 0;
292
        $statslastdate = date('Y-m-d H:i:s');
293
        if (strpos($pingdom, "class=\"up\"")) {
dmorley's avatar
dmorley committed
294
          $status = 'up';
295
        } elseif (strpos($pingdom, "class=\"down\"")) {
dmorley's avatar
dmorley committed
296
          $status = 'down';
297
        } elseif (strpos($pingdom, "class=\"paused\"")) {
dmorley's avatar
dmorley committed
298
          $status = 'paused';
dmorley's avatar
dmorley committed
299
        } else {
dmorley's avatar
dmorley committed
300
          $status  = 'error';
301 302 303
          $score = $score - 2;
        }
      } else {
dmorley's avatar
dmorley committed
304
        //pingdom url is <> 200 so stats are gone, lower score
305
        $score = $score - 2;
dmorley's avatar
dmorley committed
306 307
      }
    } else {
308 309
      //do uptimerobot API instead
      $ping = curl_init();
dmorley's avatar
dmorley committed
310
      curl_setopt($ping, CURLOPT_URL, 'https://api.uptimerobot.com/getMonitors?format=json&customUptimeRatio=7-30-60-90&responseTimes=1&responseTimesAverage=86400&apiKey=' . $row[$i]['statsurl']);
311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332
      curl_setopt($ping, CURLOPT_POST, 0);
      curl_setopt($ping, CURLOPT_HEADER, 0);
      curl_setopt($ping, CURLOPT_RETURNTRANSFER, 1);
      curl_setopt($ping, CURLOPT_CONNECTTIMEOUT, 8);
      curl_setopt($ping, CURLOPT_NOBODY, 0);
      curl_setopt($ping, CURLOPT_MAXCONNECTS, 5);
      curl_setopt($ping, CURLOPT_FOLLOWLOCATION, true);
      $uptimerobot = curl_exec($ping);
      curl_close($ping);
      $json_encap = 'jsonUptimeRobotApi()';
      $up2        = substr($uptimerobot, strlen($json_encap) - 1, strlen($uptimerobot) - strlen($json_encap));
      $uptr       = json_decode($up2);
      if ($debug) {
        print_r($uptr);
        echo '<br>';
      }
      if (!$uptr) {
        $score = $score - 2;
      }
      $responsetime    = $uptr->monitors->monitor{'0'}->responsetime{'0'}->value;
      $uptimerobotstat = $uptr->stat;
      $uptime          = $uptr->monitors->monitor{'0'}->alltimeuptimeratio;
dmorley's avatar
dmorley committed
333
      $uptime_custom   = $uptr->monitors->monitor{'0'}->customuptimeratio;
334 335 336
      $diff            = abs(strtotime(date('Y-m-d H:i:s')) - strtotime($dateadded));
      $months          = floor(($diff - $years * 365 * 60 * 60 * 24) / (30 * 60 * 60 * 24));
      if ($uptr->monitors->monitor{'0'}->status == 2) {
dmorley's avatar
dmorley committed
337
        $status = 'Up';
338 339
      }
      if ($uptr->monitors->monitor{'0'}->status == 0) {
dmorley's avatar
dmorley committed
340
        $status = 'Paused';
341 342
      }
      if ($uptr->monitors->monitor{'0'}->status == 1) {
dmorley's avatar
dmorley committed
343
        $status = 'Not Checked Yet';
344 345
      }
      if ($uptr->monitors->monitor{'0'}->status == 8) {
dmorley's avatar
dmorley committed
346
        $status = 'Seems Down';
347 348
      }
      if ($uptr->monitors->monitor{'0'}->status == 9) {
dmorley's avatar
dmorley committed
349
        $status = 'Down';
350 351
      }
      $pingdomdate = date('Y-m-d H:i:s');
dmorley's avatar
dmorley committed
352
      if ($uptimerobotstat == 'fail' || $status <> 'Up') {
353 354
        $score = $score - 2;
      }
dmorley's avatar
dmorley committed
355
    }
356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376
    if ($softwarename == 'diaspora') {
      $masterversion = $dmasterversion;
    } elseif ($softwarename == 'friendica') {
      $masterversion = $fmasterversion;
    }
    if ($score > 70) {
      $hidden = 'no';
    } else {
      $hidden = 'yes';
    }
    if ($debug) {
      echo 'Hidden: ' . $hidden . '<br>';
    }
    // lets cap the scores or you can go too high or too low to never be effected by them
    if ($score > 100) {
      $score = 100;
    } elseif ($score < 0) {
      $score = 0;
    }
    $weightedscore = ($uptime + $score + ($active_users_monthly / 19999) - ((10 - $weight) * .12));
    //sql it
MatrixCrawler's avatar
MatrixCrawler committed
377

378
    $timenow = date('Y-m-d H:i:s');
dmorley's avatar
dmorley committed
379
    $sql     = 'UPDATE pods SET secure = $2, hidden = $3, ip = $4, ipv6 = $5, monthsmonitored = $6, uptime_alltime = $7, status = $8, dateLaststats = $9, dateUpdated = $10, responsetime = $11, score = $12, adminrating = $13, country = $14, city = $15, state = $16, lat = $17, long = $18, userrating = $19, shortversion = $20, masterversion = $21, signup = $22, total_users = $23, active_users_halfyear = $24, active_users_monthly = $25, local_posts = $26, name = $27, comment_counts = $28, service_facebook = $29, service_tumblr = $30, service_twitter = $31, service_wordpress = $32, weightedscore = $33, service_xmpp = $34, softwarename = $35, sslvalid = $36, uptime_custom = $37
380
  WHERE domain = $1';
dmorley's avatar
dmorley committed
381
    $result  = pg_query_params($dbh, $sql, [$domain, $secure, $hidden, $ip, $ipv6, $months, $uptime, $status, $statslastdate, $timenow, $responsetime, $score, $adminrating, $country, $city, $state, $lat, $long, $userrating, $shortversion, $masterversion, $signup, $total_users, $active_users_halfyear, $active_users_monthly, $local_posts, $name, $comment_counts, $service_facebook, $service_tumblr, $service_twitter, $service_wordpress, $weightedscore, $service_xmpp, $softwarename, $outputsslerror, $uptime_custom]);
382 383
    $result || die('Error in SQL query3: ' . pg_last_error());

384 385 386 387 388 389 390
    if ($debug) {
      echo '<br>Score out of 100: ' . $score . '<br>';
    }
    if (!$debug) {
      echo 'Success';
    }
    //end foreach
dmorley's avatar
dmorley committed
391
  }
392
}
dmorley's avatar
dmorley committed
393 394
pg_free_result($result);
pg_close($dbh);