pull.php 12.7 KB
Newer Older
MatrixCrawler's avatar
MatrixCrawler committed
1
<?php
2 3
//$debug = isset($_GET['debug']);
$debug = true;
4 5
//$debug = isset($argv[1])?1:0;
//* Copyright (c) 2011, David Morley. This file is licensed under the Affero General Public License version 3 or later. See the COPYRIGHT file. */
6 7 8 9

// Other parameters.
$_domain = $_GET['domain'] ?? '';

10
require_once __DIR__ . '/../config.php';
dmorley's avatar
cleanup  
dmorley committed
11

dmorley's avatar
dmorley committed
12
//get master code version for diaspora pods
dmorley's avatar
cleanup  
dmorley committed
13
$mv = curl_init();
14
curl_setopt($mv, CURLOPT_URL, 'https://raw.githubusercontent.com/diaspora/diaspora/master/config/defaults.yml');
dmorley's avatar
cleanup  
dmorley committed
15 16 17 18 19 20 21
curl_setopt($mv, CURLOPT_POST, 0);
curl_setopt($mv, CURLOPT_HEADER, 0);
curl_setopt($mv, CURLOPT_CONNECTTIMEOUT, 5);
curl_setopt($mv, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($mv, CURLOPT_NOBODY, 0);
$outputmv = curl_exec($mv);
curl_close($mv);
22
preg_match('/number: "(.*?)"/', $outputmv, $version);
dmorley's avatar
cleanup  
dmorley committed
23
$dmasterversion = trim($version[1], '"');
24 25 26
if ($debug) {
  echo 'Diaspora Masterversion: ' . $dmasterversion . '<br>';
}
dmorley's avatar
cleanup  
dmorley committed
27

dmorley's avatar
dmorley committed
28
//get master code version for freindica pods
dmorley's avatar
cleanup  
dmorley committed
29
$mv = curl_init();
30
curl_setopt($mv, CURLOPT_URL, 'https://raw.githubusercontent.com/friendica/friendica/master/boot.php');
dmorley's avatar
cleanup  
dmorley committed
31 32 33 34 35 36 37
curl_setopt($mv, CURLOPT_POST, 0);
curl_setopt($mv, CURLOPT_HEADER, 0);
curl_setopt($mv, CURLOPT_CONNECTTIMEOUT, 5);
curl_setopt($mv, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($mv, CURLOPT_NOBODY, 0);
$outputmv = curl_exec($mv);
curl_close($mv);
38
preg_match('/FRIENDICA_VERSION\',      \'(.*?)\'/', $outputmv, $version);
dmorley's avatar
cleanup  
dmorley committed
39
$fmasterversion = trim($version[1], '"');
40 41 42
if ($debug) {
  echo 'Frendica Masterversion: ' . $fmasterversion . '<br>';
}
43

44 45
$dbh = pg_connect("dbname=$pgdb user=$pguser password=$pgpass");
$dbh || die('Error in connection: ' . pg_last_error());
MatrixCrawler's avatar
MatrixCrawler committed
46

dmorley's avatar
cleanup  
dmorley committed
47
//foreach pod check it and update db
48
if ($_domain) {
dmorley's avatar
dmorley committed
49
  $sql    = 'SELECT domain,stats_apikey,score,date_created,weight FROM pods WHERE domain = $1';
50
  $sleep  = '0';
51 52
  $result = pg_query_params($dbh, $sql, [$_domain]);
} elseif (PHP_SAPI === 'cli') {
dmorley's avatar
dmorley committed
53
  $sql    = 'SELECT domain,stats_apikey,score,date_created,adminrating,weight FROM pods';
54
  $sleep  = '1';
dmorley's avatar
cleanup  
dmorley committed
55
  $result = pg_query($dbh, $sql);
David Morley's avatar
David Morley committed
56 57
} else {
  die('No valid input');
dmorley's avatar
cleanup  
dmorley committed
58
}
59 60
$result || die('Error in SQL query1: ' . pg_last_error());

dmorley's avatar
cleanup  
dmorley committed
61 62
while ($row = pg_fetch_all($result)) {
  $numrows = pg_num_rows($result);
63
  for ($i = 0; $i < $numrows; $i++) {
64
    $domain    = $row[$i]['domain'];
65
    $score     = (int) $row[$i]['score'];
dmorley's avatar
dmorley committed
66
    $dateadded = $row[$i]['date_created'];
67
    $admindb   = (int) $row[$i]['adminrating'];
68 69 70 71 72 73 74 75
    $weight    = $row[$i]['weight'];
    //get ratings
    $userrate       = 0;
    $adminrate      = 0;
    $userratingavg  = [];
    $adminratingavg = [];
    $userrating     = [];
    $adminrating    = [];
dmorley's avatar
dmorley committed
76
    $sqlforr        = 'SELECT admin,rating FROM rating_comments WHERE domain = $1';
77
    $ratings        = pg_query_params($dbh, $sqlforr, [$domain]);
78 79
    $ratings || die('Error in SQL query2: ' . pg_last_error());

80 81 82 83 84 85 86 87 88 89
    $numratings = pg_num_rows($ratings);
    while ($myrow = pg_fetch_assoc($ratings)) {
      if ($myrow['admin'] == 0) {
        $userratingavg[] = $myrow['rating'];
        $userrate ++;
      } elseif ($myrow['admin'] == 1) {
        $adminratingavg[] = $myrow['rating'];
        $adminrate ++;
      }
    }
MatrixCrawler's avatar
MatrixCrawler committed
90

91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130
    if ($userrate > 0) {
      $userrating = round(array_sum($userratingavg) / $userrate, 2);
    }
    if ($adminrate > 0) {
      $adminrating = round(array_sum($adminratingavg) / $adminrate, 2);
    }
    if ($debug) {
      echo 'Domain: ' . $domain . '<br>';
    }
    if (!$userrating) {
      $userrating = 0;
    }
    if ($userrating > 10) {
      $userrating = 10;
    }
    if (!$adminrating) {
      $adminrating = 0;
    }
    if ($adminrating > 10) {
      $adminrating = 10;
    }
    if ($admindb == - 1) {
      $adminrating = - 1;
    }
    pg_free_result($ratings);
    $userrate  = 0;
    $adminrate = 0;
    unset($userratingavg);
    unset($adminratingavg);
    unset($name);
    unset($total_users);
    unset($active_users_halfyear);
    unset($active_users_monthly);
    unset($local_posts);
    unset($registrations_open);
    unset($comment_counts);
    unset($service_facebook);
    unset($service_twitter);
    unset($service_tumblr);
    unset($service_wordpess);
131
    unset($service_xmpp);
132
    unset($shortversion);
133 134 135 136
    unset($dverr);
    unset($xdver);
    unset($softwarename);
    unset($outputsslerror);
137

138 139 140 141 142 143 144 145 146 147 148
    $chss = curl_init();
    curl_setopt($chss, CURLOPT_URL, 'https://' . $domain . '/nodeinfo/1.0');
    curl_setopt($chss, CURLOPT_POST, 0);
    curl_setopt($chss, CURLOPT_HEADER, 0);
    curl_setopt($chss, CURLOPT_CONNECTTIMEOUT, 9);
    curl_setopt($chss, CURLOPT_TIMEOUT, 9);
    curl_setopt($chss, CURLOPT_RETURNTRANSFER, 1);
    curl_setopt($chss, CURLOPT_NOBODY, 0);
    $outputssl      = curl_exec($chss);
    $outputsslerror = curl_error($chss);
    curl_close($chss);
149

150 151 152
    if ($debug) {
      var_dump($outputssl);
    }
dmorley's avatar
dmorley committed
153
    if (!$outpulssl && !$domain) {
154 155
      continue;
      echo 'no connection to pod';
156
      
dmorley's avatar
dmorley committed
157
      $sql    = 'INSERT INTO checks (domain, online, error) VALUES ($1, $2, $3)';
158 159 160
      $result = pg_query_params($dbh, $sql, [$domain, false, $outputsslerror]);
      $result || die('Error in SQL query: ' . pg_last_error());
      
161 162 163 164
    }
    if ($outputssl) {
      $secure        = 'true';
      $outputresults = $outputssl;
165 166 167 168 169
      
      $sql    = 'INSERT INTO checks (domain, online) VALUES ($1, $2)';
      $result = pg_query_params($dbh, $sql, [$domain, true]);
      $result || die('Error in SQL query: ' . pg_last_error());
  
dmorley's avatar
dmorley committed
170
    } else {
171 172 173 174 175 176 177 178 179 180 181 182 183 184
      $secure        = 'false';
      $outputresults = $output;
    }
    if (stristr($outputresults, 'openRegistrations')) {
      $score = $score + 1;
      if ($debug) {
        echo 'Secure: ' . $secure . '<br>';
      }
      //get new json from nodeinfo
      $jsonssl = json_decode($outputresults);
      var_dump($jsonssl);
      if ($jsonssl->openRegistrations === true) {
        $registrations_open = 1;
      }
185
      $xdver = $jsonssl->software->version ?? 0;
186
      $dverr = explode('-', trim($xdver));
187
      $shortversion  = $dverr[0];
188
      if ($debug) {
189
        echo ' <br> Version code: ' . $shortversion . '<br>';
190
      }
191
      if (!$shortversion) {
192 193
        $score = $score - 2;
      }
194 195 196 197 198 199 200 201 202 203 204 205
      $softwarename          = $jsonssl->software->name ?? 'null';
      $name                  = $jsonssl->metadata->nodeName ?? 'null';
      $total_users           = $jsonssl->usage->users->total ?? 0;
      $active_users_halfyear = $jsonssl->usage->users->activeHalfyear ?? 0;
      $active_users_monthly  = $jsonssl->usage->users->activeMonth ?? 0;
      $local_posts           = $jsonssl->usage->localPosts ?? 0;
      $comment_counts        = $jsonssl->usage->localComments ?? 0;
      $service_facebook      = in_array('facebook', $jsonssl->services->outbound, true) ? 'true' : 'false';
      $service_twitter       = in_array('twitter', $jsonssl->services->outbound, true) ? 'true' : 'false';
      $service_tumblr        = in_array('tumblr', $jsonssl->services->outbound, true) ? 'true' : 'false';
      $service_wordpress     = in_array('wordpress', $jsonssl->services->outbound, true) ? 'true' : 'false';
      $service_xmpp          = $jsonssl->metadata->xmppChat === true ? 'true' : 'false';
206 207 208 209
    } else {
      $secure = 'false';
      $score  = $score - 1;
      $dver   = '.connect error';
210
      $shortversion  = 0;
211 212 213 214 215 216
      //could also be a ssl pod with a bad cert, I think its ok to call that a dead pod now
    }
    $signup = $registrations_open;
    if ($debug) {
      echo '<br>Signup Open: ' . $signup . '<br>';
    }
217 218 219 220 221
    $ip6cmd    = escapeshellcmd('dig +nocmd ' . $domain . ' aaaa +noall +short');
    $ipcmd     = escapeshellcmd('dig +nocmd ' . $domain . ' a +noall +short');
    $ip6 = exec($ip6cmd);
    $ip  = exec($ipcmd);
    $test   = strpos($ip6, ':');
222 223 224 225 226 227
    if ($test === false) {
      $ipv6 = 'no';
    } else {
      $ipv6 = 'yes';
    }
    if ($debug) {
228
      echo 'IP: ' . $ip . '<br>';
229
    }
230
    $location = geoip_record_by_name($ip);
231 232 233 234 235 236
    if ($debug) {
      echo ' Location: ';
      var_dump($location);
      echo '<br>';
    }
    if ($location) {
David Morley's avatar
David Morley committed
237 238 239 240 241
      $country = !empty($location['country_code']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['country_code']) : null;
      $city    = !empty($location['city']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['city']) : null;
      $state   = !empty($location['region']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['region']) : null;
      $lat     = !empty($location['latitude']) ? $location['latitude']: null;
      $long    = !empty($location['longitude']) ? $location['longitude'] : null;
242 243 244 245 246 247 248 249 250
      //if lat and long are just a generic country with no detail lets make some tail up or openmap just stacks them all on top another
      if (strlen($lat) < 4) {
        $lat = $lat + (rand(1, 15) / 10);
      }
      if (strlen($long) < 4) {
        $long = $long + (rand(1, 15) / 10);
      }
    }
    echo '<br>';
251
    $statslastdate = date('Y-m-d H:i:s');
252
      $ping = curl_init();
dmorley's avatar
dmorley committed
253
      curl_setopt($ping, CURLOPT_URL, 'https://api.uptimerobot.com/getMonitors?format=json&customUptimeRatio=7-30-60-90&responseTimes=1&responseTimesAverage=86400&apiKey=' . $row[$i]['stats_apikey']);
254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275
      curl_setopt($ping, CURLOPT_POST, 0);
      curl_setopt($ping, CURLOPT_HEADER, 0);
      curl_setopt($ping, CURLOPT_RETURNTRANSFER, 1);
      curl_setopt($ping, CURLOPT_CONNECTTIMEOUT, 8);
      curl_setopt($ping, CURLOPT_NOBODY, 0);
      curl_setopt($ping, CURLOPT_MAXCONNECTS, 5);
      curl_setopt($ping, CURLOPT_FOLLOWLOCATION, true);
      $uptimerobot = curl_exec($ping);
      curl_close($ping);
      $json_encap = 'jsonUptimeRobotApi()';
      $up2        = substr($uptimerobot, strlen($json_encap) - 1, strlen($uptimerobot) - strlen($json_encap));
      $uptr       = json_decode($up2);
      if ($debug) {
        print_r($uptr);
        echo '<br>';
      }
      if (!$uptr) {
        $score = $score - 2;
      }
      $responsetime    = $uptr->monitors->monitor{'0'}->responsetime{'0'}->value;
      $uptimerobotstat = $uptr->stat;
      $uptime          = $uptr->monitors->monitor{'0'}->alltimeuptimeratio;
dmorley's avatar
dmorley committed
276
      $uptime_custom   = $uptr->monitors->monitor{'0'}->customuptimeratio;
David Morley's avatar
David Morley committed
277 278
      $diff            = (new DateTime())->diff(new DateTime($dateadded));
      $months          = $diff->m + ($diff->y * 12);
279
      if ($uptr->monitors->monitor{'0'}->status == 2) {
dmorley's avatar
dmorley committed
280
        $status = 'Up';
281 282
      }
      if ($uptr->monitors->monitor{'0'}->status == 0) {
dmorley's avatar
dmorley committed
283
        $status = 'Paused';
284 285
      }
      if ($uptr->monitors->monitor{'0'}->status == 1) {
dmorley's avatar
dmorley committed
286
        $status = 'Not Checked Yet';
287 288
      }
      if ($uptr->monitors->monitor{'0'}->status == 8) {
dmorley's avatar
dmorley committed
289
        $status = 'Seems Down';
290 291
      }
      if ($uptr->monitors->monitor{'0'}->status == 9) {
dmorley's avatar
dmorley committed
292
        $status = 'Down';
293 294
      }
      $pingdomdate = date('Y-m-d H:i:s');
dmorley's avatar
dmorley committed
295
      if ($uptimerobotstat == 'fail' || $status <> 'Up') {
296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318
        $score = $score - 2;
      }
    if ($softwarename == 'diaspora') {
      $masterversion = $dmasterversion;
    } elseif ($softwarename == 'friendica') {
      $masterversion = $fmasterversion;
    }
    if ($score > 70) {
      $hidden = 'no';
    } else {
      $hidden = 'yes';
    }
    if ($debug) {
      echo 'Hidden: ' . $hidden . '<br>';
    }
    // lets cap the scores or you can go too high or too low to never be effected by them
    if ($score > 100) {
      $score = 100;
    } elseif ($score < 0) {
      $score = 0;
    }
    $weightedscore = ($uptime + $score + ($active_users_monthly / 19999) - ((10 - $weight) * .12));
    //sql it
MatrixCrawler's avatar
MatrixCrawler committed
319

320
    $timenow = date('Y-m-d H:i:s');
dmorley's avatar
dmorley committed
321
    $sql     = 'UPDATE pods SET secure = $2, hidden = $3, ip = $4, ipv6 = $5, monthsmonitored = $6, uptime_alltime = $7, status = $8, date_laststats = $9, date_updated = $10, responsetime = $11, score = $12, adminrating = $13, country = $14, city = $15, state = $16, lat = $17, long = $18, userrating = $19, shortversion = $20, masterversion = $21, signup = $22, total_users = $23, active_users_halfyear = $24, active_users_monthly = $25, local_posts = $26, name = $27, comment_counts = $28, service_facebook = $29, service_tumblr = $30, service_twitter = $31, service_wordpress = $32, weightedscore = $33, service_xmpp = $34, softwarename = $35, sslvalid = $36, uptime_custom = $37
322
  WHERE domain = $1';
dmorley's avatar
dmorley committed
323
    $result  = pg_query_params($dbh, $sql, [$domain, $secure, $hidden, $ip, $ipv6, $months, $uptime, $status, $statslastdate, $timenow, $responsetime, $score, $adminrating, $country, $city, $state, $lat, $long, $userrating, $shortversion, $masterversion, $signup, $total_users, $active_users_halfyear, $active_users_monthly, $local_posts, $name, $comment_counts, $service_facebook, $service_tumblr, $service_twitter, $service_wordpress, $weightedscore, $service_xmpp, $softwarename, $outputsslerror, $uptime_custom]);
324 325
    $result || die('Error in SQL query3: ' . pg_last_error());

326 327 328 329 330 331 332
    if ($debug) {
      echo '<br>Score out of 100: ' . $score . '<br>';
    }
    if (!$debug) {
      echo 'Success';
    }
    //end foreach
dmorley's avatar
cleanup  
dmorley committed
333
  }
334
}
dmorley's avatar
cleanup  
dmorley committed
335 336
pg_free_result($result);
pg_close($dbh);