pull.php 11.8 KB
Newer Older
MatrixCrawler's avatar
MatrixCrawler committed
1
<?php
2 3
//$debug = isset($_GET['debug']);
$debug = true;
4 5
//$debug = isset($argv[1])?1:0;
//* Copyright (c) 2011, David Morley. This file is licensed under the Affero General Public License version 3 or later. See the COPYRIGHT file. */
6 7 8 9

// Other parameters.
$_domain = $_GET['domain'] ?? '';

10
require_once __DIR__ . '/../config.php';
dmorley's avatar
cleanup  
dmorley committed
11

dmorley's avatar
dmorley committed
12
//get master code version for diaspora pods
dmorley's avatar
cleanup  
dmorley committed
13
$mv = curl_init();
14
curl_setopt($mv, CURLOPT_URL, 'https://raw.githubusercontent.com/diaspora/diaspora/master/config/defaults.yml');
dmorley's avatar
cleanup  
dmorley committed
15 16 17 18 19 20 21
curl_setopt($mv, CURLOPT_POST, 0);
curl_setopt($mv, CURLOPT_HEADER, 0);
curl_setopt($mv, CURLOPT_CONNECTTIMEOUT, 5);
curl_setopt($mv, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($mv, CURLOPT_NOBODY, 0);
$outputmv = curl_exec($mv);
curl_close($mv);
22
preg_match('/number: "(.*?)"/', $outputmv, $version);
dmorley's avatar
cleanup  
dmorley committed
23
$dmasterversion = trim($version[1], '"');
24 25 26
if ($debug) {
  echo 'Diaspora Masterversion: ' . $dmasterversion . '<br>';
}
dmorley's avatar
cleanup  
dmorley committed
27

dmorley's avatar
dmorley committed
28
//get master code version for freindica pods
dmorley's avatar
cleanup  
dmorley committed
29
$mv = curl_init();
30
curl_setopt($mv, CURLOPT_URL, 'https://raw.githubusercontent.com/friendica/friendica/master/boot.php');
dmorley's avatar
cleanup  
dmorley committed
31 32 33 34 35 36 37
curl_setopt($mv, CURLOPT_POST, 0);
curl_setopt($mv, CURLOPT_HEADER, 0);
curl_setopt($mv, CURLOPT_CONNECTTIMEOUT, 5);
curl_setopt($mv, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($mv, CURLOPT_NOBODY, 0);
$outputmv = curl_exec($mv);
curl_close($mv);
38
preg_match('/FRIENDICA_VERSION\',      \'(.*?)\'/', $outputmv, $version);
dmorley's avatar
cleanup  
dmorley committed
39
$fmasterversion = trim($version[1], '"');
40 41 42
if ($debug) {
  echo 'Frendica Masterversion: ' . $fmasterversion . '<br>';
}
43

44 45
$dbh = pg_connect("dbname=$pgdb user=$pguser password=$pgpass");
$dbh || die('Error in connection: ' . pg_last_error());
MatrixCrawler's avatar
MatrixCrawler committed
46

dmorley's avatar
cleanup  
dmorley committed
47
//foreach pod check it and update db
48
if ($_domain) {
dmorley's avatar
dmorley committed
49
  $sql    = 'SELECT domain,stats_apikey,score,date_created,weight FROM pods WHERE domain = $1';
noplanman's avatar
noplanman committed
50
  //$sleep  = '0';
51 52
  $result = pg_query_params($dbh, $sql, [$_domain]);
} elseif (PHP_SAPI === 'cli') {
dmorley's avatar
dmorley committed
53
  $sql    = 'SELECT domain,stats_apikey,score,date_created,adminrating,weight FROM pods';
noplanman's avatar
noplanman committed
54
  //$sleep  = '1';
dmorley's avatar
cleanup  
dmorley committed
55
  $result = pg_query($dbh, $sql);
David Morley's avatar
David Morley committed
56 57
} else {
  die('No valid input');
dmorley's avatar
cleanup  
dmorley committed
58
}
59 60
$result || die('Error in SQL query1: ' . pg_last_error());

dmorley's avatar
cleanup  
dmorley committed
61 62
while ($row = pg_fetch_all($result)) {
  $numrows = pg_num_rows($result);
63
  for ($i = 0; $i < $numrows; $i++) {
64
    $domain    = $row[$i]['domain'];
65
    $score     = (int) $row[$i]['score'];
dmorley's avatar
dmorley committed
66
    $dateadded = $row[$i]['date_created'];
67
    $admindb   = (int) $row[$i]['adminrating'];
68
    $weight    = $row[$i]['weight'];
noplanman's avatar
noplanman committed
69 70
    $sqlforr       = 'SELECT admin,rating FROM rating_comments WHERE domain = $1';
    $ratings       = pg_query_params($dbh, $sqlforr, [$domain]);
71 72
    $ratings || die('Error in SQL query2: ' . pg_last_error());

73 74 75
    if ($debug) {
      echo 'Domain: ' . $domain . '<br>';
    }
noplanman's avatar
noplanman committed
76 77 78 79 80 81 82 83 84

    $user_ratings  = [];
    $admin_ratings = [];
    while ($rating = pg_fetch_assoc($ratings)) {
      if ($rating['admin'] == 0) {
        $user_ratings[] = $rating['rating'];
      } elseif ($rating['admin'] == 1) {
        $admin_ratings[] = $rating['rating'];
      }
85
    }
noplanman's avatar
noplanman committed
86 87 88
    $user_rating  = empty($user_ratings) ? 0 : max(10, round(array_sum($user_ratings) / count($user_ratings), 2));
    $admin_rating = empty($admin_ratings) ? 0 : max(10, round(array_sum($admin_ratings) / count($admin_ratings), 2));

89
    if ($admindb == - 1) {
noplanman's avatar
noplanman committed
90
      $admin_rating = - 1;
91 92 93 94 95 96 97 98 99 100 101 102
    }
    unset($name);
    unset($total_users);
    unset($active_users_halfyear);
    unset($active_users_monthly);
    unset($local_posts);
    unset($registrations_open);
    unset($comment_counts);
    unset($service_facebook);
    unset($service_twitter);
    unset($service_tumblr);
    unset($service_wordpess);
103
    unset($service_xmpp);
104
    unset($shortversion);
105 106 107 108
    unset($dverr);
    unset($xdver);
    unset($softwarename);
    unset($outputsslerror);
109

110 111 112 113 114 115 116 117 118 119 120
    $chss = curl_init();
    curl_setopt($chss, CURLOPT_URL, 'https://' . $domain . '/nodeinfo/1.0');
    curl_setopt($chss, CURLOPT_POST, 0);
    curl_setopt($chss, CURLOPT_HEADER, 0);
    curl_setopt($chss, CURLOPT_CONNECTTIMEOUT, 9);
    curl_setopt($chss, CURLOPT_TIMEOUT, 9);
    curl_setopt($chss, CURLOPT_RETURNTRANSFER, 1);
    curl_setopt($chss, CURLOPT_NOBODY, 0);
    $outputssl      = curl_exec($chss);
    $outputsslerror = curl_error($chss);
    curl_close($chss);
121

122 123 124
    if ($debug) {
      var_dump($outputssl);
    }
dmorley's avatar
dmorley committed
125
    if (!$outpulssl && !$domain) {
126 127
      continue;
      echo 'no connection to pod';
128
      
dmorley's avatar
dmorley committed
129
      $sql    = 'INSERT INTO checks (domain, online, error) VALUES ($1, $2, $3)';
130 131 132
      $result = pg_query_params($dbh, $sql, [$domain, false, $outputsslerror]);
      $result || die('Error in SQL query: ' . pg_last_error());
      
133 134 135 136
    }
    if ($outputssl) {
      $secure        = 'true';
      $outputresults = $outputssl;
137 138 139 140 141
      
      $sql    = 'INSERT INTO checks (domain, online) VALUES ($1, $2)';
      $result = pg_query_params($dbh, $sql, [$domain, true]);
      $result || die('Error in SQL query: ' . pg_last_error());
  
dmorley's avatar
dmorley committed
142
    } else {
143 144 145 146
      $secure        = 'false';
      $outputresults = $output;
    }
    if (stristr($outputresults, 'openRegistrations')) {
noplanman's avatar
noplanman committed
147
      $score += 1;
148 149 150 151 152 153 154 155 156
      if ($debug) {
        echo 'Secure: ' . $secure . '<br>';
      }
      //get new json from nodeinfo
      $jsonssl = json_decode($outputresults);
      var_dump($jsonssl);
      if ($jsonssl->openRegistrations === true) {
        $registrations_open = 1;
      }
157
      $xdver = $jsonssl->software->version ?? 0;
158
      $dverr = explode('-', trim($xdver));
159
      $shortversion  = $dverr[0];
160
      if ($debug) {
161
        echo ' <br> Version code: ' . $shortversion . '<br>';
162
      }
noplanman's avatar
noplanman committed
163 164
      $shortversion || $score -= 2;

165 166 167 168 169 170 171 172 173 174 175 176
      $softwarename          = $jsonssl->software->name ?? 'null';
      $name                  = $jsonssl->metadata->nodeName ?? 'null';
      $total_users           = $jsonssl->usage->users->total ?? 0;
      $active_users_halfyear = $jsonssl->usage->users->activeHalfyear ?? 0;
      $active_users_monthly  = $jsonssl->usage->users->activeMonth ?? 0;
      $local_posts           = $jsonssl->usage->localPosts ?? 0;
      $comment_counts        = $jsonssl->usage->localComments ?? 0;
      $service_facebook      = in_array('facebook', $jsonssl->services->outbound, true) ? 'true' : 'false';
      $service_twitter       = in_array('twitter', $jsonssl->services->outbound, true) ? 'true' : 'false';
      $service_tumblr        = in_array('tumblr', $jsonssl->services->outbound, true) ? 'true' : 'false';
      $service_wordpress     = in_array('wordpress', $jsonssl->services->outbound, true) ? 'true' : 'false';
      $service_xmpp          = $jsonssl->metadata->xmppChat === true ? 'true' : 'false';
177 178
    } else {
      $secure = 'false';
noplanman's avatar
noplanman committed
179
      $score -= 1;
180
      $dver   = '.connect error';
181
      $shortversion  = 0;
182 183 184 185 186 187
      //could also be a ssl pod with a bad cert, I think its ok to call that a dead pod now
    }
    $signup = $registrations_open;
    if ($debug) {
      echo '<br>Signup Open: ' . $signup . '<br>';
    }
noplanman's avatar
noplanman committed
188 189
    $ip6    = exec(escapeshellcmd('dig +nocmd ' . $domain . ' aaaa +noall +short'));
    $ip     = exec(escapeshellcmd('dig +nocmd ' . $domain . ' a +noall +short'));
190
    $test   = strpos($ip6, ':');
noplanman's avatar
noplanman committed
191 192
    $ipv6   = $test === false ? 'no' : 'yes';

193
    if ($debug) {
194
      echo 'IP: ' . $ip . '<br>';
195
    }
196
    $location = geoip_record_by_name($ip);
197 198 199 200 201 202
    if ($debug) {
      echo ' Location: ';
      var_dump($location);
      echo '<br>';
    }
    if ($location) {
David Morley's avatar
David Morley committed
203 204 205
      $country = !empty($location['country_code']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['country_code']) : null;
      $city    = !empty($location['city']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['city']) : null;
      $state   = !empty($location['region']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['region']) : null;
noplanman's avatar
noplanman committed
206
      $lat     = !empty($location['latitude']) ? $location['latitude'] : null;
David Morley's avatar
David Morley committed
207
      $long    = !empty($location['longitude']) ? $location['longitude'] : null;
noplanman's avatar
noplanman committed
208 209 210 211 212

      // If lat and long are just a generic country with no detail lets make some tail up
      // else openmap just stacks them all on top of each other.
      strlen($lat) < 4 && $lat += (random_int(1, 15) / 10);
      strlen($long) < 4 && $long += (random_int(1, 15) / 10);
213 214
    }
    echo '<br>';
215
    $statslastdate = date('Y-m-d H:i:s');
216
      $ping = curl_init();
noplanman's avatar
noplanman committed
217
      curl_setopt($ping, CURLOPT_URL, 'https://api.uptimerobot.com/getMonitors?format=json&noJsonCallback=1&customUptimeRatio=7-30-60-90&responseTimes=1&responseTimesAverage=86400&apiKey=' . $row[$i]['stats_apikey']);
218 219 220 221 222 223 224
      curl_setopt($ping, CURLOPT_POST, 0);
      curl_setopt($ping, CURLOPT_HEADER, 0);
      curl_setopt($ping, CURLOPT_RETURNTRANSFER, 1);
      curl_setopt($ping, CURLOPT_CONNECTTIMEOUT, 8);
      curl_setopt($ping, CURLOPT_NOBODY, 0);
      curl_setopt($ping, CURLOPT_MAXCONNECTS, 5);
      curl_setopt($ping, CURLOPT_FOLLOWLOCATION, true);
noplanman's avatar
noplanman committed
225
      $uptr = json_decode(curl_exec($ping));
226 227 228 229 230
      curl_close($ping);
      if ($debug) {
        print_r($uptr);
        echo '<br>';
      }
noplanman's avatar
noplanman committed
231 232
      $uptr || $score -= 2;

233 234 235
      $responsetime    = $uptr->monitors->monitor{'0'}->responsetime{'0'}->value;
      $uptimerobotstat = $uptr->stat;
      $uptime          = $uptr->monitors->monitor{'0'}->alltimeuptimeratio;
dmorley's avatar
dmorley committed
236
      $uptime_custom   = $uptr->monitors->monitor{'0'}->customuptimeratio;
David Morley's avatar
David Morley committed
237 238
      $diff            = (new DateTime())->diff(new DateTime($dateadded));
      $months          = $diff->m + ($diff->y * 12);
239
      if ($uptr->monitors->monitor{'0'}->status == 2) {
dmorley's avatar
dmorley committed
240
        $status = 'Up';
241 242
      }
      if ($uptr->monitors->monitor{'0'}->status == 0) {
dmorley's avatar
dmorley committed
243
        $status = 'Paused';
244 245
      }
      if ($uptr->monitors->monitor{'0'}->status == 1) {
dmorley's avatar
dmorley committed
246
        $status = 'Not Checked Yet';
247 248
      }
      if ($uptr->monitors->monitor{'0'}->status == 8) {
dmorley's avatar
dmorley committed
249
        $status = 'Seems Down';
250 251
      }
      if ($uptr->monitors->monitor{'0'}->status == 9) {
dmorley's avatar
dmorley committed
252
        $status = 'Down';
253
      }
dmorley's avatar
cleanup  
dmorley committed
254
      if ($uptr) {
dmorley's avatar
dmorley committed
255
        $statslastdate = date('Y-m-d H:i:s');
dmorley's avatar
cleanup  
dmorley committed
256
      }
noplanman's avatar
noplanman committed
257 258 259
      ($uptimerobotstat !== 'fail' && $status === 'Up') || $score -= 2;

    if ($softwarename === 'diaspora') {
260
      $masterversion = $dmasterversion;
noplanman's avatar
noplanman committed
261
    } elseif ($softwarename === 'friendica') {
262 263
      $masterversion = $fmasterversion;
    }
noplanman's avatar
noplanman committed
264
    $hidden = $score > 70 ? 'no' : 'yes';
265 266 267 268 269 270 271 272 273 274 275
    if ($debug) {
      echo 'Hidden: ' . $hidden . '<br>';
    }
    // lets cap the scores or you can go too high or too low to never be effected by them
    if ($score > 100) {
      $score = 100;
    } elseif ($score < 0) {
      $score = 0;
    }
    $weightedscore = ($uptime + $score + ($active_users_monthly / 19999) - ((10 - $weight) * .12));
    //sql it
MatrixCrawler's avatar
MatrixCrawler committed
276

277
    $timenow = date('Y-m-d H:i:s');
dmorley's avatar
dmorley committed
278
    $sql     = 'UPDATE pods SET secure = $2, hidden = $3, ip = $4, ipv6 = $5, monthsmonitored = $6, uptime_alltime = $7, status = $8, date_laststats = $9, date_updated = $10, responsetime = $11, score = $12, adminrating = $13, country = $14, city = $15, state = $16, lat = $17, long = $18, userrating = $19, shortversion = $20, masterversion = $21, signup = $22, total_users = $23, active_users_halfyear = $24, active_users_monthly = $25, local_posts = $26, name = $27, comment_counts = $28, service_facebook = $29, service_tumblr = $30, service_twitter = $31, service_wordpress = $32, weightedscore = $33, service_xmpp = $34, softwarename = $35, sslvalid = $36, uptime_custom = $37
279
  WHERE domain = $1';
noplanman's avatar
noplanman committed
280
    $result  = pg_query_params($dbh, $sql, [$domain, $secure, $hidden, $ip, $ipv6, $months, $uptime, $status, $statslastdate, $timenow, $responsetime, $score, $admin_rating, $country, $city, $state, $lat, $long, $user_rating, $shortversion, $masterversion, $signup, $total_users, $active_users_halfyear, $active_users_monthly, $local_posts, $name, $comment_counts, $service_facebook, $service_tumblr, $service_twitter, $service_wordpress, $weightedscore, $service_xmpp, $softwarename, $outputsslerror, $uptime_custom]);
281 282
    $result || die('Error in SQL query3: ' . pg_last_error());

283 284
    if ($debug) {
      echo '<br>Score out of 100: ' . $score . '<br>';
noplanman's avatar
noplanman committed
285
    } else {
286 287 288
      echo 'Success';
    }
    //end foreach
dmorley's avatar
cleanup  
dmorley committed
289
  }
290
}