pull.php 12.1 KB
Newer Older
MatrixCrawler's avatar
MatrixCrawler committed
1
<?php
2 3
//$debug = isset($_GET['debug']);
$debug = true;
4 5
//$debug = isset($argv[1])?1:0;
//* Copyright (c) 2011, David Morley. This file is licensed under the Affero General Public License version 3 or later. See the COPYRIGHT file. */
6 7 8 9

// Other parameters.
$_domain = $_GET['domain'] ?? '';

10
require_once __DIR__ . '/../config.php';
dmorley's avatar
cleanup  
dmorley committed
11

dmorley's avatar
dmorley committed
12
//get master code version for diaspora pods
dmorley's avatar
cleanup  
dmorley committed
13
$mv = curl_init();
14
curl_setopt($mv, CURLOPT_URL, 'https://raw.githubusercontent.com/diaspora/diaspora/master/config/defaults.yml');
dmorley's avatar
cleanup  
dmorley committed
15 16 17 18 19 20 21
curl_setopt($mv, CURLOPT_POST, 0);
curl_setopt($mv, CURLOPT_HEADER, 0);
curl_setopt($mv, CURLOPT_CONNECTTIMEOUT, 5);
curl_setopt($mv, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($mv, CURLOPT_NOBODY, 0);
$outputmv = curl_exec($mv);
curl_close($mv);
22
preg_match('/number: "(.*?)"/', $outputmv, $version);
dmorley's avatar
cleanup  
dmorley committed
23
$dmasterversion = trim($version[1], '"');
24 25 26
if ($debug) {
  echo 'Diaspora Masterversion: ' . $dmasterversion . '<br>';
}
dmorley's avatar
cleanup  
dmorley committed
27

dmorley's avatar
dmorley committed
28
//get master code version for freindica pods
dmorley's avatar
cleanup  
dmorley committed
29
$mv = curl_init();
30
curl_setopt($mv, CURLOPT_URL, 'https://raw.githubusercontent.com/friendica/friendica/master/boot.php');
dmorley's avatar
cleanup  
dmorley committed
31 32 33 34 35 36 37
curl_setopt($mv, CURLOPT_POST, 0);
curl_setopt($mv, CURLOPT_HEADER, 0);
curl_setopt($mv, CURLOPT_CONNECTTIMEOUT, 5);
curl_setopt($mv, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($mv, CURLOPT_NOBODY, 0);
$outputmv = curl_exec($mv);
curl_close($mv);
38
preg_match('/FRIENDICA_VERSION\',      \'(.*?)\'/', $outputmv, $version);
dmorley's avatar
cleanup  
dmorley committed
39
$fmasterversion = trim($version[1], '"');
40 41 42
if ($debug) {
  echo 'Frendica Masterversion: ' . $fmasterversion . '<br>';
}
43

44 45
$dbh = pg_connect("dbname=$pgdb user=$pguser password=$pgpass");
$dbh || die('Error in connection: ' . pg_last_error());
MatrixCrawler's avatar
MatrixCrawler committed
46

dmorley's avatar
cleanup  
dmorley committed
47
//foreach pod check it and update db
48
if ($_domain) {
dmorley's avatar
dmorley committed
49
  $sql    = 'SELECT domain,stats_apikey,score,date_created,weight FROM pods WHERE domain = $1';
noplanman's avatar
noplanman committed
50
  //$sleep  = '0';
51 52
  $result = pg_query_params($dbh, $sql, [$_domain]);
} elseif (PHP_SAPI === 'cli') {
dmorley's avatar
dmorley committed
53
  $sql    = 'SELECT domain,stats_apikey,score,date_created,adminrating,weight FROM pods';
noplanman's avatar
noplanman committed
54
  //$sleep  = '1';
dmorley's avatar
cleanup  
dmorley committed
55
  $result = pg_query($dbh, $sql);
David Morley's avatar
David Morley committed
56 57
} else {
  die('No valid input');
dmorley's avatar
cleanup  
dmorley committed
58
}
59 60
$result || die('Error in SQL query1: ' . pg_last_error());

dmorley's avatar
cleanup  
dmorley committed
61 62
while ($row = pg_fetch_all($result)) {
  $numrows = pg_num_rows($result);
63
  for ($i = 0; $i < $numrows; $i++) {
64
    $domain    = $row[$i]['domain'];
65
    $score     = (int) $row[$i]['score'];
dmorley's avatar
dmorley committed
66
    $dateadded = $row[$i]['date_created'];
67
    $admindb   = (int) $row[$i]['adminrating'];
68
    $weight    = $row[$i]['weight'];
noplanman's avatar
noplanman committed
69 70
    $sqlforr       = 'SELECT admin,rating FROM rating_comments WHERE domain = $1';
    $ratings       = pg_query_params($dbh, $sqlforr, [$domain]);
71 72
    $ratings || die('Error in SQL query2: ' . pg_last_error());

73 74 75
    if ($debug) {
      echo 'Domain: ' . $domain . '<br>';
    }
noplanman's avatar
noplanman committed
76 77 78 79 80 81 82 83 84

    $user_ratings  = [];
    $admin_ratings = [];
    while ($rating = pg_fetch_assoc($ratings)) {
      if ($rating['admin'] == 0) {
        $user_ratings[] = $rating['rating'];
      } elseif ($rating['admin'] == 1) {
        $admin_ratings[] = $rating['rating'];
      }
85
    }
noplanman's avatar
noplanman committed
86 87 88
    $user_rating  = empty($user_ratings) ? 0 : max(10, round(array_sum($user_ratings) / count($user_ratings), 2));
    $admin_rating = empty($admin_ratings) ? 0 : max(10, round(array_sum($admin_ratings) / count($admin_ratings), 2));

89
    if ($admindb == - 1) {
noplanman's avatar
noplanman committed
90
      $admin_rating = - 1;
91 92 93 94 95 96 97 98 99 100 101 102
    }
    unset($name);
    unset($total_users);
    unset($active_users_halfyear);
    unset($active_users_monthly);
    unset($local_posts);
    unset($registrations_open);
    unset($comment_counts);
    unset($service_facebook);
    unset($service_twitter);
    unset($service_tumblr);
    unset($service_wordpess);
103
    unset($service_xmpp);
104
    unset($shortversion);
105 106 107 108
    unset($dverr);
    unset($xdver);
    unset($softwarename);
    unset($outputsslerror);
109

110 111 112 113 114 115 116 117
    $chss = curl_init();
    curl_setopt($chss, CURLOPT_URL, 'https://' . $domain . '/nodeinfo/1.0');
    curl_setopt($chss, CURLOPT_POST, 0);
    curl_setopt($chss, CURLOPT_HEADER, 0);
    curl_setopt($chss, CURLOPT_CONNECTTIMEOUT, 9);
    curl_setopt($chss, CURLOPT_TIMEOUT, 9);
    curl_setopt($chss, CURLOPT_RETURNTRANSFER, 1);
    curl_setopt($chss, CURLOPT_NOBODY, 0);
118 119
    curl_setopt($chss, CURLOPT_CERTINFO, 1);
    curl_setopt($chss, CURLOPT_VERBOSE, 1);
120 121
    $outputssl      = curl_exec($chss);
    $outputsslerror = curl_error($chss);
122 123
    $info = curl_getinfo($chss, CURLINFO_CERTINFO);
    $sslexpire = $info[0]['Expire date'];
124
    curl_close($chss);
125

126 127 128
    if ($debug) {
      var_dump($outputssl);
    }
dmorley's avatar
dmorley committed
129
    if (!$outpulssl && !$domain) {
130 131
      continue;
      echo 'no connection to pod';
132
      
dmorley's avatar
dmorley committed
133
      $sql    = 'INSERT INTO checks (domain, online, error) VALUES ($1, $2, $3)';
134 135 136
      $result = pg_query_params($dbh, $sql, [$domain, false, $outputsslerror]);
      $result || die('Error in SQL query: ' . pg_last_error());
      
137 138 139 140
    }
    if ($outputssl) {
      $secure        = 'true';
      $outputresults = $outputssl;
141 142 143 144 145
      
      $sql    = 'INSERT INTO checks (domain, online) VALUES ($1, $2)';
      $result = pg_query_params($dbh, $sql, [$domain, true]);
      $result || die('Error in SQL query: ' . pg_last_error());
  
dmorley's avatar
dmorley committed
146
    } else {
147 148 149 150
      $secure        = 'false';
      $outputresults = $output;
    }
    if (stristr($outputresults, 'openRegistrations')) {
noplanman's avatar
noplanman committed
151
      $score += 1;
152 153 154 155 156 157 158 159 160
      if ($debug) {
        echo 'Secure: ' . $secure . '<br>';
      }
      //get new json from nodeinfo
      $jsonssl = json_decode($outputresults);
      var_dump($jsonssl);
      if ($jsonssl->openRegistrations === true) {
        $registrations_open = 1;
      }
161
      $xdver = $jsonssl->software->version ?? 0;
162
      $dverr = explode('-', trim($xdver));
163
      $shortversion  = $dverr[0];
164
      if ($debug) {
165
        echo ' <br> Version code: ' . $shortversion . '<br>';
166
      }
noplanman's avatar
noplanman committed
167 168
      $shortversion || $score -= 2;

169 170 171 172 173 174 175 176 177 178 179 180
      $softwarename          = $jsonssl->software->name ?? 'null';
      $name                  = $jsonssl->metadata->nodeName ?? 'null';
      $total_users           = $jsonssl->usage->users->total ?? 0;
      $active_users_halfyear = $jsonssl->usage->users->activeHalfyear ?? 0;
      $active_users_monthly  = $jsonssl->usage->users->activeMonth ?? 0;
      $local_posts           = $jsonssl->usage->localPosts ?? 0;
      $comment_counts        = $jsonssl->usage->localComments ?? 0;
      $service_facebook      = in_array('facebook', $jsonssl->services->outbound, true) ? 'true' : 'false';
      $service_twitter       = in_array('twitter', $jsonssl->services->outbound, true) ? 'true' : 'false';
      $service_tumblr        = in_array('tumblr', $jsonssl->services->outbound, true) ? 'true' : 'false';
      $service_wordpress     = in_array('wordpress', $jsonssl->services->outbound, true) ? 'true' : 'false';
      $service_xmpp          = $jsonssl->metadata->xmppChat === true ? 'true' : 'false';
181 182
    } else {
      $secure = 'false';
noplanman's avatar
noplanman committed
183
      $score -= 1;
184
      $dver   = '.connect error';
185
      $shortversion  = 0;
186 187 188 189 190 191
      //could also be a ssl pod with a bad cert, I think its ok to call that a dead pod now
    }
    $signup = $registrations_open;
    if ($debug) {
      echo '<br>Signup Open: ' . $signup . '<br>';
    }
noplanman's avatar
noplanman committed
192
    $ip6    = exec(escapeshellcmd('dig +nocmd ' . $domain . ' aaaa +noall +short'));
193 194 195 196 197
    exec(escapeshellcmd('delv ' . $domain), $iplookup);
    $dnssec = in_array('; fully validated', $iplookup) ? 't' : 'f' ;
    preg_match('/A(.*)/', $iplookup[1], $version);
    $ip = trim($version[1]);
    $ipv6   = strpos($ip6, ':') === false ? 'no' : 'yes';
noplanman's avatar
noplanman committed
198

199
    if ($debug) {
200
      echo 'IP: ' . $ip . '<br>';
201
    }
202
    $location = geoip_record_by_name($ip);
203 204 205 206 207 208
    if ($debug) {
      echo ' Location: ';
      var_dump($location);
      echo '<br>';
    }
    if ($location) {
David Morley's avatar
David Morley committed
209 210 211
      $country = !empty($location['country_code']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['country_code']) : null;
      $city    = !empty($location['city']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['city']) : null;
      $state   = !empty($location['region']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['region']) : null;
noplanman's avatar
noplanman committed
212
      $lat     = !empty($location['latitude']) ? $location['latitude'] : null;
David Morley's avatar
David Morley committed
213
      $long    = !empty($location['longitude']) ? $location['longitude'] : null;
noplanman's avatar
noplanman committed
214 215 216 217 218

      // If lat and long are just a generic country with no detail lets make some tail up
      // else openmap just stacks them all on top of each other.
      strlen($lat) < 4 && $lat += (random_int(1, 15) / 10);
      strlen($long) < 4 && $long += (random_int(1, 15) / 10);
219 220
    }
    echo '<br>';
221
    $statslastdate = date('Y-m-d H:i:s');
222
      $ping = curl_init();
noplanman's avatar
noplanman committed
223
      curl_setopt($ping, CURLOPT_URL, 'https://api.uptimerobot.com/getMonitors?format=json&noJsonCallback=1&customUptimeRatio=7-30-60-90&responseTimes=1&responseTimesAverage=86400&apiKey=' . $row[$i]['stats_apikey']);
224 225 226 227 228 229 230
      curl_setopt($ping, CURLOPT_POST, 0);
      curl_setopt($ping, CURLOPT_HEADER, 0);
      curl_setopt($ping, CURLOPT_RETURNTRANSFER, 1);
      curl_setopt($ping, CURLOPT_CONNECTTIMEOUT, 8);
      curl_setopt($ping, CURLOPT_NOBODY, 0);
      curl_setopt($ping, CURLOPT_MAXCONNECTS, 5);
      curl_setopt($ping, CURLOPT_FOLLOWLOCATION, true);
noplanman's avatar
noplanman committed
231
      $uptr = json_decode(curl_exec($ping));
232 233 234 235 236
      curl_close($ping);
      if ($debug) {
        print_r($uptr);
        echo '<br>';
      }
noplanman's avatar
noplanman committed
237 238
      $uptr || $score -= 2;

239 240 241
      $responsetime    = $uptr->monitors->monitor{'0'}->responsetime{'0'}->value;
      $uptimerobotstat = $uptr->stat;
      $uptime          = $uptr->monitors->monitor{'0'}->alltimeuptimeratio;
dmorley's avatar
dmorley committed
242
      $uptime_custom   = $uptr->monitors->monitor{'0'}->customuptimeratio;
David Morley's avatar
David Morley committed
243 244
      $diff            = (new DateTime())->diff(new DateTime($dateadded));
      $months          = $diff->m + ($diff->y * 12);
245
      if ($uptr->monitors->monitor{'0'}->status == 2) {
dmorley's avatar
dmorley committed
246
        $status = 'Up';
247 248
      }
      if ($uptr->monitors->monitor{'0'}->status == 0) {
dmorley's avatar
dmorley committed
249
        $status = 'Paused';
250 251
      }
      if ($uptr->monitors->monitor{'0'}->status == 1) {
dmorley's avatar
dmorley committed
252
        $status = 'Not Checked Yet';
253 254
      }
      if ($uptr->monitors->monitor{'0'}->status == 8) {
dmorley's avatar
dmorley committed
255
        $status = 'Seems Down';
256 257
      }
      if ($uptr->monitors->monitor{'0'}->status == 9) {
dmorley's avatar
dmorley committed
258
        $status = 'Down';
259
      }
dmorley's avatar
cleanup  
dmorley committed
260
      if ($uptr) {
dmorley's avatar
dmorley committed
261
        $statslastdate = date('Y-m-d H:i:s');
dmorley's avatar
cleanup  
dmorley committed
262
      }
noplanman's avatar
noplanman committed
263 264 265
      ($uptimerobotstat !== 'fail' && $status === 'Up') || $score -= 2;

    if ($softwarename === 'diaspora') {
266
      $masterversion = $dmasterversion;
noplanman's avatar
noplanman committed
267
    } elseif ($softwarename === 'friendica') {
268 269
      $masterversion = $fmasterversion;
    }
noplanman's avatar
noplanman committed
270
    $hidden = $score > 70 ? 'no' : 'yes';
271 272 273 274 275 276 277 278 279 280 281
    if ($debug) {
      echo 'Hidden: ' . $hidden . '<br>';
    }
    // lets cap the scores or you can go too high or too low to never be effected by them
    if ($score > 100) {
      $score = 100;
    } elseif ($score < 0) {
      $score = 0;
    }
    $weightedscore = ($uptime + $score + ($active_users_monthly / 19999) - ((10 - $weight) * .12));
    //sql it
MatrixCrawler's avatar
MatrixCrawler committed
282

283
    $timenow = date('Y-m-d H:i:s');
284
    $sql     = 'UPDATE pods SET secure = $2, hidden = $3, ip = $4, ipv6 = $5, monthsmonitored = $6, uptime_alltime = $7, status = $8, date_laststats = $9, date_updated = $10, responsetime = $11, score = $12, adminrating = $13, country = $14, city = $15, state = $16, lat = $17, long = $18, userrating = $19, shortversion = $20, masterversion = $21, signup = $22, total_users = $23, active_users_halfyear = $24, active_users_monthly = $25, local_posts = $26, name = $27, comment_counts = $28, service_facebook = $29, service_tumblr = $30, service_twitter = $31, service_wordpress = $32, weightedscore = $33, service_xmpp = $34, softwarename = $35, sslvalid = $36, uptime_custom = $37, dnssec = $38, sslexpire = $39
285
  WHERE domain = $1';
286 287 288

    $result  = pg_query_params($dbh, $sql, [$domain, $secure, $hidden, $ip, $ipv6, $months, $uptime, $status, $statslastdate, $timenow, $responsetime, $score, $admin_rating, $country, $city, $state, $lat, $long, $user_rating, $shortversion, $masterversion, $signup, $total_users, $active_users_halfyear, $active_users_monthly, $local_posts, $name, $comment_counts, $service_facebook, $service_tumblr, $service_twitter, $service_wordpress, $weightedscore, $service_xmpp, $softwarename, $outputsslerror, $uptime_custom, $dnssec, $sslexpire]);

289 290
    $result || die('Error in SQL query3: ' . pg_last_error());

291 292
    if ($debug) {
      echo '<br>Score out of 100: ' . $score . '<br>';
noplanman's avatar
noplanman committed
293
    } else {
294 295 296
      echo 'Success';
    }
    //end foreach
dmorley's avatar
cleanup  
dmorley committed
297
  }
298
}