pull.php 14.6 KB
Newer Older
MatrixCrawler's avatar
MatrixCrawler committed
1
<?php
dmorley's avatar
dmorley committed
2 3
$debug = isset($_GET['debug'])?1:0;
$debug=1;
4 5
//$debug = isset($argv[1])?1:0;
//* Copyright (c) 2011, David Morley. This file is licensed under the Affero General Public License version 3 or later. See the COPYRIGHT file. */
6
require_once __DIR__ . '/../config.php';
dmorley's avatar
cleanup  
dmorley committed
7

dmorley's avatar
dmorley committed
8
//get master code version for diaspora pods
dmorley's avatar
cleanup  
dmorley committed
9
$mv = curl_init();
10
curl_setopt($mv, CURLOPT_URL, 'https://raw.githubusercontent.com/diaspora/diaspora/master/config/defaults.yml');
dmorley's avatar
cleanup  
dmorley committed
11 12 13 14 15 16 17 18 19
curl_setopt($mv, CURLOPT_POST, 0);
curl_setopt($mv, CURLOPT_HEADER, 0);
curl_setopt($mv, CURLOPT_CONNECTTIMEOUT, 5);
curl_setopt($mv, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($mv, CURLOPT_NOBODY, 0);
$outputmv = curl_exec($mv);
curl_close($mv);
preg_match('/number: "(.*?)"/',$outputmv,$version);
$dmasterversion = trim($version[1], '"');
20
if ($debug) {echo 'Diaspora Masterversion: ' . $dmasterversion . '<br>';}
dmorley's avatar
cleanup  
dmorley committed
21

dmorley's avatar
dmorley committed
22
//get master code version for freindica pods
dmorley's avatar
cleanup  
dmorley committed
23
$mv = curl_init();
24
curl_setopt($mv, CURLOPT_URL, 'https://raw.githubusercontent.com/friendica/friendica/master/boot.php');
dmorley's avatar
cleanup  
dmorley committed
25 26 27 28 29 30 31 32 33
curl_setopt($mv, CURLOPT_POST, 0);
curl_setopt($mv, CURLOPT_HEADER, 0);
curl_setopt($mv, CURLOPT_CONNECTTIMEOUT, 5);
curl_setopt($mv, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($mv, CURLOPT_NOBODY, 0);
$outputmv = curl_exec($mv);
curl_close($mv);
preg_match('/FRIENDICA_VERSION\',      \'(.*?)\'/',$outputmv,$version);
$fmasterversion = trim($version[1], '"');
34
if ($debug) {echo 'Frendica Masterversion: ' . $fmasterversion . '<br>';}
dmorley's avatar
cleanup  
dmorley committed
35 36 37
$dbh = pg_connect("dbname=$pgdb user=$pguser password=$pgpass");
$dbh2 = pg_connect("dbname=$pgdb user=$pguser password=$pgpass"); 
if (!$dbh) {
38
  die('Error in connection: ' . pg_last_error());
dmorley's avatar
cleanup  
dmorley committed
39
}
MatrixCrawler's avatar
MatrixCrawler committed
40

dmorley's avatar
cleanup  
dmorley committed
41 42 43 44
//foreach pod check it and update db
$domain = isset($_GET['domain'])?$_GET['domain']:null;
if ($domain) {
  $sql = "SELECT domain,pingdomurl,score,datecreated,weight FROM pods WHERE domain = $1";
45
  $sleep= '0';
dmorley's avatar
cleanup  
dmorley committed
46 47
  $result = pg_query_params($dbh, $sql, array($domain));
} else {
48 49
  $sql = 'SELECT domain,pingdomurl,score,datecreated,adminrating,weight FROM pods';
  $sleep= '1';
dmorley's avatar
cleanup  
dmorley committed
50 51
  $result = pg_query($dbh, $sql);
}
52
if (!$result) {die('Error in SQL query1: ' . pg_last_error());}
dmorley's avatar
cleanup  
dmorley committed
53 54 55 56
 
while ($row = pg_fetch_all($result)) {
  $numrows = pg_num_rows($result);
   for ($i = 0; $i < $numrows; $i++) {
57 58 59 60
     $domain =  $row[$i]['domain'];
     $score = $row[$i]['score'];
     $dateadded = $row[$i]['datecreated'];
     $admindb = $row[$i]['adminrating'];
61
     $weight = $row[$i]['weight'];
dmorley's avatar
cleanup  
dmorley committed
62 63 64 65 66
     //get ratings
     $userrate=0;$adminrate=0;$userratingavg = array();$adminratingavg = array();$userrating = array();$adminrating = array();
     $sqlforr = "SELECT * FROM rating_comments WHERE domain = $1";
     $ratings = pg_query_params($dbh, $sqlforr, array($domain));
     if (!$ratings) {
67
       die('Error in SQL query2: ' . pg_last_error());
dmorley's avatar
cleanup  
dmorley committed
68 69 70 71 72 73 74 75 76
     }
     $numratings = pg_num_rows($ratings);
     while($myrow = pg_fetch_assoc($ratings)) {
       if ($myrow['admin'] == 0) {
         $userratingavg[] = $myrow['rating'];$userrate++;
       } elseif ($myrow['admin'] == 1) {
         $adminratingavg[] = $myrow['rating'];$adminrate++;
       }  
     }
MatrixCrawler's avatar
MatrixCrawler committed
77

dmorley's avatar
cleanup  
dmorley committed
78 79
     if ($userrate > 0) {$userrating = round(array_sum($userratingavg) / $userrate,2);}
     if ($adminrate > 0) {$adminrating = round(array_sum($adminratingavg) / $adminrate,2);}
80
     if ($debug) {echo 'Domain: ' . $domain . '<br>';}
dmorley's avatar
cleanup  
dmorley committed
81 82 83 84 85
     if (!$userrating) {$userrating=0;}
     if ($userrating > 10) {$userrating=10;}
     if (!$adminrating) {$adminrating=0;}
     if ($adminrating > 10) {$adminrating=10;}
     if ($admindb == -1) {$adminrating=-1;}
86
     pg_free_result($ratings);
dmorley's avatar
cleanup  
dmorley committed
87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105
     $userrate=0;$adminrate=0;
     unset($userratingavg);
     unset($adminratingavg);
     unset($name);
     unset($total_users);
     unset($active_users_halfyear);
     unset($active_users_monthly);
     unset($local_posts);
     unset($registrations_open);
     unset($comment_counts);
     unset($service_facebook);
     unset($service_twitter);
     unset($service_tumblr);
     unset($service_wordpess);
     unset($dver);
     unset($dverr);
     unset($xdver);
     unset($xmpp);
     unset($softwarename);
dmorley's avatar
dmorley committed
106
     unset($outputsslerror);
dmorley's avatar
cleanup  
dmorley committed
107
     $chss = curl_init();
108
     curl_setopt($chss, CURLOPT_URL, 'https://' . $domain . '/nodeinfo/1.0');
dmorley's avatar
cleanup  
dmorley committed
109
     curl_setopt($chss, CURLOPT_POST, 0);
David Morley's avatar
David Morley committed
110 111 112
     curl_setopt($chss, CURLOPT_HEADER, 0);
     curl_setopt($chss, CURLOPT_CONNECTTIMEOUT, 9);
     curl_setopt($chss, CURLOPT_TIMEOUT, 9);
dmorley's avatar
dmorley committed
113
     curl_setopt($chss, CURLOPT_RETURNTRANSFER, 1);
dmorley's avatar
cleanup  
dmorley committed
114 115
     curl_setopt($chss, CURLOPT_NOBODY, 0);
     $outputssl = curl_exec($chss);      
dmorley's avatar
dmorley committed
116
     $outputsslerror = curl_error($chss);
dmorley's avatar
cleanup  
dmorley committed
117
     curl_close($chss);
118

dmorley's avatar
cleanup  
dmorley committed
119
     $ch = curl_init();
120
     curl_setopt($ch, CURLOPT_URL, 'http://' . $domain . '/nodeinfo/1.0');
dmorley's avatar
cleanup  
dmorley committed
121 122
     curl_setopt($ch, CURLOPT_POST, 0);
     curl_setopt($ch, CURLOPT_HEADER, 0);
David Morley's avatar
David Morley committed
123 124
     curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 9);
     curl_setopt($ch, CURLOPT_TIMEOUT, 9);
dmorley's avatar
cleanup  
dmorley committed
125 126 127 128
     curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
     curl_setopt($ch, CURLOPT_NOBODY, 0);
     $output = curl_exec($ch);
     curl_close($ch);
129 130 131 132
     if ($debug) {echo 'not-e'; print $output;}
     if ($debug) {echo 'e'; var_dump($outputssl);}
     if (!$output && !$outpulssl && !$domain) {continue;echo 'no connection to pod';}
     if ($outputssl) {$secure = 'true';$outputresults =$outputssl;} elseif ($output) {$secure = 'false';$outputresults =$output;}
dmorley's avatar
cleanup  
dmorley committed
133 134
     if (stristr($outputresults, 'openRegistrations')) {
       $score = $score +1;
135
       if ($debug) {echo 'Secure: ' . $secure . '<br>';}
dmorley's avatar
cleanup  
dmorley committed
136 137 138 139 140
       //get new json from nodeinfo
       $jsonssl = json_decode($outputresults);
       var_dump($jsonssl);
       if ($jsonssl->openRegistrations === true) {$registrations_open=1;}
       $xdver = isset($jsonssl->software->version)?$jsonssl->software->version:0;
141
       $dverr = explode('-',trim($xdver));
dmorley's avatar
cleanup  
dmorley committed
142
       $dver = $dverr[0];
143
       if ($debug) {echo ' <br> Version code: ' . $dver . '<br>';}
dmorley's avatar
cleanup  
dmorley committed
144
       if (!$dver) {$score = $score-2;}
145 146
       $softwarename = isset($jsonssl->software->name)?$jsonssl->software->name: 'null';
       $name = isset($jsonssl->metadata->nodeName)?$jsonssl->metadata->nodeName: 'null';
dmorley's avatar
cleanup  
dmorley committed
147 148 149 150 151 152 153 154 155 156 157
       $total_users = isset($jsonssl->usage->users->total)?$jsonssl->usage->users->total:0;
       $active_users_halfyear = isset($jsonssl->usage->users->activeHalfyear)?$jsonssl->usage->users->activeHalfyear:0;
       $active_users_monthly = isset($jsonssl->usage->users->activeMonth)?$jsonssl->usage->users->activeMonth:0;
       $local_posts = isset($jsonssl->usage->localPosts)?$jsonssl->usage->localPosts:0;
       $comment_counts = isset($jsonssl->usage->localComments)?$jsonssl->usage->localComments:0;
       if (array_search('facebook', $jsonssl->services->outbound) !== false) {$service_facebook='true';} else {$service_facebook='false';}
       if (array_search('twitter', $jsonssl->services->outbound) !== false) {$service_twitter='true';} else {$service_twitter='false';}
       if (array_search('tumblr', $jsonssl->services->outbound) !== false) {$service_tumblr='true';} else {$service_tumblr='false';}
       if (array_search('wordpress', $jsonssl->services->outbound) !== false) {$service_wordpress='true';} else {$service_wordpress='false';}
       if ($jsonssl->metadata->xmppChat === true) {$xmpp = 'true';} else {$xmpp = 'false';}
     } else {
158
     $secure= 'false';
dmorley's avatar
cleanup  
dmorley committed
159
     $score = $score - 1;
160
     $dver = '.connect error';
dmorley's avatar
cleanup  
dmorley committed
161 162 163 164 165
     $dverr=0;
     //no diaspora cookie on either, lets set this one as hidden and notify someone its not really a pod
     //could also be a ssl pod with a bad cert, I think its ok to call that a dead pod now
     }
     $signup = $registrations_open;
166
     if ($debug) {echo '<br>Signup Open: ' . $signup . '<br>';}
dmorley's avatar
cleanup  
dmorley committed
167 168 169 170
     $ip6 = escapeshellcmd('dig +nocmd '.$domain.' aaaa +noall +short');
     $ip = escapeshellcmd('dig +nocmd '.$domain.' a +noall +short');
     $ip6num = exec($ip6);
     $ipnum = exec($ip);
171
     $test = strpos($ip6num, ':');
dmorley's avatar
cleanup  
dmorley committed
172
     if ($test === false) {
173
       $ipv6= 'no';
dmorley's avatar
cleanup  
dmorley committed
174
     } else {
175
       $ipv6= 'yes';
dmorley's avatar
cleanup  
dmorley committed
176
     }
177
     if ($debug) {echo 'IP: ' . $ipnum . '<br>';}
dmorley's avatar
cleanup  
dmorley committed
178
     $location = geoip_record_by_name($ipnum);
179
     if ($debug) {echo ' Location: '; var_dump($location); echo '<br>';}
dmorley's avatar
cleanup  
dmorley committed
180
     if ($location) {
181 182 183 184 185
       $ipdata = 'Country: ' . $location['country_name'] . "\n";
       $whois = 'Country: ' . $location['country_name'] . "\n Lat:" . $location['latitude'] . ' Long:' . $location['longitude'];
       $country=$location['country_code'];
       $city=isset($location->city)?iconv('UTF-8', 'UTF-8//IGNORE', $location->city):null;
       $state= '';
dmorley's avatar
cleanup  
dmorley committed
186 187
       $months=0;
       $uptime=0;
188 189
       $lat=$location['latitude'];
       $long=$location['longitude'];
dmorley's avatar
cleanup  
dmorley committed
190 191 192 193 194 195 196 197
       //if lat and long are just a generic country with no detail lets make some tail up or openmap just stacks them all on top another
       if (strlen($lat) < 4) {
         $lat = $lat + (rand(1, 15) / 10);
       }
       if (strlen($long) < 4) {
         $long = $long + (rand(1, 15) / 10);
       }
     }
198 199
     echo '<br>';
     $connection= '';
dmorley's avatar
cleanup  
dmorley committed
200
     $pingdomdate = date('Y-m-d H:i:s');
201
     if (strpos($row[$i]['pingdomurl'], 'pingdom.com')) {
dmorley's avatar
cleanup  
dmorley committed
202
     //curl the pingdom page 
203
        $ping = curl_init();
204
        $thismonth = '/' . date('Y') . '/' . date('m');
205 206 207 208 209 210 211 212 213 214 215 216
        curl_setopt($ping, CURLOPT_URL, $row[$i]['pingdomurl'].$thismonth);
	if ($debug) {echo $row[$i]['pingdomurl'].$thismonth;}
        curl_setopt($ping, CURLOPT_POST, 0);
        curl_setopt($ping, CURLOPT_HEADER, 1);
        curl_setopt($ping, CURLOPT_RETURNTRANSFER, 1);
        curl_setopt($ping, CURLOPT_CONNECTTIMEOUT, 8);
        curl_setopt($ping, CURLOPT_NOBODY, 0);
        curl_setopt($ping, CURLOPT_MAXCONNECTS, 5);
        curl_setopt($ping, CURLOPT_FOLLOWLOCATION, true);
        $pingdom = curl_exec($ping);
	$info = curl_getinfo($ping);
        curl_close($ping);
217
        if ($debug) {echo '<br>Pingdom code: ' . $info['http_code'] . '<br>';}
dmorley's avatar
cleanup  
dmorley committed
218 219 220 221 222 223 224
        if ($info['http_code'] == 200) {
          //response time
          preg_match_all('/<h3>Avg. resp. time this month<\/h3>
          <p class="large">(.*?)</',$pingdom,$matcheach);
          $responsetime = $matcheach[1][0];
          //months monitored
          preg_match_all('/"historySelect">\s*(.*?)\s*<\/select/is',$pingdom,$matchhistory);
225
          $implodemonths = implode(' ', $matchhistory[1]);
dmorley's avatar
cleanup  
dmorley committed
226 227 228 229
          preg_match_all('/<option(.*?)/s',$implodemonths,$matchdates);
          $months = isset($matchdates[0])?count($matchdates[0]):0;
          //uptime %
          preg_match_all('/<h3>Uptime this month<\/h3>\s*<p class="large">(.*?)%</',$pingdom,$matchper);
230
          $uptime = isset($matchper[1][0])?preg_replace('/,/', '.', $matchper[1][0]):0;
dmorley's avatar
cleanup  
dmorley committed
231
          $pingdomdate = date('Y-m-d H:i:s');
232 233 234 235
          if (strpos($pingdom,"class=\"up\"")) { $live= 'up'; }
          elseif (strpos($pingdom,"class=\"down\"")) { $live= 'down'; }
          elseif (strpos($pingdom,"class=\"paused\"")) { $live= 'paused';}
          else {$live = 'error';$score = $score - 2;}
dmorley's avatar
cleanup  
dmorley committed
236 237 238 239 240 241 242
        } else {
        //pingdom url is <> 200 so stats are gone, lower score
        $score=$score-2;
      }
    } else {
    //do uptimerobot API instead
    $ping = curl_init();
243
    curl_setopt($ping, CURLOPT_URL, 'http://api.uptimerobot.com/getMonitors?format=json&customUptimeRatio=7-30-60-90&responseTimes=1&responseTimesAverage=86400&apiKey=' . $row[$i]['pingdomurl']);
dmorley's avatar
cleanup  
dmorley committed
244 245 246 247 248 249 250 251 252
    curl_setopt($ping, CURLOPT_POST, 0);
    curl_setopt($ping, CURLOPT_HEADER, 0);
    curl_setopt($ping, CURLOPT_RETURNTRANSFER, 1);
    curl_setopt($ping, CURLOPT_CONNECTTIMEOUT, 8);
    curl_setopt($ping, CURLOPT_NOBODY, 0);
    curl_setopt($ping, CURLOPT_MAXCONNECTS, 5);
    curl_setopt($ping, CURLOPT_FOLLOWLOCATION, true);
    $uptimerobot = curl_exec($ping);
    curl_close($ping);
253
    $json_encap = 'jsonUptimeRobotApi()';
dmorley's avatar
cleanup  
dmorley committed
254 255
    $up2 = substr ($uptimerobot, strlen($json_encap) - 1, strlen ($uptimerobot) - strlen($json_encap)); 
    $uptr = json_decode($up2);
256
    if ($debug) {print_r($uptr);echo '<br>';}
dmorley's avatar
cleanup  
dmorley committed
257 258 259 260 261 262
    if (!$uptr) {$score=$score-2;}
    $responsetime = $uptr->monitors->monitor{'0'}->responsetime{'0'}->value;
    $uptimerobotstat = $uptr->stat;
    $uptime = $uptr->monitors->monitor{'0'}->alltimeuptimeratio;
    $diff = abs(strtotime(date('Y-m-d H:i:s')) - strtotime($dateadded));
    $months = floor(($diff - $years * 365*60*60*24) / (30*60*60*24));
263 264 265 266 267
    if ($uptr->monitors->monitor{'0'}->status == 2) {$live = 'Up';}
    if ($uptr->monitors->monitor{'0'}->status == 0) {$live = 'Paused';}
    if ($uptr->monitors->monitor{'0'}->status == 1) {$live = 'Not Checked Yet';}
    if ($uptr->monitors->monitor{'0'}->status == 8) {$live = 'Seems Down';}
    if ($uptr->monitors->monitor{'0'}->status == 9) {$live = 'Down';}
dmorley's avatar
cleanup  
dmorley committed
268
    $pingdomdate =  date('Y-m-d H:i:s');
269
    if ($uptimerobotstat == 'fail' || $live <> 'Up') {
dmorley's avatar
cleanup  
dmorley committed
270 271 272
      $score=$score-2;
    }
  }
273
  if ($softwarename == 'diaspora') {$masterversion = $dmasterversion;} elseif ($softwarename == 'friendica') {$masterversion = $fmasterversion;}
dmorley's avatar
dmorley committed
274
     if ($score > 70) {
275
       $hidden = 'no';
276
     } else {
277
       $hidden = 'yes';
278
     }
279
     if ($debug) {echo 'Hidden: ' . $hidden . '<br>';}
280 281 282 283 284 285
     // lets cap the scores or you can go too high or too low to never be effected by them
     if ($score > 100) {
       $score = 100;
     } elseif ($score < 0) {
       $score = 0;
     }
dmorley's avatar
cleanup  
dmorley committed
286 287
  $weightedscore = ($uptime + $score + ($active_users_monthly/19999) - ((10 - $weight) *.12));
  //sql it
MatrixCrawler's avatar
MatrixCrawler committed
288

dmorley's avatar
cleanup  
dmorley committed
289 290 291 292 293
  $timenow = date('Y-m-d H:i:s');
  $sql = "UPDATE pods SET Hgitdate=$1, Hencoding=$2, secure=$3, hidden=$4, Hruntime=$5, Hgitref=$6, ip=$7, ipv6=$8, monthsmonitored=$9, 
  uptimelast7=$10, status=$11, dateLaststats=$12, dateUpdated=$13, responsetimelast7=$14, score=$15, adminrating=$16, country=$17, city=$18, 
  state=$19, lat=$20, long=$21, postalcode='', connection=$22, whois=$23, userrating=$24, longversion=$25, shortversion=$26, 
  masterversion=$27, signup=$28, total_users=$29, active_users_halfyear=$30, active_users_monthly=$31, local_posts=$32, name=$33, 
dmorley's avatar
dmorley committed
294
  comment_counts=$35, service_facebook=$36, service_tumblr=$37, service_twitter=$38, service_wordpress=$39, weightedscore=$40, xmpp=$41, softwarename=$42, sslvalid=$43
dmorley's avatar
cleanup  
dmorley committed
295
  WHERE domain=$34";
dmorley's avatar
dmorley committed
296
  $result = pg_query_params($dbh, $sql, array($gitdate, $encoding, $secure, $hidden, $runtime, $gitrev, $ipnum, $ipv6, $months, $uptime, $live, $pingdomdate, $timenow, $responsetime, $score, $adminrating, $country, $city, $state, $lat, $long, $dver, $whois, $userrating, $xdver, $dver, $masterversion, $signup, $total_users, $active_users_halfyear, $active_users_monthly, $local_posts, $name, $domain, $comment_counts, $service_facebook, $service_tumblr, $service_twitter, $service_wordpress, $weightedscore, $xmpp, $softwarename, $outputsslerror));
dmorley's avatar
cleanup  
dmorley committed
297
  if (!$result) {
298
    die('Error in SQL query3: ' . pg_last_error());
dmorley's avatar
cleanup  
dmorley committed
299
  }
300 301
  if ($debug) {echo '<br>Score out of 100: ' . $score . '<br>';}
  if (!$debug) {echo 'Success';}
dmorley's avatar
cleanup  
dmorley committed
302 303 304 305 306
  //end foreach
  }
}     
pg_free_result($result);
pg_close($dbh);