status of all the things in the db

.gitlab-ci.yml

@@ -4,18 +4,17 @@ cache:
   - vendor/
 
 before_script:
-- apt-get update -yqq
-- apt-get install git -yqq
+- apk add -U git
 - docker-php-ext-install -j$(nproc) bcmath
 - curl -sS https://getcomposer.org/installer | php
 - php composer.phar install
 
 test:7.2:
-  image: php:7.2-cli
+  image: php:7.2-cli-alpine
   script:
   - php composer.phar check-code
 
 test:7.3-rc:
-  image: php:7.3-rc-cli
+  image: php:7.3-rc-cli-alpine
   script:
   - php composer.phar check-code

CHANGELOG.md

@@ -9,6 +9,21 @@ The format is based on [Keep a Changelog] and this project adheres to [Semantic
 ### Fixed
 ### Security
 
+## [2.5.0]
+:exclamation: DB migrations required! (see [SQL migration script][2.5.0-sql-migration])  
+### Added
+- Meta data table for status and misc storage
+- go.php can take a software= parameter and do a random pod on only one software/network
+### Changed
+- Status page is now /status from /db/status.php
+### Deprecated
+- db/pull.sh is no longer needed or used, cron can use db/update.php directly
+### Removed
+- db/last.data can be deleted from your system
+### Fixed
+- Moved status to a new table vs flat file as updates do not work right if not run every hour
+### Security
+
 ## [2.4.2] - 2018-10-18
 ### Fixed
 - Fixed db setup calls for cleaner links
@@ -116,12 +131,14 @@ The format is based on [Keep a Changelog] and this project adheres to [Semantic
 ### Fixed
 - Fix ipv6
 
+[2.5.0-sql-migration]: https://git.feneas.org/diasporg/Poduptime/blob/master/db/migrations/2.4.1-2.5.0.sql
 [2.4.1-sql-migration]: https://git.feneas.org/diasporg/Poduptime/blob/master/db/migrations/2.4.0-2.4.1.sql
 [2.4.0-sql-migration]: https://git.feneas.org/diasporg/Poduptime/blob/master/db/migrations/2.3.0-2.4.0.sql
 [2.3.0-sql-migration]: https://git.feneas.org/diasporg/Poduptime/blob/master/db/migrations/2.2.0-2.3.0.sql
 [2.2.0-sql-migration]: https://git.feneas.org/diasporg/Poduptime/blob/master/db/migrations/2.1.4-2.2.0.sql
 
 [Unreleased]: https://git.feneas.org/diasporg/Poduptime/compare/master...develop
+[2.5.0]: https://git.feneas.org/diasporg/Poduptime/compare/2.4.1...2.5.0
 [2.4.1]: https://git.feneas.org/diasporg/Poduptime/compare/2.4.0...2.4.1
 [2.4.0]: https://git.feneas.org/diasporg/Poduptime/compare/2.3.1...2.4.0
 [2.3.1]: https://git.feneas.org/diasporg/Poduptime/compare/2.3.0...2.3.1

README.md

@@ -44,15 +44,15 @@ psql -U podupuser podupdb < db/tables.sql
 1. Edit `config.php` to add your DB and file settings
 2. Touch add.log in location you configured in config.php
 3. Create your backup folder
-4. Add a pod and run `db/pull.sh init` 
+4. Add a pod and run `php db/update.php init` 
 
 # To Use:
 
-run `db/pull.sh` manually or with cron to update your data  
-run `db/pull.sh debug` to debug output  
-run `db/pull.sh sqldebug` to debug sql  
-run `db/pull.sh develop` to run without email alerts to end users  
-run `db/pull.sh Check_System_Deleted` to re-check system deleted pods as needed  
+run `php db/update.php` to update your data  
+run `php db/update.php debug` to debug output  
+run `php db/update.php sqldebug` to debug sql  
+run `php db/update.php develop` to run without email alerts to end users  
+run `php db/update.php Check_System_Deleted` to re-check system deleted pods as needed  
 These commands can be combined  
 
 # To Upgrade:
@@ -66,7 +66,8 @@ psql -U podupuser podupdb < db/migrations/xxx.sql (see db/migrations/README.md f
 # Status
 
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/e4fdde7b4ec34c56a86938d3ba6e3983)](https://www.codacy.com/app/diasporg/Poduptime?utm_source=git.feneas.org&amp;utm_medium=referral&amp;utm_content=diasporg/Poduptime&amp;utm_campaign=Badge_Grade)
-[![Build status](https://badge.buildkite.com/2a2d6f9ff932fc359416fff31ee72c3dce703569a801f83af6.svg)](https://buildkite.com/diasporg/poduptime)
+[![pipeline status](https://git.feneas.org/diasporg/Poduptime/badges/develop/pipeline.svg)](https://git.feneas.org/diasporg/Poduptime/commits/develop)
+[![coverage report](https://git.feneas.org/diasporg/Poduptime/badges/develop/coverage.svg)](https://git.feneas.org/diasporg/Poduptime/commits/develop)
 
 
 ============================

composer.json

@@ -35,7 +35,7 @@
     },
     "scripts": {
         "check-code": [
-            "vendor/bin/phpcs --standard=PSR2 -snp --encoding=utf-8 --report-width=150 lib *.php"
+            "vendor/bin/phpcs --standard=PSR2 -snp --encoding=utf-8 --report-width=150 db lib *.php"
         ]
     }
 }

config.php.example

@@ -35,13 +35,13 @@ return [
     //DNS server for dnssec testing. 1.1.1.1 tests the best
     'dnsserver'   => '',
 
-    //CA for curl to use - full file path (pull.sh will update this monthly)
+    //CA for curl to use - full file path
     'cafullpath'  => '',
 
     //Mapbox.com API key. https://www.mapbox.com/help/how-access-tokens-work/
     'mapboxkey'   => '',
 
-    //Geolite2-city database file in mmdb format - full file path (pull.sh will update this monthly)
+    //Geolite2-city database file in mmdb format - full file path
     'geoip2db'    => '',
 
     //detectlanguage.com api key

db/add.php

@@ -29,7 +29,7 @@ try {
     FROM pods
   ');
 } catch (\RedBeanPHP\RedException $e) {
-    die('Error in SQL query: ' . $e->getMessage());
+    debug('Error in SQL query: ' . sprintf($e->getMessage()));
 }
 
 $stop = false;
@@ -55,7 +55,7 @@ foreach ($pods as $pod) {
 
             R::store($p);
         } catch (\RedBeanPHP\RedException $e) {
-            die('Error in SQL query: ' . $e->getMessage());
+            debug('Error in SQL query: ' . sprintf($e->getMessage()));
         }
 
         echo <<<EOF
@@ -88,17 +88,9 @@ if (!$stop) {
         $link = max($info['links'])['href'];
     }
 
-    $chss = curl_init();
-    curl_setopt($chss, CURLOPT_URL, $link);
-    curl_setopt($chss, CURLOPT_POST, 0);
-    curl_setopt($chss, CURLOPT_HEADER, 0);
-    curl_setopt($chss, CURLOPT_CONNECTTIMEOUT, 5);
-    curl_setopt($chss, CURLOPT_RETURNTRANSFER, 1);
-    curl_setopt($chss, CURLOPT_NOBODY, 0);
-    $outputssl = curl_exec($chss);
-    curl_close($chss);
-
-    if ($outputssl && stripos($outputssl, 'openRegistrations') !== false) {
+    $output = curl($link, true);
+
+    if ($output['body'] && stripos($output['body'], 'openRegistrations') !== false) {
         echo 'Your pod has ssl and is valid<br>';
 
         $publickey = md5(uniqid($_domain, true));
@@ -114,7 +106,7 @@ if (!$stop) {
 
             R::store($p);
         } catch (\RedBeanPHP\RedException $e) {
-            die('Error in SQL query: ' . $e->getMessage());
+            debug('Error in SQL query: ' . sprintf($e->getMessage()));
         }
 
         if ($_email) {
@@ -132,7 +124,6 @@ if (!$stop) {
         }
 
         echo 'Data successfully inserted! Your pod will be checked and live on the list in a few hours!';
-
     } else {
         echo 'Could not validate your pod, check your setup!<br>Take a look at <a href="' . $link . '">your /nodeinfo</a>';
     }

db/backup.php

@@ -16,7 +16,7 @@ if (!is_cli()) {
 $keep_for    = 60 * 60 * 6; // 6 hours
 $backup_file = c('backup_dir') . '/dump_' . date('Ymd_His') . '.sql';
 
-printf("Making backup of '%s' to '%s'...", c('pgdb'), $backup_file);
+debug("Making backup of '%s' to '%s'...", c('pgdb'), $backup_file);
 system(sprintf(
     'export PGPASSWORD=%3$s &&' .
     '"%1$s" --clean --format=tar --username=%2$s %4$s >> "%5$s"',
@@ -26,7 +26,7 @@ system(sprintf(
     c('pgdb'),
     $backup_file
 ), $exit_code);
-printf(" %s\n", $exit_code === 0 ? 'Success!' : 'Failed.');
+debug(" %s\n", $exit_code === 0 ? 'Success!' : 'Failed.');
 
 $dirh = dir(c('backup_dir'));
 while ($entry = $dirh->read()) {
@@ -48,3 +48,5 @@ while ($entry = $dirh->read()) {
         unlink($file) ? 'Success!' : 'Failed.'
     );
 }
+
+updateMeta('backup');

db/migrations/2.4.1-2.5.0.sql

+CREATE TABLE meta (
+ id serial8 UNIQUE PRIMARY KEY,
+ name text,
+ value text,
+ date_created timestamp DEFAULT current_timestamp
+);
+INSERT INTO meta (name) VALUES('languages_updated');
+INSERT INTO meta (name) VALUES('cacert_updated');
+INSERT INTO meta (name) VALUES('geoip_updated');
+INSERT INTO meta (name) VALUES('masterversions_updated');
+INSERT INTO meta (name) VALUES('federation_updated');
+INSERT INTO meta (name) VALUES('statstable_updated');
+INSERT INTO meta (name) VALUES('backup');
+INSERT INTO meta (name) VALUES('pods_updated');

db/pull.sh

-#!/usr/bin/env bash
-
-SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
-cd "$SCRIPT_DIR"
-
-FLAG_FILE="/tmp/poduptime.updating"
-HOUR=$(date +%H)
-DAY=$(date +%d)
-HAPPY=" :-)"
-SAD=" :-("
-
-if [ -f "$FLAG_FILE" ]; then
-  echo "Already running update"
-  exit 1
-fi
-touch "$FLAG_FILE"
-
-# https://stackoverflow.com/a/26820300
-printf "%s" "Checking for internet..."
-if ! wget -q --spider --tries=2 --timeout=15 https://www.google.com; then
-  echo "$SAD"
-  echo "Could not update pods as no internet"
-  rm "$FLAG_FILE"
-  exit 1
-fi
-echo "$HAPPY"
-
-if [ "$HOUR" = 23 ] || [ "$1" = 'init' ]; then
-  printf "%s" "Updating master versions..."
-  if php update-masterversions.php; then
-    echo "$HAPPY"
-  else
-    echo "$SAD"
-  fi
-  printf "%s" "Updating Monthy Stats Table..."
-  if php update-monthly-stats.php; then
-    echo "$HAPPY"
-  else
-    echo "$SAD"
-  fi
-  printf "%s" "Crawling and updating from the federation..."
-  if php update-remote-data.php; then
-    echo "$HAPPY"
-  else
-    echo "$SAD"
-  fi
-  if [ "$DAY" = 23 ]  || [ "$1" = 'init' ]; then
-    printf "%s" "Updating CA..."
-    CACERT_FILE="$(php -r "echo (require __DIR__ . '/../config.php')['cafullpath'];")"
-    if curl -Lss https://curl.haxx.se/ca/cacert.pem -o "$CACERT_FILE"; then
-      echo "$HAPPY"
-    else
-      echo "$SAD"
-    fi
-    printf "%s" "Updating GeoIP2 DB..."
-    GEODB_FILE="$(php -r "echo (require __DIR__ . '/../config.php')['geoip2db'];")"
-    if funzip <(curl -Lss http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz) > "$GEODB_FILE"; then
-      echo "$HAPPY"
-    else
-      echo "$SAD"
-    fi
-  fi
-fi
-
-echo "Updating in new pod data...";
-php update.php $@
-touch last.data
-echo "Finished update!"
-
-echo "Backing up..."
-php backup.php
-echo
-
-rm "$FLAG_FILE"

db/release-update-lock.php

+<?php
+
+declare(strict_types=1);
+
+require_once __DIR__ . '/../boot.php';
+
+updateMeta('pods_updating', false);

db/status.php

-<?php
-
-/**
- * Check status of previously pulled data.
- */
-
-declare(strict_types=1);
-
-$dur = time() - filemtime('last.data');
-echo $dur;
-if ($dur > 4500) {
-    http_response_code(500);
-} else {
-    http_response_code(200);
-}

db/tables.sql

@@ -96,12 +96,17 @@ CREATE TABLE masterversions (
  date_checked timestamp DEFAULT current_timestamp
 );
 
-CREATE TABLE monthlystats (
+CREATE TABLE meta (
  id serial8 UNIQUE PRIMARY KEY,
- total_users int,
- total_posts int,
- total_comments int,
- total_pods int,
- total_uptime int,
- date_checked timestamp DEFAULT current_timestamp
+ name text,
+ value text,
+ date_created timestamp DEFAULT current_timestamp
 );
+INSERT INTO meta (name) VALUES('languages_updated');
+INSERT INTO meta (name) VALUES('cacert_updated');
+INSERT INTO meta (name) VALUES('geoip_updated');
+INSERT INTO meta (name) VALUES('masterversions_updated');
+INSERT INTO meta (name) VALUES('federation_updated');
+INSERT INTO meta (name) VALUES('statstable_updated');
+INSERT INTO meta (name) VALUES('backup');
+INSERT INTO meta (name) VALUES('pods_updated');

db/update-masterversions.php

@@ -22,6 +22,7 @@ $softwares = [
     'pleroma'      => ['repo' => 'pleroma%2fpleroma', 'gitsite' => 'git.pleroma.social', 'gittype' => 'gitlab', 'devbranch' => 'develop'],
     'socialhome'   => ['repo' => 'jaywink/socialhome', 'gitsite' => 'api.github.com', 'gittype' => 'github', 'devbranch' => ''],
     'social-relay' => ['repo' => 'jaywink/social-relay', 'gitsite' => 'api.github.com', 'gittype' => 'github', 'devbranch' => ''],
+    'writefreely'  => ['repo' => 'writeas/writefreely', 'gitsite' => 'api.github.com', 'gittype' => 'github', 'devbranch' => 'develop'],
     'ganggo'       => ['repo' => 'ganggo%2fganggo', 'gitsite' => 'git.feneas.org', 'gittype' => 'gitlab', 'devbranch' => ''],
     'funkwhale'    => ['repo' => 'funkwhale%2ffunkwhale', 'gitsite' => 'code.eliotberriot.com', 'gittype' => 'gitlab', 'devbranch' => 'develop'],
 ];
@@ -52,7 +53,7 @@ foreach ($softwares as $software => $details) {
                 }
                 R::store($m);
             } catch (\RedBeanPHP\RedException $e) {
-                die('Error in SQL query: ' . $e->getMessage());
+                debug('Error in SQL query: ' . sprintf($e->getMessage()));
             }
         }
     } elseif ($details['gittype'] === 'gitlab') {
@@ -76,10 +77,12 @@ foreach ($softwares as $software => $details) {
                 }
                 R::store($m);
             } catch (\RedBeanPHP\RedException $e) {
-                die('Error in SQL query: ' . $e->getMessage());
+                debug('Error in SQL query: ' . sprintf($e->getMessage()));
             }
         }
     }
 
     printf('%s:%s:%s ', $software, $masterversion, $devlastcommit ?: 'n/a');
 }
+
+updateMeta('masterversions_updated');

db/update-monthly-stats.php

@@ -19,17 +19,17 @@ try {
     $monthly_totals = R::getAll("
         SELECT
             to_char(date_checked, 'yyyy-mm') AS yymm,
-            sum(total_users) / count(DISTINCT to_char(date_checked, 'HH24 dd')) as users,
-            sum(local_posts) / count(DISTINCT to_char(date_checked, 'HH24 dd')) as posts,
-            sum(comment_counts) / count(DISTINCT to_char(date_checked, 'HH24 dd')) as comments,
-            count(domain) / count(DISTINCT to_char(date_checked, 'HH24 dd')) as pods,
+            sum(total_users) / COUNT(DISTINCT CASE WHEN domain = 'diasp.org' THEN id ELSE NULL END) as users,
+            sum(local_posts) / COUNT(DISTINCT CASE WHEN domain = 'diasp.org' THEN id ELSE NULL END) as posts,
+            sum(comment_counts) / COUNT(DISTINCT CASE WHEN domain = 'diasp.org' THEN id ELSE NULL END) as comments,
+            count(domain) / COUNT(DISTINCT CASE WHEN domain = 'diasp.org' THEN id ELSE NULL END) as pods,
             count(nullif(online, false)) as uptime, 
             count(nullif(online, true)) as downtime
         FROM checks
         GROUP BY yymm
     ");
 } catch (\RedBeanPHP\RedException $e) {
-    die('Error in SQL query: ' . $e->getMessage());
+    debug('Error in SQL query: ' . sprintf($e->getMessage()));
 }
 
 foreach ($monthly_totals as $monthly) {
@@ -51,6 +51,8 @@ foreach ($monthly_totals as $monthly) {
 
         R::store($p);
     } catch (\RedBeanPHP\RedException $e) {
-        die('Error in SQL query: ' . $e->getMessage());
+        debug('Error in SQL query: ' . sprintf($e->getMessage()));
     }
 }
+
+updateMeta('statstable_updated');

db/update-remote-data.php

@@ -22,7 +22,7 @@ try {
     ';
     $pods = R::getAll($sql);
 } catch (\RedBeanPHP\RedException $e) {
-    die('Error in SQL query: ' . $e->getMessage());
+    debug('Error in SQL query: ' . sprintf($e->getMessage()));
 }
 
 //get all existing pod domains
@@ -37,19 +37,22 @@ if ($pods = json_decode(file_get_contents('https://the-federation.info/graphql?q
 
         //limiting to diaspora compatible for now
         if (in_array('diaspora', $protocols, true)) {
-            $foundpods[] = $poddata['host'];
+            $foundpods[] = strtolower(preg_replace('(^https?://)', '', $poddata['host']));
         }
     }
 }
 
 if ($pods = json_decode(file_get_contents('https://diasp.org/pods.json'), true)) {
     foreach ($pods ?? [] as $poddata) {
-        $foundpods[] = $poddata['host'];
+        $foundpods[] = strtolower($poddata['host']);
     }
 }
 
 $results = array_diff($foundpods, $existingpods);
 foreach ($results as $result) {
-    echo $result . ' ';
-    echo exec("php-cgi add.php domain={$result}") . "\r\n";
+    echo ' domain ' . $result . ' trying... ';
+    $add = __DIR__ . '/add.php';
+    echo exec("php-cgi {$add} domain={$result}") . "\r\n";
 }
+
+updateMeta('federation_updated');

db/update.php

@@ -22,6 +22,16 @@ use Longman\IPTools\Ip;
 use Poduptime\PodStatus;
 use RedBeanPHP\R;
 
+if (getMeta('pods_updating')) {
+    die('already running');
+};
+
+if (!is_connected()) {
+    die('no internet');
+}
+
+$time_start = microtime(true);
+
 $debug    = isset($_GET['debug']) || (isset($argv) && in_array('debug', $argv, true));
 $sqldebug = isset($_GET['sqldebug']) || (isset($argv) && in_array('sqldebug', $argv, true));
 $develop  = isset($_GET['develop']) || (isset($argv) && in_array('develop', $argv, true));
@@ -52,14 +62,16 @@ try {
     } elseif (is_cli() && (isset($argv) && in_array('Check_System_Deleted', $argv, true))) {
         $sql  .= ' WHERE status = ? ORDER BY id';
         $pods = R::getAll($sql, [PodStatus::SYSTEM_DELETED]);
+        updateMeta('pods_updating', true);
     } elseif (is_cli()) {
         $sql  .= ' WHERE status < ? ORDER BY id';
         $pods = R::getAll($sql, [PodStatus::PAUSED]);
+        updateMeta('pods_updating', true);
     }
 } catch (\RedBeanPHP\RedException $e) {
-    die('Error in SQL query: ' . $e->getMessage());
+    debug('Error in SQL query: ' . sprintf($e->getMessage()));
 } catch (\MaxMind\Db\Reader\InvalidDatabaseException $e) {
-    die('Invalid GeoIP database: ' . $e->getMessage());
+    debug('Invalid GeoIP database: ' . sprintf($e->getMessage()));
 }
 
 foreach ($pods as $pod) {
@@ -83,7 +95,7 @@ foreach ($pods as $pod) {
             WHERE domain = ?
         ', [$domain]);
     } catch (\RedBeanPHP\RedException $e) {
-        die('Error in SQL query: ' . $e->getMessage());
+        debug('Error in SQL query: ' . sprintf($e->getMessage()));
     }
 
     debug('Domain', $domain);
@@ -162,21 +174,21 @@ foreach ($pods as $pod) {
                 echo $c;
             }
         } catch (\RedBeanPHP\RedException $e) {
-            die('Error in SQL query: ' . $e->getMessage());
+            debug('Error in SQL query: ' . sprintf($e->getMessage()));
         }
 
         debug('Version code', $shortversion);
 
         try {
             $masterdata = R::getRow('
-                SELECT version, devlastcommit, releasedate
+                SELECT version, devlastcommit, releasedate, date_checked
                 FROM masterversions
                 WHERE software = ?
                 ORDER BY id
                 DESC LIMIT 1
             ', [$softwarename]);
         } catch (\RedBeanPHP\RedException $e) {
-            die('Error in SQL query: ' . $e->getMessage());
+            debug('Error in SQL query: ' . sprintf($e->getMessage()));
         }
 
         $masterversion = ($masterdata['version'] ?? '0.0.0.0');
@@ -201,7 +213,7 @@ foreach ($pods as $pod) {
                 LIMIT 1
             ', [$domain]);
         } catch (\RedBeanPHP\RedException $e) {
-            die('Error in SQL query: ' . $e->getMessage());
+            debug('Error in SQL query: ' . sprintf($e->getMessage()));
         }
 
         $lastdatechecked = ($lastpodupdates['date_checked'] ?? date('Y-m-d H:i:s.u'));
@@ -215,7 +227,6 @@ foreach ($pods as $pod) {
             //tmp//if pod is on the development branch - see when you last updated your pod and when the last commit was made to dev branch - if the repo is active and your not updating every 120 days why are you on dev branch?
 
             if ($updategap + $devlastdays > 200) {
-
                 debug('Outdated More than 200 days', 'Yes');
 
                 $podminhelp = 'Your code base seems too out of date to be used. Last time you updated was ' . $updategap;
@@ -229,15 +240,12 @@ foreach ($pods as $pod) {
             $score     -= 2;
             $updategap = date_diff(new DateTime($lastdatechecked), new DateTime($releasedate))->format('%a');
             $podminhelp = 'Your code base seems too out of date to be used. Current version is ' . $masterversion . ' and you are running ' . $shortversion;
-
         } elseif ($updategap - date_diff(new DateTime($releasedate), new DateTime())->format('%a') > 200) {
-
             debug('Outdated more than 200 days since x ', 'Yes');
 
             $score     -= 2;
             $updategap = date_diff(new DateTime($lastdatechecked), new DateTime($releasedate))->format('%a');
             $podminhelp = 'Your code base seems too out of date to be used. Last time you updated was ' . $updategap;
-
         } else {
             $updategap = date_diff(new DateTime($lastdatechecked), new DateTime($releasedate))->format('%a');
         }
@@ -247,16 +255,20 @@ foreach ($pods as $pod) {
         $status = PodStatus::UP;
     }
 
+    $langhours  = Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('languages_updated', 'date_created'))->diffInHours();
+
     // Default to the already saved language.
     $detectedlanguage = $language;
+    $languagesupdated = false;
 
     $language_snippet = getWebsiteLanguageSnippetFromUrl("https://{$domain}/");
     if (!$language_snippet) {
         $detectedlanguage = null;
         --$score;
         $podminhelp = 'Unable to render the html on your main page https://' . $domain;
-    } elseif ($debug || Carbon::now()->hour === 12) {
+    } elseif ($debug || $langhours > 24) {
         $detectedlanguage = detectWebsiteLanguageFromSnippet($language_snippet);
+        $languagesupdated = true;
     }
 
     debug('Detected Language', $detectedlanguage);
@@ -276,7 +288,7 @@ foreach ($pods as $pod) {
                 echo $c;
             }
         } catch (\RedBeanPHP\RedException $e) {
-            die('Error in SQL query: ' . $e->getMessage());
+            debug('Error in SQL query: ' . sprintf($e->getMessage()));
         }
 
         --$score;
@@ -318,7 +330,7 @@ foreach ($pods as $pod) {
 
         debug('Location', json_encode($geo->raw), true);
     }
-    echo $newline;
+
     $statslastdate = date('Y-m-d H:i:s');
 
     $diff   = (new DateTime())->diff(new DateTime($dateadded));
@@ -337,7 +349,7 @@ foreach ($pods as $pod) {
         $avglatency = $checks['latency'] ?? 0;
         $uptime     = $checks['online'] ?? 0;
     } catch (\RedBeanPHP\RedException $e) {
-        die('Error in SQL query: ' . $e->getMessage());
+        debug('Error in SQL query: ' . sprintf($e->getMessage()));
     }
 
     debug('Uptime', $uptime);
@@ -420,10 +432,47 @@ foreach ($pods as $pod) {
             echo $newline;
         }
     } catch (\RedBeanPHP\RedException $e) {
-        die('Error in SQL query: ' . $e->getMessage());
+        debug('Error in SQL query: ' . sprintf($e->getMessage()));
+    }
+
+    debug('Success', $domain);
+}
+
+updateMeta('pods_updated');
+$time_end = microtime(true);
+$execution_time = ($time_end - $time_start)/60;
+updateMeta('pods_update_runtime', round($execution_time));
+if ($languagesupdated) {
+    updateMeta('languages_updated');
+}
+
+//check daily updates
+if (!$_domain) {
+    if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('masterversions_updated', 'date_created'))->diffInHours() > 12) {
+        require __DIR__ . '/update-masterversions.php';
+    }
+
+    if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('statstable_updated', 'date_created'))->diffInHours() > 23) {
+        require __DIR__ . '/update-monthly-stats.php';
+    }
+
+    if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('cacert_updated', 'date_created'))->diffInHours() > 40) {
+        $cacert = c('cafullpath');
+        file_put_contents($cacert, file_get_contents("https://curl.haxx.se/ca/cacert.pem"));
+        updateMeta('cacert_updated');
+    }
+
+    if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('geoip_updated', 'date_created'))->diffInHours() > 45) {
+        $geo2ip = c('geoip2db');
+        file_put_contents($geo2ip, file_get_contents("compress.zlib://http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz"));
+        updateMeta('geoip_updated');
+    }
+
+    if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('federation_updated', 'date_created'))->diffInHours() > 9) {
+        require __DIR__ . '/update-remote-data.php';
     }
 
-    echo 'Success ' . $domain;
+    require __DIR__ . '/backup.php';
 
-    echo $newline;
+    updateMeta('pods_updating', false);
 }

go.php

@@ -11,7 +11,8 @@ use Jaybizzle\CrawlerDetect\CrawlerDetect;
 use RedBeanPHP\R;
 
 // Other parameters.
-$_domain = $_GET['domain'] ?? '';
+$_domain   = $_GET['domain'] ?? '';
+$_software = $_GET['software'] ?? '';
 
 require_once __DIR__ . '/boot.php';
 
@@ -20,6 +21,19 @@ try {
         $click  = 'manualclick';
         $domain = R::getCell('SELECT domain FROM pods WHERE domain LIKE ?', [$_domain]);
         $domain || die('unknown domain');
+    } elseif ($_software) {
+        $click  = 'autoclick';
+        $domain = R::getCell('
+            SELECT domain
+            FROM pods
+            WHERE signup
+                AND uptime_alltime > 95
+                AND monthsmonitored > 1
+                AND softwarename = ?
+            ORDER BY random()
+            LIMIT 1
+        ', [$_software]);
+        $domain || die('no domains exist');
     } else {
         $click  = 'autoclick';
         $domain = R::getCell('

index.php

@@ -9,7 +9,6 @@ use Carbon\Carbon;
 
 require_once __DIR__ . '/boot.php';
 
-$lastfile   = 'db/last.data';
 $input      = isset($_GET['input']) ? substr($_GET['input'], 1) : null;
 $mapview    = isset($_GET['mapview']) || $input === 'map';
 $statsview  = isset($_GET['statsview']) || $input === 'stats';
@@ -18,8 +17,10 @@ $podminedit = isset($_GET['podminedit']) || $input === 'podminedit';
 $edit       = isset($_GET['edit']) || $input === 'edit';
 $add        = isset($_GET['add']) || $input === 'add';
 $gettoken   = isset($_GET['gettoken']) || $input === 'gettoken';
+$status     = isset($_GET['status']) || $input === 'status';
 $simpleview = !($mapview || $podmin || $podminedit || $statsview);
 $fullview   = false;
+
 ?>
 <!doctype html>
 <html lang="en">
@@ -120,6 +121,8 @@ $navs = [
             include_once __DIR__ . '/statsview.php';
         } elseif ($podmin) {
             include_once __DIR__ . '/podmin.php';
+        } elseif ($status) {
+            include_once __DIR__ . '/status.php';
         } elseif ($podminedit) {
             include_once __DIR__ . '/podminedit.php';
         } elseif ($edit) {
@@ -136,7 +139,7 @@ $navs = [
     </div>
 </main>
 <footer class="ml-3 text-muted">
-    <small>Data refreshed <?php echo Carbon::createFromTimestamp(filemtime($lastfile))->diffForHumans(); ?></small>
+    <small><a href="/status">Data refreshed <?php echo Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('pods_updated', 'date_created'))->diffForHumans(); ?></a></small>
 </footer>
 <input type="hidden" name="input" value="<?php echo $input ?>"/>
 <script src="node_modules/jquery/dist/jquery.min.js"></script>

lib/functions.php

@@ -7,6 +7,7 @@
 declare(strict_types=1);
 
 use DetectLanguage\DetectLanguage;
+use RedBeanPHP\R;
 
 /**
  * Helper to get config values.
@@ -210,3 +211,59 @@ function debugToConsole($data): void
 
     echo "<script>console.log('Debug Objects: {$output}');</script>";
 }
+
+/**
+ * Update meta table
+ *