Commit 83355645 authored by dmorley's avatar dmorley

remove pull.sh and have update.php do it all

parent 556ef128
Pipeline #746 failed with stage
in 3 minutes and 46 seconds
......@@ -13,10 +13,11 @@ The format is based on [Keep a Changelog] and this project adheres to [Semantic
:exclamation: DB migrations required! (see [SQL migration script][2.5.0-sql-migration])
### Added
- Meta data table for status and misc storage
- go.php can take a software= paramater and do a random pod on only one software/network
- go.php can take a software= parameter and do a random pod on only one software/network
### Changed
- Status page is now /status from /db/status.php
### Deprecated
- db/pull.sh is no longer needed or used, cron can use db/update.php directly
### Removed
- db/last.data can be deleted from your system
### Fixed
......
......@@ -35,13 +35,13 @@ return [
//DNS server for dnssec testing. 1.1.1.1 tests the best
'dnsserver' => '',
//CA for curl to use - full file path (pull.sh will update this monthly)
//CA for curl to use - full file path
'cafullpath' => '',
//Mapbox.com API key. https://www.mapbox.com/help/how-access-tokens-work/
'mapboxkey' => '',
//Geolite2-city database file in mmdb format - full file path (pull.sh will update this monthly)
//Geolite2-city database file in mmdb format - full file path
'geoip2db' => '',
//detectlanguage.com api key
......
......@@ -4,4 +4,11 @@ CREATE TABLE meta (
value text,
date_created timestamp DEFAULT current_timestamp
);
INSERT INTO meta (name) VALUES('languages_updated');
\ No newline at end of file
INSERT INTO meta (name) VALUES('languages_updated');
INSERT INTO meta (name) VALUES('cacert_updated');
INSERT INTO meta (name) VALUES('geoip_updated');
INSERT INTO meta (name) VALUES('masterversions_updated');
INSERT INTO meta (name) VALUES('federation_updated');
INSERT INTO meta (name) VALUES('statstable_updated');
INSERT INTO meta (name) VALUES('backup');
INSERT INTO meta (name) VALUES('pods_updated');
#!/usr/bin/env bash
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
cd "$SCRIPT_DIR"
FLAG_FILE="/tmp/poduptime.updating"
HOUR=$(date +%H)
DAY=$(date +%d)
HAPPY=" :-)"
SAD=" :-("
if [ -f "$FLAG_FILE" ]; then
echo "Already running update"
exit 1
fi
touch "$FLAG_FILE"
# https://stackoverflow.com/a/26820300
printf "%s" "Checking for internet..."
if ! wget -q --spider --tries=2 --timeout=15 https://www.google.com; then
echo "$SAD"
echo "Could not update pods as no internet"
rm "$FLAG_FILE"
exit 1
fi
echo "$HAPPY"
if [ "$HOUR" = 23 ] || [ "$1" = 'init' ]; then
printf "%s" "Updating master versions..."
if php update-masterversions.php; then
echo "$HAPPY"
else
echo "$SAD"
fi
printf "%s" "Updating Monthy Stats Table..."
if php update-monthly-stats.php; then
echo "$HAPPY"
else
echo "$SAD"
fi
printf "%s" "Crawling and updating from the federation..."
if php update-remote-data.php; then
echo "$HAPPY"
else
echo "$SAD"
fi
if [ "$DAY" = 23 ] || [ "$1" = 'init' ]; then
printf "%s" "Updating CA..."
CACERT_FILE="$(php -r "echo (require __DIR__ . '/../config.php')['cafullpath'];")"
if curl -Lss https://curl.haxx.se/ca/cacert.pem -o "$CACERT_FILE"; then
echo "$HAPPY"
else
echo "$SAD"
fi
printf "%s" "Updating GeoIP2 DB..."
GEODB_FILE="$(php -r "echo (require __DIR__ . '/../config.php')['geoip2db'];")"
if funzip <(curl -Lss http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz) > "$GEODB_FILE"; then
echo "$HAPPY"
else
echo "$SAD"
fi
fi
fi
echo "Updating in new pod data...";
php update.php $@
echo "Finished update!"
echo "Backing up..."
php backup.php
echo
rm "$FLAG_FILE"
......@@ -55,4 +55,4 @@ foreach ($monthly_totals as $monthly) {
}
}
updateMeta('monthlystats');
updateMeta('statstable_updated');
......@@ -54,4 +54,4 @@ foreach ($results as $result) {
echo exec("php-cgi add.php domain={$result}") . "\r\n";
}
updateMeta('federation_crawldate');
updateMeta('federation_updated');
......@@ -21,6 +21,18 @@ use GeoIp2\Database\Reader;
use Longman\IPTools\Ip;
use Poduptime\PodStatus;
use RedBeanPHP\R;
//
//updateMeta('pods_updating', false);
if (getMeta('pods_updating')) {
die('already running');
};
if (!is_connected()) {
die('no internet');
}
updateMeta('pods_updating', true);
$time_start = microtime(true);
......@@ -55,7 +67,7 @@ try {
$sql .= ' WHERE status = ? ORDER BY id';
$pods = R::getAll($sql, [PodStatus::SYSTEM_DELETED]);
} elseif (is_cli()) {
$sql .= ' WHERE status < ? ORDER BY id';
$sql .= ' WHERE status < ? ORDER BY id LIMIT 50';
$pods = R::getAll($sql, [PodStatus::PAUSED]);
}
} catch (\RedBeanPHP\RedException $e) {
......@@ -441,3 +453,37 @@ updateMeta('pods_update_runtime', round($execution_time));
if ($languagesupdated) {
updateMeta('languages_updated');
}
//check daily updates
if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('masterversions_updated', 'date_created'))->diffInHours() > 12) {
require_once('update-masterversions.php');
}
if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('statstable_updated', 'date_created'))->diffInHours() > 23) {
require_once('update-monthly-stats.php');
}
if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('cacert_updated', 'date_created'))->diffInHours() > 40) {
$cacert = c('cafullpath');
$getcacert = new NPM\Xec\Command("curl -Lss https://curl.haxx.se/ca/cacert.pem -o {$cacert}");
$getcacert->throwExceptionOnError(false);
$getcacert->execute();
updateMeta('cacert_updated');
}
if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('geoip_updated', 'date_created'))->diffInHours() > 45) {
$geo2ip = c('geoip2db');
$getgeo2ip = new NPM\Xec\Command("funzip <(curl -Lss http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz) > {$geo2ip}");
$getgeo2ip->throwExceptionOnError(false);
$getgeo2ip->execute();
updateMeta('cacert_updated');
updateMeta('geoip_updated');
}
if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('federation_updated', 'date_created'))->diffInHours() > 9) {
require_once('update-remote-data.php');
}
require_once('backup.php');
updateMeta('pods_updating', false);
......@@ -252,3 +252,17 @@ function getMeta($name, $select = 'value')
$result = $metaresult[$select];
return $result;
}
/**
* Check for internet connection
*
* @return bool
*/
function is_connected() {
$connected = @fsockopen("www.google.com", 80);
if ($connected){
fclose($connected);
return true;
}
return false;
}
......@@ -21,6 +21,15 @@ $lang_updated_stats = Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('languag
echo "Last Language check was " . $lang_updated_stats . " hours<br>";
$back_updated_stats = Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('backup', 'date_created'))->diffInMinutes();
echo "Last SQL Backup was " . $back_updated_stats . " minutes<br>";
echo "Last Local SQL Backup was " . $back_updated_stats . " minutes<br>";
$masterversions_updated_stats = Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('masterversions_updated', 'date_created'))->diffInHours();
echo "Last Masterversion update from repos was " . $masterversions_updated_stats . " hours<br>";
$federation_updated_stats = Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('federation_updated', 'date_created'))->diffInHours();
echo "Last Federation crawl was " . $federation_updated_stats . " hours<br>";
$statstable_updated_stats = Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('statstable_updated', 'date_created'))->diffInHours();
echo "Last Monthly stats table update was " . $statstable_updated_stats . " hours<br>";
echo "</small>";
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment