Unverified Commit 3cd60e3e authored by Armando Lüscher's avatar Armando Lüscher Committed by GitHub

Merge pull request #179 from diasporg/geoip2

move to supported db for IP geo data
parents 9b31febe 41bc19bd
......@@ -12,4 +12,7 @@
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
\ No newline at end of file
along with this program. If not, see <http://www.gnu.org/licenses/>.
This product includes GeoLite2 data created by MaxMind, available from
<a href="http://www.maxmind.com">http://www.maxmind.com</a>.
......@@ -15,6 +15,7 @@
* Use the git API for release versions, check development releases on pods https://github.com/diasporg/Poduptime/issues/143
* Forbid access to files that should be cli only https://github.com/diasporg/Poduptime/issues/152
* Move from bower to yarn for packages
* NOTE config.php.examples change to full paths for 2 items!
## End Users
* Show version and update in full view cleaner https://github.com/diasporg/Poduptime/issues/143
......
......@@ -34,6 +34,7 @@ psql -U podupuser podupdb < db/tables.sql
Edit `config.php` to add your DB and file settings.
touch add.log in location you configured in config.php
start your first run with `db/pull.sh init`
run `db/pull.sh` manually or with cron to update your data
run `db/pull.sh debug` to debug output
......
......@@ -12,7 +12,8 @@
"noplanman/xec": "0.1.0",
"jaybizzle/crawler-detect": "1.*",
"twbs/bootstrap": "^4.1.0",
"patrickschur/language-detection": "^3.3"
"patrickschur/language-detection": "^3.3",
"geoip2/geoip2": "^2.9"
},
"require-dev": {
"squizlabs/php_codesniffer": "^3.3"
......
{
"_readme": [
"This file locks the dependencies of your project to a known state",
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
"This file is @generated automatically"
],
"content-hash": "d447ea75efb92876c1bee13e93c2c5aa",
"content-hash": "1cd7c802d1459c30d044708a4ecc98b0",
"packages": [
{
"name": "commerceguys/enum",
......@@ -44,6 +44,62 @@
"description": "A PHP 5.4+ enumeration library.",
"time": "2015-02-27T21:36:56+00:00"
},
{
"name": "composer/ca-bundle",
"version": "1.1.1",
"source": {
"type": "git",
"url": "https://github.com/composer/ca-bundle.git",
"reference": "d2c0a83b7533d6912e8d516756ebd34f893e9169"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/composer/ca-bundle/zipball/d2c0a83b7533d6912e8d516756ebd34f893e9169",
"reference": "d2c0a83b7533d6912e8d516756ebd34f893e9169",
"shasum": ""
},
"require": {
"ext-openssl": "*",
"ext-pcre": "*",
"php": "^5.3.2 || ^7.0"
},
"require-dev": {
"phpunit/phpunit": "^4.8.35 || ^5.7 || ^6.5",
"psr/log": "^1.0",
"symfony/process": "^2.5 || ^3.0 || ^4.0"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "1.x-dev"
}
},
"autoload": {
"psr-4": {
"Composer\\CaBundle\\": "src"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Jordi Boggiano",
"email": "j.boggiano@seld.be",
"homepage": "http://seld.be"
}
],
"description": "Lets you find a path to the system CA bundle, and includes a fallback to the Mozilla CA bundle.",
"keywords": [
"cabundle",
"cacert",
"certificate",
"ssl",
"tls"
],
"time": "2018-03-29T19:57:20+00:00"
},
{
"name": "gabordemooij/redbean",
"version": "v5.1",
......@@ -85,6 +141,58 @@
],
"time": "2018-04-01T11:51:37+00:00"
},
{
"name": "geoip2/geoip2",
"version": "v2.9.0",
"source": {
"type": "git",
"url": "https://github.com/maxmind/GeoIP2-php.git",
"reference": "a807fbf65212eef5d8d2db1a1b31082b53633d77"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/maxmind/GeoIP2-php/zipball/a807fbf65212eef5d8d2db1a1b31082b53633d77",
"reference": "a807fbf65212eef5d8d2db1a1b31082b53633d77",
"shasum": ""
},
"require": {
"maxmind-db/reader": "~1.0",
"maxmind/web-service-common": "~0.5",
"php": ">=5.4"
},
"require-dev": {
"friendsofphp/php-cs-fixer": "2.*",
"phpunit/phpunit": "4.*",
"squizlabs/php_codesniffer": "3.*"
},
"type": "library",
"autoload": {
"psr-4": {
"GeoIp2\\": "src"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"Apache-2.0"
],
"authors": [
{
"name": "Gregory J. Oschwald",
"email": "goschwald@maxmind.com",
"homepage": "http://www.maxmind.com/"
}
],
"description": "MaxMind GeoIP2 PHP API",
"homepage": "https://github.com/maxmind/GeoIP2-php",
"keywords": [
"IP",
"geoip",
"geoip2",
"geolocation",
"maxmind"
],
"time": "2018-04-10T15:32:59+00:00"
},
{
"name": "jaybizzle/crawler-detect",
"version": "v1.2.64",
......@@ -134,6 +242,108 @@
],
"time": "2018-06-22T20:32:56+00:00"
},
{
"name": "maxmind-db/reader",
"version": "v1.3.0",
"source": {
"type": "git",
"url": "https://github.com/maxmind/MaxMind-DB-Reader-php.git",
"reference": "e042b4f8a2dff41e19019faf16427178b07fbd58"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/maxmind/MaxMind-DB-Reader-php/zipball/e042b4f8a2dff41e19019faf16427178b07fbd58",
"reference": "e042b4f8a2dff41e19019faf16427178b07fbd58",
"shasum": ""
},
"require": {
"php": ">=5.4"
},
"require-dev": {
"friendsofphp/php-cs-fixer": "2.*",
"phpunit/phpunit": "4.* || 5.*",
"satooshi/php-coveralls": "1.0.*",
"squizlabs/php_codesniffer": "3.*"
},
"suggest": {
"ext-bcmath": "bcmath or gmp is required for decoding larger integers with the pure PHP decoder",
"ext-gmp": "bcmath or gmp is required for decoding larger integers with the pure PHP decoder",
"ext-maxminddb": "A C-based database decoder that provides significantly faster lookups"
},
"type": "library",
"autoload": {
"psr-4": {
"MaxMind\\Db\\": "src/MaxMind/Db"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"Apache-2.0"
],
"authors": [
{
"name": "Gregory J. Oschwald",
"email": "goschwald@maxmind.com",
"homepage": "http://www.maxmind.com/"
}
],
"description": "MaxMind DB Reader API",
"homepage": "https://github.com/maxmind/MaxMind-DB-Reader-php",
"keywords": [
"database",
"geoip",
"geoip2",
"geolocation",
"maxmind"
],
"time": "2018-02-21T21:23:33+00:00"
},
{
"name": "maxmind/web-service-common",
"version": "v0.5.0",
"source": {
"type": "git",
"url": "https://github.com/maxmind/web-service-common-php.git",
"reference": "61a9836fa3bb1743ab89752bae5005d71e78c73b"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/maxmind/web-service-common-php/zipball/61a9836fa3bb1743ab89752bae5005d71e78c73b",
"reference": "61a9836fa3bb1743ab89752bae5005d71e78c73b",
"shasum": ""
},
"require": {
"composer/ca-bundle": "^1.0.3",
"ext-curl": "*",
"ext-json": "*",
"php": ">=5.4"
},
"require-dev": {
"friendsofphp/php-cs-fixer": "2.*",
"phpunit/phpunit": "4.*",
"squizlabs/php_codesniffer": "3.*"
},
"type": "library",
"autoload": {
"psr-4": {
"MaxMind\\Exception\\": "src/Exception",
"MaxMind\\WebService\\": "src/WebService"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"Apache-2.0"
],
"authors": [
{
"name": "Gregory Oschwald",
"email": "goschwald@maxmind.com"
}
],
"description": "Internal MaxMind Web Service API",
"homepage": "https://github.com/maxmind/web-service-common-php",
"time": "2018-02-12T22:31:54+00:00"
},
{
"name": "nesbot/carbon",
"version": "1.31.1",
......
......@@ -5,13 +5,13 @@
declare(strict_types=1);
//backup directory
$backup_dir = '/backup';
//backup directory - full dir path
$backup_dir = __DIR__ . '/backup';
//log directory
$log_dir = '/log';
//log directory - full dir path
$log_dir = __DIR__ . '/log';
//location of pg dump
//location of pg dump - full dir path
$pg_dump_dir = '/usr/bin';
//db host
......@@ -32,8 +32,11 @@ $adminemail = '';
//DNS server for dnssec testing. 1.1.1.1 tests the best
$dnsserver = '';
//CA for curl to use - suggest wget http://curl.haxx.se/ca/cacert.pem as they pull from mozilla if you use a system CA then certs like wosign and startssl will be valid but users will be unable to connect to them
//CA for curl to use - full file path (pull.sh will update this monthly)
$cafullpath = '';
//Mapbox.com API key. https://www.mapbox.com/help/how-access-tokens-work/
$mapboxkey = '';
//Geolite2-city database file in mmdb format - full file path (pull.sh will update this monthly)
$geoip2db = '';
\ No newline at end of file
......@@ -15,7 +15,7 @@ require_once __DIR__ . '/../config.php';
define('PODUPTIME', microtime(true));
$log = new Logging();
$log->lfile(__DIR__ . '/../' . $log_dir . '/add.log');
$log->lfile( $log_dir . '/add.log');
if (!($_domain = $_GET['domain'] ?? null)) {
$log->lwrite('no domain given');
die('no pod domain given');
......
......@@ -15,15 +15,15 @@ require_once __DIR__ . '/../config.php';
$keep = (60 * 60 * 6) * 1;
$dump_date = date('Ymd_Hs');
$file_name = __DIR__ . '/..' . $backup_dir . '/dump_' . $dump_date . '.sql';
$file_name = $backup_dir . '/dump_' . $dump_date . '.sql';
system("export PGPASSWORD=$pgpass && $pg_dump_dir/pg_dump --username=$pguser $pgdb >> $file_name");
echo "pg backup of $pgdb made";
$dirh = dir(__DIR__ . '/..' . $backup_dir);
$dirh = dir($backup_dir);
while ($entry = $dirh->read()) {
$old_file_time = (date('U') - $keep);
$file_created = filectime(__DIR__ . '/..' . "$backup_dir/$entry");
$file_created = filectime( "$backup_dir/$entry");
if ($file_created < $old_file_time && !is_dir($entry)) {
if (unlink(__DIR__ . '/..' . "$backup_dir/$entry")) {
if (unlink( "$backup_dir/$entry")) {
echo 'Cleaned up old backups';
}
}
......
......@@ -14,6 +14,7 @@ if ($_SERVER['SERVER_ADDR'] !== $_SERVER['REMOTE_ADDR']) {
use LanguageDetection\Language;
use Poduptime\PodStatus;
use RedBeanPHP\R;
use GeoIp2\Database\Reader;
$debug = isset($_GET['debug']) || (isset($argv) && in_array('debug', $argv, true));
$sqldebug = isset($_GET['sqldebug']) || (isset($argv) && in_array('sqldebug', $argv, true));
......@@ -36,6 +37,9 @@ $sqldebug && R::fancyDebug(true);
R::testConnection() || die('Error in DB connection');
R::usePartialBeans(true);
// Setup GeoIP Database
$reader = new Reader($geoip2db);
try {
$sql = '
SELECT domain, score, date_created, adminrating, weight, hidden, podmin_notify, email, masterversion, shortversion, status
......@@ -235,14 +239,15 @@ foreach ($pods as $pod) {
_debug('IPv6', $ipv6);
_debug('Iplookupv6', $iplookupv6, true);
$location = geoip_record_by_name($ip);
_debug('Location', $location, true);
$countryname = !empty($location['country_name']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['country_name']) : null;
$country = !empty($location['country_code']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['country_code']) : null;
$city = !empty($location['city']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['city']) : null;
$state = !empty($location['region']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['region']) : null;
$lat = !empty($location['latitude']) ? $location['latitude'] : 0;
$long = !empty($location['longitude']) ? $location['longitude'] : 0;
$geo = $reader->city($ip);
$countryname = $geo->country->name ?? null ?: null;
$country = $geo->country->isoCode ?? null ?: null;
$city = $geo->city->name ?? null ?: null;
$state = $geo->mostSpecificSubdivision->name ?? null ?: null;
$lat = $geo->location->latitude ?? null ?: 0;
$long = $geo->location->longitude ?? null ?: 0;
_debug('Location', json_encode($geo->raw), true);
echo $newline;
$statslastdate = date('Y-m-d H:i:s');
......
#!/usr/bin/env sh
#!/usr/bin/env bash
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
cd "$SCRIPT_DIR"
FLAG_FILE="/tmp/poduptime.pulling"
HOUR=`date +%H`
HOUR=$(date +%H)
DAY=$(date +%d)
HAPPY=" :-)"
SAD=" :-("
......@@ -24,7 +25,7 @@ if ! wget -q --spider --tries=2 --timeout=15 https://www.google.com; then
fi
echo "$HAPPY"
if [ "$HOUR" = 23 ]; then
if [ "$HOUR" = 23 ] || [ "$@" = "init" ]; then
printf "%s" "Pulling in master versions..."
if php pull-masterversions.php; then
echo "$HAPPY"
......@@ -43,11 +44,21 @@ if [ "$HOUR" = 23 ]; then
else
echo "$SAD"
fi
printf "%s" "Updating CA..."
if wget -q https://curl.haxx.se/ca/cacert.pem -O ../cacert.pem; then
echo "$HAPPY"
else
echo "$SAD"
if [ "$DAY" = 23 ] || [ "$@" = "init" ]; then
printf "%s" "Updating CA..."
CACERT_FILE="$(php -r "include __DIR__ . '/../config.php'; echo \$cafullpath;")"
if curl -Lss https://curl.haxx.se/ca/cacert.pem -o "$CACERT_FILE"; then
echo "$HAPPY"
else
echo "$SAD"
fi
printf "%s" "Updating GeoIP2 DB..."
GEODB_FILE="$(php -r "include __DIR__ . '/../config.php'; echo \$geoip2db;")"
if funzip <(curl -Lss http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz) > "$GEODB_FILE"; then
echo "$HAPPY"
else
echo "$SAD"
fi
fi
fi
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment