From 0d35258398ac2852fff4bce77bff4eec2439c8cf Mon Sep 17 00:00:00 2001 From: david Date: Sun, 8 Jul 2018 11:54:14 -0700 Subject: [PATCH] move to supported db for IP geo data --- composer.json | 3 +- composer.lock | 214 ++++++++++++++++++++++++++++++++++++++++++++- config.php.example | 11 ++- db/pull.php | 21 +++-- 4 files changed, 234 insertions(+), 15 deletions(-) diff --git a/composer.json b/composer.json index f2bc87b..dd0963e 100644 --- a/composer.json +++ b/composer.json @@ -12,7 +12,8 @@ "noplanman/xec": "0.1.0", "jaybizzle/crawler-detect": "1.*", "twbs/bootstrap": "^4.1.0", - "patrickschur/language-detection": "^3.3" + "patrickschur/language-detection": "^3.3", + "geoip2/geoip2": "^2.9" }, "require-dev": { "squizlabs/php_codesniffer": "^3.3" diff --git a/composer.lock b/composer.lock index 62315dd..f2f6969 100644 --- a/composer.lock +++ b/composer.lock @@ -1,10 +1,10 @@ { "_readme": [ "This file locks the dependencies of your project to a known state", - "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", + "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", "This file is @generated automatically" ], - "content-hash": "d447ea75efb92876c1bee13e93c2c5aa", + "content-hash": "1cd7c802d1459c30d044708a4ecc98b0", "packages": [ { "name": "commerceguys/enum", @@ -44,6 +44,62 @@ "description": "A PHP 5.4+ enumeration library.", "time": "2015-02-27T21:36:56+00:00" }, + { + "name": "composer/ca-bundle", + "version": "1.1.1", + "source": { + "type": "git", + "url": "https://github.com/composer/ca-bundle.git", + "reference": "d2c0a83b7533d6912e8d516756ebd34f893e9169" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/composer/ca-bundle/zipball/d2c0a83b7533d6912e8d516756ebd34f893e9169", + "reference": "d2c0a83b7533d6912e8d516756ebd34f893e9169", + "shasum": "" + }, + "require": { + "ext-openssl": "*", + "ext-pcre": "*", + "php": "^5.3.2 || ^7.0" + }, + "require-dev": { + "phpunit/phpunit": "^4.8.35 || ^5.7 || ^6.5", + "psr/log": "^1.0", + "symfony/process": "^2.5 || ^3.0 || ^4.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.x-dev" + } + }, + "autoload": { + "psr-4": { + "Composer\\CaBundle\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Jordi Boggiano", + "email": "j.boggiano@seld.be", + "homepage": "http://seld.be" + } + ], + "description": "Lets you find a path to the system CA bundle, and includes a fallback to the Mozilla CA bundle.", + "keywords": [ + "cabundle", + "cacert", + "certificate", + "ssl", + "tls" + ], + "time": "2018-03-29T19:57:20+00:00" + }, { "name": "gabordemooij/redbean", "version": "v5.1", @@ -85,6 +141,58 @@ ], "time": "2018-04-01T11:51:37+00:00" }, + { + "name": "geoip2/geoip2", + "version": "v2.9.0", + "source": { + "type": "git", + "url": "https://github.com/maxmind/GeoIP2-php.git", + "reference": "a807fbf65212eef5d8d2db1a1b31082b53633d77" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/maxmind/GeoIP2-php/zipball/a807fbf65212eef5d8d2db1a1b31082b53633d77", + "reference": "a807fbf65212eef5d8d2db1a1b31082b53633d77", + "shasum": "" + }, + "require": { + "maxmind-db/reader": "~1.0", + "maxmind/web-service-common": "~0.5", + "php": ">=5.4" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "2.*", + "phpunit/phpunit": "4.*", + "squizlabs/php_codesniffer": "3.*" + }, + "type": "library", + "autoload": { + "psr-4": { + "GeoIp2\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "Gregory J. Oschwald", + "email": "goschwald@maxmind.com", + "homepage": "http://www.maxmind.com/" + } + ], + "description": "MaxMind GeoIP2 PHP API", + "homepage": "https://github.com/maxmind/GeoIP2-php", + "keywords": [ + "IP", + "geoip", + "geoip2", + "geolocation", + "maxmind" + ], + "time": "2018-04-10T15:32:59+00:00" + }, { "name": "jaybizzle/crawler-detect", "version": "v1.2.64", @@ -134,6 +242,108 @@ ], "time": "2018-06-22T20:32:56+00:00" }, + { + "name": "maxmind-db/reader", + "version": "v1.3.0", + "source": { + "type": "git", + "url": "https://github.com/maxmind/MaxMind-DB-Reader-php.git", + "reference": "e042b4f8a2dff41e19019faf16427178b07fbd58" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/maxmind/MaxMind-DB-Reader-php/zipball/e042b4f8a2dff41e19019faf16427178b07fbd58", + "reference": "e042b4f8a2dff41e19019faf16427178b07fbd58", + "shasum": "" + }, + "require": { + "php": ">=5.4" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "2.*", + "phpunit/phpunit": "4.* || 5.*", + "satooshi/php-coveralls": "1.0.*", + "squizlabs/php_codesniffer": "3.*" + }, + "suggest": { + "ext-bcmath": "bcmath or gmp is required for decoding larger integers with the pure PHP decoder", + "ext-gmp": "bcmath or gmp is required for decoding larger integers with the pure PHP decoder", + "ext-maxminddb": "A C-based database decoder that provides significantly faster lookups" + }, + "type": "library", + "autoload": { + "psr-4": { + "MaxMind\\Db\\": "src/MaxMind/Db" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "Gregory J. Oschwald", + "email": "goschwald@maxmind.com", + "homepage": "http://www.maxmind.com/" + } + ], + "description": "MaxMind DB Reader API", + "homepage": "https://github.com/maxmind/MaxMind-DB-Reader-php", + "keywords": [ + "database", + "geoip", + "geoip2", + "geolocation", + "maxmind" + ], + "time": "2018-02-21T21:23:33+00:00" + }, + { + "name": "maxmind/web-service-common", + "version": "v0.5.0", + "source": { + "type": "git", + "url": "https://github.com/maxmind/web-service-common-php.git", + "reference": "61a9836fa3bb1743ab89752bae5005d71e78c73b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/maxmind/web-service-common-php/zipball/61a9836fa3bb1743ab89752bae5005d71e78c73b", + "reference": "61a9836fa3bb1743ab89752bae5005d71e78c73b", + "shasum": "" + }, + "require": { + "composer/ca-bundle": "^1.0.3", + "ext-curl": "*", + "ext-json": "*", + "php": ">=5.4" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "2.*", + "phpunit/phpunit": "4.*", + "squizlabs/php_codesniffer": "3.*" + }, + "type": "library", + "autoload": { + "psr-4": { + "MaxMind\\Exception\\": "src/Exception", + "MaxMind\\WebService\\": "src/WebService" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "Gregory Oschwald", + "email": "goschwald@maxmind.com" + } + ], + "description": "Internal MaxMind Web Service API", + "homepage": "https://github.com/maxmind/web-service-common-php", + "time": "2018-02-12T22:31:54+00:00" + }, { "name": "nesbot/carbon", "version": "1.31.1", diff --git a/config.php.example b/config.php.example index 4082a1c..53106b1 100644 --- a/config.php.example +++ b/config.php.example @@ -5,13 +5,13 @@ declare(strict_types=1); -//backup directory +//backup directory - app directory path $backup_dir = '/backup'; -//log directory +//log directory - app directory path $log_dir = '/log'; -//location of pg dump +//location of pg dump - full dir path $pg_dump_dir = '/usr/bin'; //db host @@ -32,8 +32,11 @@ $adminemail = ''; //DNS server for dnssec testing. 1.1.1.1 tests the best $dnsserver = ''; -//CA for curl to use - suggest wget http://curl.haxx.se/ca/cacert.pem as they pull from mozilla if you use a system CA then certs like wosign and startssl will be valid but users will be unable to connect to them +//CA for curl to use - full path $cafullpath = ''; //Mapbox.com API key. https://www.mapbox.com/help/how-access-tokens-work/ $mapboxkey = ''; + +//GeoIp2 city database file in mmdb format - full path +$geoip2db = ''; \ No newline at end of file diff --git a/db/pull.php b/db/pull.php index a16e21b..219eebb 100644 --- a/db/pull.php +++ b/db/pull.php @@ -14,6 +14,7 @@ if ($_SERVER['SERVER_ADDR'] !== $_SERVER['REMOTE_ADDR']) { use LanguageDetection\Language; use Poduptime\PodStatus; use RedBeanPHP\R; +use GeoIp2\Database\Reader; $debug = isset($_GET['debug']) || (isset($argv) && in_array('debug', $argv, true)); $sqldebug = isset($_GET['sqldebug']) || (isset($argv) && in_array('sqldebug', $argv, true)); @@ -36,6 +37,9 @@ $sqldebug && R::fancyDebug(true); R::testConnection() || die('Error in DB connection'); R::usePartialBeans(true); +// Setup GeoIP Database +$reader = new Reader($geoip2db); + try { $sql = ' SELECT domain, score, date_created, adminrating, weight, hidden, podmin_notify, email, masterversion, shortversion, status @@ -235,14 +239,15 @@ foreach ($pods as $pod) { _debug('IPv6', $ipv6); _debug('Iplookupv6', $iplookupv6, true); - $location = geoip_record_by_name($ip); - _debug('Location', $location, true); - $countryname = !empty($location['country_name']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['country_name']) : null; - $country = !empty($location['country_code']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['country_code']) : null; - $city = !empty($location['city']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['city']) : null; - $state = !empty($location['region']) ? iconv('UTF-8', 'UTF-8//IGNORE', $location['region']) : null; - $lat = !empty($location['latitude']) ? $location['latitude'] : 0; - $long = !empty($location['longitude']) ? $location['longitude'] : 0; + $geo = $reader->city($ip); + $countryname = !empty($geo->country->name) ? iconv('UTF-8', 'UTF-8//IGNORE', $geo->country->name) : null; + $country = !empty($geo->country->isoCode) ? iconv('UTF-8', 'UTF-8//IGNORE', $geo->country->isoCode) : null; + $city = !empty($geo->city->name) ? iconv('UTF-8', 'UTF-8//IGNORE', $geo->city->name) : null; + $state = !empty($geo->mostSpecificSubdivision->name) ? iconv('UTF-8', 'UTF-8//IGNORE', $geo->mostSpecificSubdivision->name) : null; + $lat = !empty($geo->location->latitude) ? $geo->location->latitude : 0; + $long = !empty($geo->location->longitude) ? $geo->location->longitude : 0; + + _debug('Location', json_encode($geo->raw), true); echo $newline; $statslastdate = date('Y-m-d H:i:s'); -- GitLab