Merge branch 'pullmore' into 'develop'

Pull cleanup round 900

See merge request !207

CHANGELOG.md

@@ -7,6 +7,8 @@ The format is based on [Keep a Changelog] and this project adheres to [Semantic
 - Config syntax has changed to array style (#155)
 - Added `pghost` config to set database port
 - Added `CONTRIBUTING.md`
+- Podmin email shares details on why pod is failing
+- Only retrieve location data for remote servers / IPs
 ### Changed
 - Introduce proper changelog format (#189)
 - Moved DB migration scripts into `db` folder
@@ -15,6 +17,11 @@ The format is based on [Keep a Changelog] and this project adheres to [Semantic
 - Use pretty URLs (see nginx.example)
 - Open pod URLs in a new tab
 - Use detectlanguage.com API for language guess
+- Only use JSON data, ignore HTML when returned
+- Allow curl redirect on home page check
+- Score now goes to -5000 before a pod is removed so dead pods get checked a while then removed for good
+- Move functions to dedicated file to allow reuse
+- Backup script rewrite
 ### Deprecated
 ### Removed
 ### Fixed

boot.php

@@ -9,33 +9,6 @@ declare(strict_types=1);
 use DetectLanguage\DetectLanguage;
 use RedBeanPHP\R;
 
-/**
- * Helper to get config values.
- *
- * @param null|string $param
- * @param null|mixed  $default
- *
- * @return mixed|null
- */
-function c(string $param = null, $default = null)
-{
-    static $config;
-    if ($config === null) {
-        $config = require __DIR__ . '/config.php';
-        is_array($config) || die('Invalid config format.');
-    }
-
-    if ($param === null) {
-        return $config;
-    }
-
-    if (array_key_exists($param, $config)) {
-        return $config[$param];
-    }
-
-    return $default;
-}
-
 define('PODUPTIME', microtime(true));
 
 require_once __DIR__ . '/vendor/autoload.php';

composer.json

@@ -22,13 +22,15 @@
         "maxmind-db/reader": "^1.3",
         "matriphe/iso-639": "^1.2",
         "rinvex/country": "^3.1",
-        "detectlanguage/detectlanguage": "^2.2"
+        "detectlanguage/detectlanguage": "^2.2",
+        "longman/ip-tools": "^1.2"
     },
     "require-dev": {
         "squizlabs/php_codesniffer": "^3.3"
     },
     "autoload": {
-        "classmap": ["lib"]
+        "classmap": ["lib"],
+        "files": ["lib/functions.php"]
     },
     "scripts": {
         "check-code": [

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "ae2037263bedfa647fb7fe96311a84da",
+    "content-hash": "9ac3659b3b8b6f4a29e9a44e0dfd105f",
     "packages": [
         {
             "name": "commerceguys/enum",
@@ -289,6 +289,62 @@
             ],
             "time": "2018-07-30T20:23:10+00:00"
         },
+        {
+            "name": "longman/ip-tools",
+            "version": "1.2.1",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/akalongman/php-ip-tools.git",
+                "reference": "6c050dfbf91811d14b9b3aa31fb7116eac0f0a18"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/akalongman/php-ip-tools/zipball/6c050dfbf91811d14b9b3aa31fb7116eac0f0a18",
+                "reference": "6c050dfbf91811d14b9b3aa31fb7116eac0f0a18",
+                "shasum": ""
+            },
+            "require": {
+                "ext-bcmath": "*",
+                "php": ">=5.5"
+            },
+            "require-dev": {
+                "phpspec/phpspec": "~2.1",
+                "phpunit/phpunit": "~4.1",
+                "squizlabs/php_codesniffer": "~2.3"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Longman\\IPTools\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Avtandil Kikabidze aka LONGMAN",
+                    "email": "akalongman@gmail.com",
+                    "homepage": "https://longman.me",
+                    "role": "Developer"
+                }
+            ],
+            "description": "PHP IP Tools for manipulation with IPv4 and IPv6",
+            "homepage": "https://github.com/akalongman/php-ip-tools",
+            "keywords": [
+                "IP",
+                "Match",
+                "compare",
+                "ipv4",
+                "ipv6",
+                "mask",
+                "subnet",
+                "tools",
+                "utilities"
+            ],
+            "time": "2016-10-23T20:08:46+00:00"
+        },
         {
             "name": "matriphe/iso-639",
             "version": "1.2",

db/backup.php

@@ -6,25 +6,45 @@
 
 declare(strict_types=1);
 
-if (PHP_SAPI !== 'cli') {
+require_once __DIR__ . '/../boot.php';
+
+if (!is_cli()) {
     header('HTTP/1.0 403 Forbidden');
     exit;
 }
 
-$c = require __DIR__ . '/../config.php';
+$keep_for    = 60 * 60 * 6; // 6 hours
+$backup_file = c('backup_dir') . '/dump_' . date('Ymd_His') . '.sql';
+
+printf("Making backup of '%s' to '%s'...", c('pgdb'), $backup_file);
+system(sprintf(
+    'export PGPASSWORD=%3$s &&' .
+    '"%1$s" --clean --format=tar --username=%2$s %4$s >> "%5$s"',
+    c('pg_dump_dir') . '/pg_dump',
+    c('pguser'),
+    c('pgpass'),
+    c('pgdb'),
+    $backup_file
+), $exit_code);
+printf(" %s\n", $exit_code === 0 ? 'Success!' : 'Failed.');
 
-$keep      = (60 * 60 * 6) * 1;
-$dump_date = date('Ymd_Hs');
-$file_name = "{$c['backup_dir']}/dump_{$dump_date}.sql";
-system("export PGPASSWORD={$c['pgpass']} && {$c['pg_dump_dir']}/pg_dump --clean --format=tar --username={$c['pguser']} {$c['pgdb']} >> {$file_name}");
-echo "pg backup of {$c['pgdb']} made";
-$dirh = dir($c['backup_dir']);
+$dirh = dir(c('backup_dir'));
 while ($entry = $dirh->read()) {
-    $old_file_time = date('U') - $keep;
-    $file_created  = filectime("{$c['backup_dir']}/{$entry}");
-    if ($file_created < $old_file_time && !is_dir($entry)) {
-        if (unlink("{$c['backup_dir']}/{$entry}")) {
-            echo 'Cleaned up old backups';
-        }
+    $file = c('backup_dir') . "/{$entry}";
+
+    // Skip dotfiles and non-files (folders, symlinks, etc.).
+    if ($entry[0] === '.' || !is_file($file)) {
+        continue;
     }
+
+    if (filectime($file) + $keep_for > time()) {
+        //echo "Don't delete {$entry}\n";
+        continue;
+    }
+
+    printf(
+        "Removing old file '%s'... %s\n",
+        $entry,
+        unlink($file) ? 'Success!' : 'Failed.'
+    );
 }

db/monthly_stats.php

@@ -6,15 +6,15 @@
 
 declare(strict_types=1);
 
-if (PHP_SAPI !== 'cli') {
-    header('HTTP/1.0 403 Forbidden');
-    exit;
-}
-
 use RedBeanPHP\R;
 
 require_once __DIR__ . '/../boot.php';
 
+if (!is_cli()) {
+    header('HTTP/1.0 403 Forbidden');
+    exit;
+}
+
 try {
     $monthly_totals = R::getAll("
         SELECT

db/podcrawler.php

@@ -6,15 +6,15 @@
 
 declare(strict_types=1);
 
-if (PHP_SAPI !== 'cli') {
-    header('HTTP/1.0 403 Forbidden');
-    exit;
-}
-
 use RedBeanPHP\R;
 
 require_once __DIR__ . '/../boot.php';
 
+if (!is_cli()) {
+    header('HTTP/1.0 403 Forbidden');
+    exit;
+}
+
 try {
     $sql  = '
         SELECT domain, status

db/pull-masterversions.php

@@ -6,15 +6,15 @@
 
 declare(strict_types=1);
 
-if (PHP_SAPI !== 'cli') {
-    header('HTTP/1.0 403 Forbidden');
-    exit;
-}
-
 use RedBeanPHP\R;
 
 require_once __DIR__ . '/../boot.php';
 
+if (!is_cli()) {
+    header('HTTP/1.0 403 Forbidden');
+    exit;
+}
+
 $softwares = [
     'diaspora'     => ['repo' => 'diaspora/diaspora', 'gitsite' => 'api.github.com', 'gittype' => 'github', 'devbranch' => 'develop'],
     'friendica'    => ['repo' => 'friendica/friendica', 'gitsite' => 'api.github.com', 'gittype' => 'github', 'devbranch' => 'develop'],

lib/functions.php

+<?php
+
+/**
+ * Collection of functions.
+ */
+
+declare(strict_types=1);
+
+use DetectLanguage\DetectLanguage;
+
+/**
+ * Helper to get config values.
+ *
+ * @param null|string $param
+ * @param null|mixed  $default
+ *
+ * @return mixed|null
+ */
+function c(string $param = null, $default = null)
+{
+    static $config;
+    if ($config === null) {
+        $config = require __DIR__ . '/../config.php';
+        is_array($config) || die('Invalid config format.');
+    }
+
+    if ($param === null) {
+        return $config;
+    }
+
+    if (array_key_exists($param, $config)) {
+        return $config[$param];
+    }
+
+    return $default;
+}
+
+/**
+ * Check if started from cli.
+ *
+ * @return bool
+ */
+function is_cli(): bool
+{
+    if (defined('STDIN')) {
+        return true;
+    }
+
+    if (PHP_SAPI === 'cli') {
+        return true;
+    }
+
+    if (PHP_SAPI === 'cgi-fcgi') {
+        return true;
+    }
+
+    if (array_key_exists('SHELL', $_ENV)) {
+        return true;
+    }
+
+    if (empty($_SERVER['REMOTE_ADDR']) && !isset($_SERVER['HTTP_USER_AGENT']) && count($_SERVER['argv']) > 0) {
+        return true;
+    }
+
+    if (!array_key_exists('REQUEST_METHOD', $_SERVER)) {
+        return true;
+    }
+
+    return false;
+}
+
+/**
+ * Output a debug message and variable value
+ *
+ * @param string $label
+ * @param mixed  $var
+ * @param bool   $dump
+ *
+ * @return void
+ */
+function debug($label, $var = null, $dump = false): void
+{
+    global $debug, $newline;
+
+    if (!$debug) {
+        return;
+    }
+
+    $output = (string) $var;
+    if ($dump || is_array($var)) {
+        $output = print_r($var, true);
+    } elseif (is_bool($var)) {
+        $output = $var ? 'true' : 'false';
+    }
+
+    printf('%s: %s%s', $label, $output, $newline);
+}
+
+/**
+ * Execute cURL request and return array of data.
+ *
+ * @param string $url
+ * @param bool   $follow
+ *
+ * @return array
+ */
+function curl(string $url, bool $follow = false): array
+{
+    $chss = curl_init();
+    curl_setopt($chss, CURLOPT_URL, $url);
+    curl_setopt($chss, CURLOPT_CONNECTTIMEOUT, 5);
+    curl_setopt($chss, CURLOPT_TIMEOUT, 15);
+    curl_setopt($chss, CURLOPT_RETURNTRANSFER, true);
+    curl_setopt($chss, CURLOPT_FAILONERROR, true);
+    curl_setopt($chss, CURLOPT_CERTINFO, true);
+    curl_setopt($chss, CURLOPT_FOLLOWLOCATION, $follow);
+    curl_setopt($chss, CURLOPT_CAINFO, c('cafullpath'));
+    $data = [
+        'body'     => curl_exec($chss),
+        'error'    => curl_error($chss),
+        'info'     => curl_getinfo($chss, CURLINFO_CERTINFO),
+        'code'     => curl_getinfo($chss, CURLINFO_RESPONSE_CODE),
+        'conntime' => curl_getinfo($chss, CURLINFO_CONNECT_TIME),
+        'nstime'   => curl_getinfo($chss, CURLINFO_NAMELOOKUP_TIME),
+    ];
+    curl_close($chss);
+
+    return $data;
+}
+
+/**
+ * Get a language snippet from a given URL.
+ *
+ * @param string $url
+ *
+ * @return null|string
+ */
+function getWebsiteLanguageSnippetFromUrl(string $url): ?string
+{
+    $curl = curl($url, true);
+
+    if (!$curl['body']) {
+        return null;
+    }
+
+    libxml_use_internal_errors(true);
+
+    $d = new DOMDocument;
+    $d->loadHTML($curl['body']);
+
+    $snippet = $d->getElementsByTagName('title')->item(0)->textContent ?? '';
+
+    for ($type = 1; $type < 6; $type++) {
+        foreach ($d->getElementsByTagName('h' . $type) as $h) {
+            // Ignore possibly generic "JavaScript required" texts.
+            if (stripos($h->textContent, 'javascript') === false) {
+                $snippet .= ' ' . $h->textContent;
+            }
+        }
+    }
+
+    // Get descriptions of meta tags.
+    foreach ($d->getElementsByTagName('meta') as $meta) {
+        if (strtolower($meta->getAttribute('name')) === 'description') {
+            $snippet .= ' ' . $meta->getAttribute('value');
+        }
+    }
+
+    return $snippet;
+}
+
+/**
+ * Detect the language of the given text snippet.
+ *
+ * @param string $snippet
+ *
+ * @return null|string
+ */
+function detectWebsiteLanguageFromSnippet(string $snippet): ?string
+{
+    if (!$snippet) {
+        return null;
+    }
+
+    return DetectLanguage::simpleDetect($snippet);
+}
+
+/**
+ * Detect the website language of the given URL.
+ *
+ * @param string $url
+ *
+ * @return null|string
+ */
+function detectWebsiteLanguageFromUrl(string $url): ?string
+{
+    return detectWebsiteLanguageFromSnippet(
+        getWebsiteLanguageSnippetFromUrl($url)
+    );
+}

showmap.php

@@ -28,7 +28,8 @@ try {
         FROM pods
         WHERE long != ''
             AND lat != ''
-            AND status < ?
+            AND status < ? 
+            AND score > 0
     ", [PodStatus::RECHECK]);
 } catch (\RedBeanPHP\RedException $e) {
     die('Error in SQL query: ' . $e->getMessage());

statsviewjs.php

@@ -20,9 +20,10 @@ try {
             round(avg(uptime_alltime),2) AS uptime
         FROM pods
         WHERE status < ?
+        AND score > 0
         GROUP BY softwarename
         ORDER BY softwarename
-    ', [PodStatus::SYSTEM_DELETED]);
+    ', [PodStatus::PAUSED]);
 } catch (\RedBeanPHP\RedException $e) {
     die('Error in SQL query: ' . $e->getMessage());
 }

tabledata.php

@@ -18,9 +18,10 @@ try {
     $pods = R::getAll('
         SELECT domain, dnssec, podmin_statement, sslexpire, masterversion, shortversion, softwarename, daysmonitored, monthsmonitored, score, signup, name, country, countryname, city, state, detectedlanguage, uptime_alltime, active_users_halfyear, active_users_monthly, service_facebook, service_twitter, service_tumblr, service_wordpress, service_xmpp, latency, date_updated, ipv6, total_users, local_posts, comment_counts, userrating, status
         FROM pods
-        WHERE status < ?
+        WHERE status < ? 
+        AND score > 0
         ORDER BY weightedscore DESC
-    ', [PodStatus::SYSTEM_DELETED]);
+    ', [PodStatus::RECHECK]);
 } catch (\RedBeanPHP\RedException $e) {
     die('Error in SQL query: ' . $e->getMessage());
 }