Unverified Commit 37ccff13 authored by David Morley's avatar David Morley Committed by GitHub

users line chart (#136)

* WIP stab at a users line chart

* one way to make it a 2 sec query

* move to a new table

* much closer now

* directory not needed here

* cleanup

* fix uptime

* WIP stab at a users line chart

* one way to make it a 2 sec query

* move to a new table

* much closer now

* directory not needed here

* cleanup

* fix uptime

* why ip6 is so different than ip, well fixed now we just need t/f for ip6

* Slight code cleanup and remove redundant timestamp assignment.

* Simplify IPv6 block, as we don't need the address itself any more.

* Update composer requirements.
parent 9e6f6946
......@@ -39,7 +39,7 @@ sudo nano /etc/postgresql/vx.x/main/pg_hba.conf
# restart postgresql
# import database structure
psql -u podupuser podupdb < db/tables.sql
psql -U podupuser podupdb < db/tables.sql
```
Edit `config.php` to add your DB and file settings.
......@@ -53,7 +53,7 @@ cd Poduptime
git pull
bower install
composer install
psql -u podupuser podupdb < db/migrationx.sql (see db/version.md for proper migration version)
psql -u podupuser podupdb < db/migrationx.sql (see db/version.md for proper migration versions)
```
============================
......
{
"_readme": [
"This file locks the dependencies of your project to a known state",
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "b753491dc03e2084e9587d0bbafad98c",
......@@ -46,16 +46,16 @@
},
{
"name": "gabordemooij/redbean",
"version": "v5.0",
"version": "v5.1",
"source": {
"type": "git",
"url": "https://github.com/gabordemooij/redbean.git",
"reference": "44229acdf493e2229c1e33e3add211f23c05ee4d"
"reference": "a02e58ad5519149f572559a8293b5dffea385956"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/gabordemooij/redbean/zipball/44229acdf493e2229c1e33e3add211f23c05ee4d",
"reference": "44229acdf493e2229c1e33e3add211f23c05ee4d",
"url": "https://api.github.com/repos/gabordemooij/redbean/zipball/a02e58ad5519149f572559a8293b5dffea385956",
"reference": "a02e58ad5519149f572559a8293b5dffea385956",
"shasum": ""
},
"require": {
......@@ -75,28 +75,28 @@
{
"name": "Gabor de Mooij",
"email": "gabor@redbeanphp.com",
"homepage": "http://redbeanphp.com"
"homepage": "https://redbeanphp.com"
}
],
"description": "RedBeanPHP ORM",
"homepage": "http://redbeanphp.com/",
"homepage": "https://redbeanphp.com/",
"keywords": [
"orm"
],
"time": "2017-10-19T21:38:38+00:00"
"time": "2018-04-01T11:51:37+00:00"
},
{
"name": "jaybizzle/crawler-detect",
"version": "v1.2.61",
"version": "v1.2.62",
"source": {
"type": "git",
"url": "https://github.com/JayBizzle/Crawler-Detect.git",
"reference": "9f22764fa72f6a30211fe16cf28a1f43642136de"
"reference": "f9767578e00f87a081835b49adc7c71074a5b46c"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/9f22764fa72f6a30211fe16cf28a1f43642136de",
"reference": "9f22764fa72f6a30211fe16cf28a1f43642136de",
"url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/f9767578e00f87a081835b49adc7c71074a5b46c",
"reference": "f9767578e00f87a081835b49adc7c71074a5b46c",
"shasum": ""
},
"require": {
......@@ -132,7 +132,7 @@
"crawlerdetect",
"php crawler detect"
],
"time": "2018-03-19T20:27:17+00:00"
"time": "2018-03-27T18:25:43+00:00"
},
{
"name": "noplanman/xec",
......
CREATE TABLE monthlystats (
id serial8 UNIQUE PRIMARY KEY,
total_users int,
total_posts int,
total_comments int,
total_pods int,
total_uptime int,
date_checked timestamp DEFAULT current_timestamp
);
<?php
use RedBeanPHP\R;
require_once __DIR__ . '/../vendor/autoload.php';
require_once __DIR__ . '/../config.php';
define('PODUPTIME', microtime(true));
// Set up global DB connection.
R::setup("pgsql:host={$pghost};dbname={$pgdb}", $pguser, $pgpass, true);
R::testConnection() || die('Error in DB connection');
R::usePartialBeans(true);
try {
$monthly_totals = R::getAll("
SELECT
to_char(date_checked, 'yyyy-mm') AS yymm,
sum(total_users) / count(DISTINCT to_char(date_checked, 'HH24 yy')) / count(DISTINCT to_char(date_checked, 'dd yy')) as users,
sum(local_posts) / count(DISTINCT to_char(date_checked, 'HH24 yy')) / count(DISTINCT to_char(date_checked, 'dd yy')) as posts,
sum(comment_counts) / count(DISTINCT to_char(date_checked, 'HH24 yy')) / count(DISTINCT to_char(date_checked, 'dd yy')) as comments,
count(domain) / count(DISTINCT to_char(date_checked, 'HH24 yy')) / count(DISTINCT to_char(date_checked, 'dd yy')) as pods,
count(nullif(online, false)) as uptime,
count(nullif(online, true)) as downtime
FROM checks
GROUP BY yymm
");
} catch (\RedBeanPHP\RedException $e) {
die('Error in SQL query: ' . $e->getMessage());
}
foreach ($monthly_totals as $monthly) {
// Format date to timestamp.
$timestamp = $monthly['yymm'].'-01 01:01:01-01';
try {
$p = R::findOrCreate('monthlystats', ['date_checked' => $timestamp]);
$p['total_users'] = $monthly['users'];
$p['total_posts'] = $monthly['posts'];
$p['total_comments'] = $monthly['comments'];
$p['total_pods'] = $monthly['pods'];
if ($monthly['downtime']) {
$p['total_uptime'] = round($monthly['downtime'] / $monthly['uptime'] * 100);
} else {
$p['total_uptime'] = 100;
}
R::store($p);
} catch (\RedBeanPHP\RedException $e) {
die('Error in SQL query: ' . $e->getMessage());
}
}
......@@ -179,16 +179,11 @@ foreach ($pods as $pod) {
preg_match('/A\s(.*)/', $getaonly[0], $aversion);
$ip = trim($aversion[1]) ?? '';
}
$ipv6 = false;
$iplookupv6 = explode(PHP_EOL, trim($delv->execute(['AAAA'], null, 15)->stdout));
$getaaaaonly = array_values(preg_grep('/\s+IN\s+AAAA\s+.*/', $iplookupv6));
if ($getaaaaonly) {
preg_match('/AAAA\s(.*)/', $getaaaaonly[0], $aaaaversion);
$ipv6 = trim($aaaaversion[1]) ?? '';
}
$ip || $score -= 2;
$iplookupv6 = explode(PHP_EOL, trim($delv->execute(['AAAA'], null, 15)->stdout));
$ipv6 = (bool) preg_grep('/\s+IN\s+AAAA\s+.*/', $iplookupv6);
_debug('IPv4', $ip);
_debug('Iplookupv4', $iplookupv4, true);
_debug('IPv6', $ipv6);
......
......@@ -24,11 +24,16 @@ if ! wget -q --spider --tries=2 --timeout=15 https://www.google.com; then
fi
echo "$HAPPY"
if [ "$HOUR" = 6 ]; then
if [ "$HOUR" = 1 ]; then
echo "Pulling in master versions...";
php pull-masterversions.php
echo
printf "%s" "Updating Monthy Stats Table..."
if php monthly_stats.php; then
echo "$HAPPY"
else
echo "$SAD"
fi
printf "%s" "Updating CA..."
if wget -q https://curl.haxx.se/ca/cacert.pem -O ../cacert.pem; then
echo "$HAPPY"
......
......@@ -48,6 +48,7 @@ CREATE TABLE pods (
date_laststats timestamp DEFAULT current_timestamp,
date_created timestamp DEFAULT current_timestamp
);
CREATE TABLE rating_comments (
id serial8 UNIQUE PRIMARY KEY,
domain text NOT NULL,
......@@ -59,6 +60,7 @@ CREATE TABLE rating_comments (
userurl text,
date_created timestamp DEFAULT current_timestamp
);
CREATE TABLE apikeys (
id serial8 UNIQUE PRIMARY KEY,
key text,
......@@ -94,3 +96,13 @@ CREATE TABLE masterversions (
version text,
date_checked timestamp DEFAULT current_timestamp
);
CREATE TABLE monthlystats (
id serial8 UNIQUE PRIMARY KEY,
total_users int,
total_posts int,
total_comments int,
total_pods int,
total_uptime int,
date_checked timestamp DEFAULT current_timestamp
);
If new install import tables.sql and do not perform migrations
If upgrading migrations are:
v1.0 -> v2.0 = migration00001.sql
v2.0 -> v2.1 = migration00002.sql
v2.1 -> v2.x = migration00003.sql
v2.1 -> v2.1.3 = migration00003.sql
v2.1.4 -> v2.x = migration00004.sql
To support the original apiv1 you should import:
pods_apiv1.sql
......
<div class="container-fluid">
<div class="row">
<div class="col-xs-12 col-sm-6 col-lg-4 text-center">Users by network<br>
<canvas id="total_network_users" width="100" height="100"></canvas>
<center><canvas id="total_network_users" width="300" height="350"></canvas></center>
</div>
<div class="col-xs-12 col-sm-6 col-lg-4 text-center">Pods by network<br>
<canvas id="total_network_pods" width="100" height="100"></canvas>
<center><canvas id="total_network_pods" width="300" height="350"></canvas></center>
</div>
<div class="col-xs-12 col-sm-6 col-lg-4 text-center">Uptime by network<br>
<canvas id="total_network_uptime" width="100" height="100"></canvas>
<center><canvas id="total_network_uptime" width="300" height="350"></canvas></center>
</div>
</div>
<br>
<div class="row">
<div class="col-lg-12 text-center">Average Network User Growth<br>
<canvas id="user_growth" width="1000" height="100"></canvas>
</div>
</div>
</div>
......@@ -19,6 +19,19 @@ try {
die('Error in SQL query: ' . $e->getMessage());
}
try {
$check_totals = R::getAll("
SELECT
to_char(date_checked, 'yyyy-mm') AS yymm,
total_users AS users
FROM monthlystats
GROUP BY yymm, users
ORDER BY yymm
");
} catch (\RedBeanPHP\RedException $e) {
die('Error in SQL query: ' . $e->getMessage());
}
?>
<script>
/**
......@@ -38,6 +51,28 @@ try {
hoverBackgroundColor: ["#FF6360", "#36A2AD", "#FFCE10", "#419615", "#A569AA", "#EB980A"]
}]
},
options: {
responsive: false,
maintainAspectRatio: false
}
});
}
function addLineChart(id, data) {
new Chart(document.getElementById(id), {
type: "line",
data: {
labels: <?php echo json_encode(array_column($check_totals, 'yymm')); ?>,
datasets: [{
data: <?php echo json_encode(array_column($check_totals, 'users')); ?>,
label: 'Users',
fill: false,
borderColor: "#2ecc71",
backgroundColor: "#2ecc71",
borderWidth: 4,
pointHoverRadius: 6
}
]
},
options: {
responsive: true,
maintainAspectRatio: true
......@@ -48,4 +83,5 @@ try {
addPieChart('total_network_users', <?php echo json_encode(array_column($totals, 'users')); ?>);
addPieChart('total_network_pods', <?php echo json_encode(array_column($totals, 'pods')); ?>);
addPieChart('total_network_uptime', <?php echo json_encode(array_column($totals, 'uptime')); ?>);
addLineChart('user_growth', <?php echo json_encode(array_column($check_totals, 'users')); ?>);
</script>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment