Commit dfb5f29d authored by dmorley's avatar dmorley

Merge branch 'speedup-update' into 'develop'

Speedup update

See merge request !248
parents 6de9a4e3 ba1054a7
Pipeline #2986 failed with stage
in 5 seconds
......@@ -11,8 +11,10 @@ The format is based on [Keep a Changelog] and this project adheres to [Semantic
## [2.6.3]
### Added
- limit pull to 500 random pods per hour as the entire pull is 7 hours now. TEMP fix
### Changed
- js for table no longer filters pods that are signup only, this is a 20 second improvment on page load
- update remote data less often to save update time
### Deprecated
### Removed
### Fixed
......
......@@ -66,7 +66,7 @@ try {
$sql .= ' WHERE status = ? ORDER BY id';
$pods = R::getAll($sql, [PodStatus::SYSTEM_DELETED]);
} else {
$sql .= ' WHERE status < ? ORDER BY id';
$sql .= ' WHERE status < ? ORDER BY random() LIMIT 450';
$pods = R::getAll($sql, [PodStatus::PAUSED]);
}
}
......@@ -108,7 +108,7 @@ foreach ($pods as $pod) {
$user_rating = round(array_sum($user_ratings) / count($user_ratings), 2);
}
$nodeinfo_meta = curl("https://{$domain}/.well-known/nodeinfo");
$nodeinfo_meta = curl("https://{$domain}/.well-known/nodeinfo", false, 5);
// Default link to fetch node info.
$nodeinfo_url = "https://{$domain}/nodeinfo/1.0";
......@@ -125,7 +125,7 @@ foreach ($pods as $pod) {
debug('Nodeinfo link', $nodeinfo_url);
$nodeinfo = curl($nodeinfo_url);
$nodeinfo = curl($nodeinfo_url, false, 5);
$outputssl = $nodeinfo['body'];
$outputsslerror = $nodeinfo['error'];
$info = $nodeinfo['info'];
......@@ -277,18 +277,20 @@ foreach ($pods as $pod) {
$detectedlanguage = $language;
$languagesupdated = false;
$language_snippet = getWebsiteLanguageSnippetFromUrl("https://{$domain}/");
if (!$language_snippet) {
$detectedlanguage = null;
--$score;
$podminhelp = 'Unable to render the html on your main page https://' . $domain;
} elseif ($debug || $langhours > 24) {
$detectedlanguage = detectWebsiteLanguageFromSnippet($language_snippet);
$languagesupdated = true;
debug('Detected Language', $detectedlanguage);
if ($debug || $langhours > 24) {
$language_snippet = getWebsiteLanguageSnippetFromUrl("https://{$domain}/");
if (!$language_snippet) {
$detectedlanguage = null;
--$score;
$podminhelp = 'Unable to render the html on your main page https://' . $domain;
} else {
$detectedlanguage = detectWebsiteLanguageFromSnippet($language_snippet);
$languagesupdated = true;
debug('Detected Language', $detectedlanguage);
}
}
if (!$jsonssl || !$language_snippet) {
if (!$jsonssl || !isset($language_snippet)) {
debug('Connection', 'Can not connect to pod');
try {
......@@ -403,7 +405,7 @@ foreach ($pods as $pod) {
if ($score < 1) {
$weightedscore = 0;
}
if ($score < -6000) {
if ($score < -101) {
$status = PodStatus::SYSTEM_DELETED;
}
......@@ -474,24 +476,24 @@ if (!$_domain) {
updateMeta('languages_updated');
}
if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('masterversions_updated', 'date_created'))->diffInHours() > 12 || $init) {
if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('masterversions_updated', 'date_created'))->diffInHours() > 48 || $init) {
require __DIR__ . '/update-masterversions.php';
}
if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('statstable_updated', 'date_created'))->diffInHours() > 23 || $init) {
if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('statstable_updated', 'date_created'))->diffInHours() > 48 || $init) {
require __DIR__ . '/update-monthly-stats.php';
}
if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('cacert_updated', 'date_created'))->diffInHours() > 40 || $init) {
if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('cacert_updated', 'date_created'))->diffInHours() > 400 || $init) {
copy('https://curl.haxx.se/ca/cacert.pem', c('cafullpath'));
updateMeta('cacert_updated');
}
if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('sitemap_updated', 'date_created'))->diffInHours() > 2 || $init) {
if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('sitemap_updated', 'date_created'))->diffInHours() > 48 || $init) {
require __DIR__ . '/update-sitemap.php';
}
if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('geoip_updated', 'date_created'))->diffInHours() > 45 || $init) {
if (Carbon::createFromFormat('Y-m-d H:i:s.u', getMeta('geoip_updated', 'date_created'))->diffInHours() > 400 || $init) {
copy('compress.zlib://http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz', c('geoip2db'));
updateMeta('geoip_updated');
}
......
......@@ -113,7 +113,13 @@ function curl(string $url, bool $follow = false, $timeout = 15): array
curl_setopt($chss, CURLOPT_TIMEOUT, $timeout);
curl_setopt($chss, CURLOPT_RETURNTRANSFER, true);
curl_setopt($chss, CURLOPT_FAILONERROR, true);
curl_setopt($chss, CURLOPT_CERTINFO, true);
curl_setopt($chss, CURLOPT_CERTINFO, false);
curl_setopt($chss, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4 );
curl_setopt($chss, CURLOPT_ENCODING, '');
curl_setopt($chss, CURLOPT_TCP_FASTOPEN, '1L');
curl_setopt($chss, CURLOPT_MAXCONNECTS, 50);
curl_setopt($chss, CURLOPT_SSL_FALSESTART, false);
curl_setopt($chss, CURLOPT_MAXREDIRS, 3);
curl_setopt($chss, CURLOPT_FOLLOWLOCATION, $follow);
curl_setopt($chss, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0');
curl_setopt($chss, CURLOPT_CAINFO, c('cafullpath'));
......
......@@ -18,7 +18,7 @@ $hiddensoftwares = c('hidden-softwares');
try {
$pods = R::getAll('
SELECT domain, dnssec, podmin_statement, sslexpire, masterversion, shortversion, softwarename, daysmonitored, monthsmonitored, score, signup, protocols, name, country, countryname, city, state, detectedlanguage, uptime_alltime, active_users_halfyear, active_users_monthly, services, service_xmpp, latency, date_updated, ipv6, total_users, local_posts, comment_counts, userrating, status
SELECT domain, dnssec, podmin_statement, masterversion, shortversion, softwarename, daysmonitored, monthsmonitored, score, signup, protocols, name, country, countryname, city, state, detectedlanguage, uptime_alltime, active_users_halfyear, active_users_monthly, services, service_xmpp, latency, date_updated, ipv6, total_users, local_posts, comment_counts, userrating, status, date_laststats
FROM pods
WHERE status < ?
AND score > 0
......@@ -34,7 +34,8 @@ $country_code = $_SERVER['HTTP_CF_IPCOUNTRY'] ?? '';
foreach ($pods as $pod) {
$humanmonitored = Carbon::now()->subDays($pod['daysmonitored'])->diffForHumans(null, true);
$tip = "Over the last {$humanmonitored} uptime was {$pod['uptime_alltime']}% and response time from Los Angeles was {$pod['latency']}ms.";
$last_podcheck = Carbon::createFromFormat('Y-m-d H:i:s', $pod['date_laststats'])->diffForHumans(null, true);
$tip = "Pod last checked {$last_podcheck} ago, over the last {$humanmonitored} uptime was {$pod['uptime_alltime']}% and response time from Los Angeles was {$pod['latency']}ms.";
if (($_COOKIE['domain'] ?? null) === $pod['domain']) {
echo '<tr><td title="This is the last pod you visited from this site. ' . $tip . '" data-placement="right" data-toggle="tooltip" class="bg-blue"><a class="text-white url" target="_pod" href="/go.php?domain=' . $pod['domain'] . '">' . $pod['domain'] . '</a></td>';
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment