Skip to content

Commit bef8a82

Browse files
timotei-litespeedtimotei-litespeed
authored
v7.1-b5: Move crawler status to const (#826)
1 parent 6eb6f9b commit bef8a82

File tree

4 files changed

+61
-46
lines changed

4 files changed

+61
-46
lines changed

‎cli/crawler.cls.php

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -84,14 +84,14 @@ public function list()
8484

8585
$list = array();
8686
foreach ($crawler_list as $i => $v) {
87-
$hit = !empty($summary['crawler_stats'][$i]['H']) ? $summary['crawler_stats'][$i]['H'] : 0;
88-
$miss = !empty($summary['crawler_stats'][$i]['M']) ? $summary['crawler_stats'][$i]['M'] : 0;
87+
$hit = !empty($summary['crawler_stats'][$i][Crawler2::STATUS_HIT]) ? $summary['crawler_stats'][$i][Crawler2::STATUS_HIT] : 0;
88+
$miss = !empty($summary['crawler_stats'][$i][Crawler2::STATUS_MISS]) ? $summary['crawler_stats'][$i][Crawler2::STATUS_MISS] : 0;
8989

90-
$blacklisted = !empty($summary['crawler_stats'][$i]['B']) ? $summary['crawler_stats'][$i]['B'] : 0;
91-
$blacklisted += !empty($summary['crawler_stats'][$i]['N']) ? $summary['crawler_stats'][$i]['N'] : 0;
90+
$blacklisted = !empty($summary['crawler_stats'][$i][Crawler2::STATUS_BLACKLIST]) ? $summary['crawler_stats'][$i][Crawler2::STATUS_BLACKLIST] : 0;
91+
$blacklisted += !empty($summary['crawler_stats'][$i][Crawler2::STATUS_NOCACHE]) ? $summary['crawler_stats'][$i][Crawler2::STATUS_NOCACHE] : 0;
9292

93-
if (isset($summary['crawler_stats'][$i]['W'])) {
94-
$waiting = $summary['crawler_stats'][$i]['W'] ?: 0;
93+
if (isset($summary['crawler_stats'][$i][Crawler2::STATUS_WAIT])) {
94+
$waiting = $summary['crawler_stats'][$i][Crawler2::STATUS_WAIT] ?: 0;
9595
} else {
9696
$waiting = $summary['list_size'] - $hit - $miss - $blacklisted;
9797
}

‎src/crawler-map.cls.php

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ public function save_map_status($list, $curr_crawler)
7171
$wpdb->query("UPDATE `$this->_tb` SET res = $sql_res WHERE id IN ( $id_all )");
7272

7373
// Add blacklist
74-
if ($bit == 'B' || $bit == 'N') {
74+
if ($bit == Crawler::STATUS_BLACKLIST || $bit == Crawler::STATUS_NOCACHE) {
7575
$q = "SELECT a.id, a.url FROM `$this->_tb_blacklist` a LEFT JOIN `$this->_tb` b ON b.url=a.url WHERE b.id IN ( $id_all )";
7676
$existing = $wpdb->get_results($q, ARRAY_A);
7777
// Update current crawler status tag in existing blacklist
@@ -129,7 +129,7 @@ public function save_map_status($list, $curr_crawler)
129129
self::debug("Update map reason [code] $code [pos] left $curr_crawler right -$right_pos [count] $count");
130130

131131
// Update blacklist reason
132-
if ($bit == 'B' || $bit == 'N') {
132+
if ($bit == Crawler::STATUS_BLACKLIST || $bit == Crawler::STATUS_NOCACHE) {
133133
$count = $wpdb->query(
134134
"UPDATE `$this->_tb_blacklist` a LEFT JOIN `$this->_tb` b ON b.url = a.url SET a.reason=CONCAT(SUBSTRING_INDEX(a.reason, ',', $curr_crawler), '$code', SUBSTRING_INDEX(a.reason, ',', -$right_pos)) WHERE b.id IN (" .
135135
implode(',', $v2) .
@@ -161,7 +161,7 @@ public function blacklist_add($id)
161161

162162
// Build res&reason
163163
$total_crawler = count(Crawler::cls()->list_crawlers());
164-
$res = str_repeat('B', $total_crawler);
164+
$res = str_repeat(Crawler::STATUS_BLACKLIST, $total_crawler);
165165
$reason = implode(',', array_fill(0, $total_crawler, 'Man'));
166166

167167
$row = $wpdb->get_row("SELECT a.url, b.id FROM `$this->_tb` a LEFT JOIN `$this->_tb_blacklist` b ON b.url = a.url WHERE a.id = '$id'", ARRAY_A);
@@ -200,7 +200,15 @@ public function blacklist_del($id)
200200
$id = (int) $id;
201201
self::debug('blacklist delete [id] ' . $id);
202202

203-
$wpdb->query("UPDATE `$this->_tb` SET res=REPLACE(REPLACE(res, 'N', '-'), 'B', '-') WHERE url=(SELECT url FROM `$this->_tb_blacklist` WHERE id='$id')");
203+
$sql = sprintf(
204+
"UPDATE `%s` SET res=REPLACE(REPLACE(res, '%s', '-'), '%s', '-') WHERE url=(SELECT url FROM `%s` WHERE id=%d)",
205+
$this->_tb,
206+
Crawler::STATUS_NOCACHE,
207+
Crawler::STATUS_BLACKLIST,
208+
$this->_tb_blacklist,
209+
$id
210+
);
211+
$wpdb->query($sql);
204212
$wpdb->query("DELETE FROM `$this->_tb_blacklist` WHERE id='$id'");
205213
}
206214

@@ -219,7 +227,8 @@ public function blacklist_empty()
219227
}
220228

221229
self::debug('Truncate blacklist');
222-
$wpdb->query("UPDATE `$this->_tb` SET res=REPLACE(REPLACE(res, 'N', '-'), 'B', '-')");
230+
$sql = sprintf("UPDATE `%s` SET res=REPLACE(REPLACE(res, '%s', '-'), '%s', '-')", $this->_tb, Crawler::STATUS_NOCACHE, Crawler::STATUS_BLACKLIST);
231+
$wpdb->query($sql);
223232
$wpdb->query("TRUNCATE `$this->_tb_blacklist`");
224233
}
225234

@@ -304,13 +313,13 @@ public function list_map($limit, $offset = false)
304313
if (!empty($_POST['kw'])) {
305314
$q = "SELECT * FROM `$this->_tb` WHERE url LIKE %s";
306315
if ($type == 'hit') {
307-
$q .= " AND res LIKE '%H%'";
316+
$q .= " AND res LIKE '%" . Crawler::STATUS_HIT . "%'";
308317
}
309318
if ($type == 'miss') {
310-
$q .= " AND res LIKE '%M%'";
319+
$q .= " AND res LIKE '%" . Crawler::STATUS_MISS . "%'";
311320
}
312321
if ($type == 'blacklisted') {
313-
$q .= " AND res LIKE '%B%'";
322+
$q .= " AND res LIKE '%" . Crawler::STATUS_BLACKLIST . "%'";
314323
}
315324
$q .= ' ORDER BY id LIMIT %d, %d';
316325
$where = '%' . $wpdb->esc_like($_POST['kw']) . '%';
@@ -319,13 +328,13 @@ public function list_map($limit, $offset = false)
319328

320329
$q = "SELECT * FROM `$this->_tb`";
321330
if ($type == 'hit') {
322-
$q .= " WHERE res LIKE '%H%'";
331+
$q .= " WHERE res LIKE '%" . Crawler::STATUS_HIT . "%'";
323332
}
324333
if ($type == 'miss') {
325-
$q .= " WHERE res LIKE '%M%'";
334+
$q .= " WHERE res LIKE '%" . Crawler::STATUS_MISS . "%'";
326335
}
327336
if ($type == 'blacklisted') {
328-
$q .= " WHERE res LIKE '%B%'";
337+
$q .= " WHERE res LIKE '%" . Crawler::STATUS_BLACKLIST . "%'";
329338
}
330339
$q .= ' ORDER BY id LIMIT %d, %d';
331340
// self::debug("q=$q offset=$offset, limit=$limit");
@@ -347,13 +356,13 @@ public function count_map()
347356

348357
$type = Router::verify_type();
349358
if ($type == 'hit') {
350-
$q .= " WHERE res LIKE '%H%'";
359+
$q .= " WHERE res LIKE '%" . Crawler::STATUS_HIT . "%'";
351360
}
352361
if ($type == 'miss') {
353-
$q .= " WHERE res LIKE '%M%'";
362+
$q .= " WHERE res LIKE '%" . Crawler::STATUS_MISS . "%'";
354363
}
355364
if ($type == 'blacklisted') {
356-
$q .= " WHERE res LIKE '%B%'";
365+
$q .= " WHERE res LIKE '%" . Crawler::STATUS_BLACKLIST . "%'";
357366
}
358367

359368
return $wpdb->get_var($q);

‎src/crawler.cls.php

Lines changed: 26 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,12 @@ class Crawler extends Root
2626
const FAST_USER_AGENT = 'lscache_runner';
2727
const CHUNKS = 10000;
2828

29+
const STATUS_WAIT = 'W';
30+
const STATUS_HIT = 'H';
31+
const STATUS_MISS = 'M';
32+
const STATUS_BLACKLIST = 'B';
33+
const STATUS_NOCACHE = 'N';
34+
2935
private $_sitemeta = 'meta.data';
3036
private $_resetfile;
3137
private $_end_reason;
@@ -909,10 +915,10 @@ private function _multi_request($rows, $options)
909915
$CRAWLER_DROP_DOMAIN = defined('LITESPEED_CRAWLER_DROP_DOMAIN') ? LITESPEED_CRAWLER_DROP_DOMAIN : false;
910916
$curls = array();
911917
foreach ($rows as $row) {
912-
if (substr($row['res'], $this->_summary['curr_crawler'], 1) == 'B') {
918+
if (substr($row['res'], $this->_summary['curr_crawler'], 1) == self::STATUS_BLACKLIST) {
913919
continue;
914920
}
915-
if (substr($row['res'], $this->_summary['curr_crawler'], 1) == 'N') {
921+
if (substr($row['res'], $this->_summary['curr_crawler'], 1) == self::STATUS_NOCACHE) {
916922
continue;
917923
}
918924

@@ -971,10 +977,10 @@ private function _multi_request($rows, $options)
971977
// curl done
972978
$ret = array();
973979
foreach ($rows as $row) {
974-
if (substr($row['res'], $this->_summary['curr_crawler'], 1) == 'B') {
980+
if (substr($row['res'], $this->_summary['curr_crawler'], 1) == self::STATUS_BLACKLIST) {
975981
continue;
976982
}
977-
if (substr($row['res'], $this->_summary['curr_crawler'], 1) == 'N') {
983+
if (substr($row['res'], $this->_summary['curr_crawler'], 1) == self::STATUS_NOCACHE) {
978984
continue;
979985
}
980986
// self::debug('-----debug3');
@@ -1005,16 +1011,16 @@ private function _multi_request($rows, $options)
10051011
*/
10061012
private function _status2title($status)
10071013
{
1008-
if ($status == 'H') {
1014+
if ($status == self::STATUS_HIT) {
10091015
return '✅ Hit';
10101016
}
1011-
if ($status == 'M') {
1017+
if ($status == self::STATUS_MISS) {
10121018
return '😊 Miss';
10131019
}
1014-
if ($status == 'B') {
1020+
if ($status == self::STATUS_BLACKLIST) {
10151021
return '😅 Blacklisted';
10161022
}
1017-
if ($status == 'N') {
1023+
if ($status == self::STATUS_NOCACHE) {
10181024
return '😅 Blacklisted';
10191025
}
10201026
return '🛸 Unknown';
@@ -1030,40 +1036,40 @@ private function _status_parse($header, $code, $url)
10301036
{
10311037
// self::debug('http status code: ' . $code . ' [headers]', $header);
10321038
if ($code == 201) {
1033-
return 'H';
1039+
return self::STATUS_HIT;
10341040
}
10351041

10361042
if (stripos($header, 'X-Litespeed-Cache-Control: no-cache') !== false) {
10371043
// If is from DIVI, taken as miss
10381044
if (defined('LITESPEED_CRAWLER_IGNORE_NONCACHEABLE') && LITESPEED_CRAWLER_IGNORE_NONCACHEABLE) {
1039-
return 'M';
1045+
return self::STATUS_MISS;
10401046
}
10411047

10421048
// If blacklist is disabled
10431049
if ((defined('LITESPEED_CRAWLER_DISABLE_BLOCKLIST') && LITESPEED_CRAWLER_DISABLE_BLOCKLIST) || apply_filters('litespeed_crawler_disable_blocklist', false, $url)) {
1044-
return 'M';
1050+
return self::STATUS_MISS;
10451051
}
10461052

1047-
return 'N'; // Blacklist
1053+
return self::STATUS_NOCACHE; // Blacklist
10481054
}
10491055

10501056
$_cache_headers = array('x-qc-cache', 'x-lsadc-cache', 'x-litespeed-cache');
10511057

10521058
foreach ($_cache_headers as $_header) {
10531059
if (stripos($header, $_header) !== false) {
10541060
if (stripos($header, $_header . ': miss') !== false) {
1055-
return 'M'; // Miss
1061+
return self::STATUS_MISS; // Miss
10561062
}
1057-
return 'H'; // Hit
1063+
return self::STATUS_HIT; // Hit
10581064
}
10591065
}
10601066

10611067
// If blacklist is disabled
10621068
if ((defined('LITESPEED_CRAWLER_DISABLE_BLOCKLIST') && LITESPEED_CRAWLER_DISABLE_BLOCKLIST) || apply_filters('litespeed_crawler_disable_blocklist', false, $url)) {
1063-
return 'M';
1069+
return self::STATUS_MISS;
10641070
}
10651071

1066-
return 'B'; // Blacklist
1072+
return self::STATUS_BLACKLIST; // Blacklist
10671073
}
10681074

10691075
/**
@@ -1383,10 +1389,10 @@ public function display_status($status_row, $reason_set)
13831389

13841390
$_status_list = array(
13851391
'-' => 'default',
1386-
'M' => 'primary',
1387-
'H' => 'success',
1388-
'B' => 'danger',
1389-
'N' => 'warning',
1392+
self::STATUS_MISS => 'primary',
1393+
self::STATUS_HIT => 'success',
1394+
self::STATUS_BLACKLIST => 'danger',
1395+
self::STATUS_NOCACHE => 'warning',
13901396
);
13911397

13921398
$reason_set = explode(',', $reason_set);

‎tpl/crawler/summary.tpl.php

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -141,14 +141,14 @@
141141
</thead>
142142
<tbody>
143143
<?php foreach ($crawler_list as $i => $v) :
144-
$hit = !empty($summary['crawler_stats'][$i]['H']) ? $summary['crawler_stats'][$i]['H'] : 0;
145-
$miss = !empty($summary['crawler_stats'][$i]['M']) ? $summary['crawler_stats'][$i]['M'] : 0;
144+
$hit = !empty($summary['crawler_stats'][$i][Crawler::STATUS_HIT]) ? $summary['crawler_stats'][$i][Crawler::STATUS_HIT] : 0;
145+
$miss = !empty($summary['crawler_stats'][$i][Crawler::STATUS_MISS]) ? $summary['crawler_stats'][$i][Crawler::STATUS_MISS] : 0;
146146

147-
$blacklisted = !empty($summary['crawler_stats'][$i]['B']) ? $summary['crawler_stats'][$i]['B'] : 0;
148-
$blacklisted += !empty($summary['crawler_stats'][$i]['N']) ? $summary['crawler_stats'][$i]['N'] : 0;
147+
$blacklisted = !empty($summary['crawler_stats'][$i][Crawler::STATUS_BLACKLIST]) ? $summary['crawler_stats'][$i][Crawler::STATUS_BLACKLIST] : 0;
148+
$blacklisted += !empty($summary['crawler_stats'][$i][Crawler::STATUS_NOCACHE]) ? $summary['crawler_stats'][$i][Crawler::STATUS_NOCACHE] : 0;
149149

150-
if (isset($summary['crawler_stats'][$i]['W'])) {
151-
$waiting = $summary['crawler_stats'][$i]['W'] ?: 0;
150+
if (isset($summary['crawler_stats'][$i][Crawler::STATUS_WAIT])) {
151+
$waiting = $summary['crawler_stats'][$i][Crawler::STATUS_WAIT] ?: 0;
152152
} else {
153153
$waiting = $summary['list_size'] - $hit - $miss - $blacklisted;
154154
}

0 commit comments

Comments
 (0)