This repository has been archived by the owner on Jul 7, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 15
/
algolia_push_articles.js
104 lines (78 loc) · 2.44 KB
/
algolia_push_articles.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
var algoliasearch = require('algoliasearch');
var cheerio = require('cheerio');
var fs = require('fs');
var slugify = require('slugify');
slugify.extend({'/': '-'});
var algolia_config = require("./algolia");
var client = algoliasearch(algolia_config.application_id, algolia_config.api_key);
var index = client.initIndex(algolia_config.index_name);
var basePath = algolia_config.site_path;
var folders = ["eli5", "history", "horizen", "privacy", "resources", "technology"];
// Loop through all the files in the temp directory
var walkSync = function (dir, filelist) {
var files = fs.readdirSync(basePath + "/" + dir);
filelist = filelist || [];
files.forEach(function (file) {
if (fs.statSync(basePath + "/" + dir + "/" + file).isDirectory()) {
filelist = walkSync(dir + "/" + file, filelist);
} else {
filelist.push({
path: dir + "/" + file,
url: dir + "/",
id: slugify(dir)
});
}
});
return filelist;
};
var FILE_LIST = [
{
path: "index.html",
url: "/",
id: 'homepage'
}
];
folders.forEach(function (f) {
FILE_LIST = walkSync(f, FILE_LIST);
});
var MAX_LENGTH = 9500; // limit 10KB
var ALGOLIA_OBJECTS = [];
FILE_LIST.forEach(function (f) {
var $ = cheerio.load(fs.readFileSync(basePath + "/" + f.path));
var content = $(".post-content").text();
content = content.replace(/\s\s+/g, ' ');
if (content.length > MAX_LENGTH) {
content = content.substr(0, MAX_LENGTH);
}
var topic = null, level = null;
if ($(".js-PostContent").length) {
topic = $(".js-PostContent").data("topic");
level = $(".js-PostContent").data("level");
}
if (topic === "technology" && level === "expert") {
// TODO: temporary ignore Technology Expert
return;
}
if (topic === "privacy" && level === "expert") {
// TODO: temporary ignore Privacy Expert
return;
}
ALGOLIA_OBJECTS.push({
objectID: f.id,
url: f.url,
title: $("title").text(),
content: content,
topic: topic,
level: level
});
});
// clear index before update
index.clearIndex(function(err, content) {
if (err) throw err;
console.log(content);
// update latest articles
index.saveObjects(ALGOLIA_OBJECTS, function (err, content) {
if (err) throw err;
console.log(content);
});
});