forked from allanlepp/te_rss
-
Notifications
You must be signed in to change notification settings - Fork 0
/
parser_wordpress.py
45 lines (30 loc) · 2.26 KB
/
parser_wordpress.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
import parsers_common
import parsers_datetime
def fill_article_dict(articleDataDict, pageTree, domain):
articleDataDict["titles"] = parsers_common.xpath_to("list", pageTree, '//a[@class="uu-list-title"]/text()')
articleDataDict["urls"] = parsers_common.xpath_to("list", pageTree, '//a[@class="uu-list-title"]/@href')
# muudame suuna sobivaks
articleDataDict = parsers_common.dict_reverse_order(articleDataDict)
for i in parsers_common.article_urls_range(articleDataDict["urls"]):
# titles
curArtTitle = parsers_common.get(articleDataDict["titles"], i)
curArtTitle = parsers_common.str_remove_clickbait(curArtTitle)
articleDataDict["titles"] = parsers_common.list_add_or_assign(articleDataDict["titles"], i, curArtTitle)
if parsers_common.should_get_article_body(i):
curArtUrl = parsers_common.get(articleDataDict["urls"], i)
# load article into tree
pageTree = parsers_common.get_article_tree(domain, curArtUrl, cache='cacheAll')
# author
curArtAuthor = parsers_common.xpath_to("single", pageTree, '/html/head/meta[@name="twitter:data1"]/@content')
articleDataDict["authors"] = parsers_common.list_add_or_assign(articleDataDict["authors"], i, curArtAuthor)
# description
curArtDesc = parsers_common.xpath_to("single", pageTree, '//div[@class="col-12 col-lg-8 mx-lg-auto uu-content-text"]', parent=True)
articleDataDict["descriptions"] = parsers_common.list_add_or_assign(articleDataDict["descriptions"], i, curArtDesc)
# image
curArtImg = parsers_common.xpath_to("single", pageTree, '/html/head/meta[@property="og:image"]/@content')
articleDataDict["images"] = parsers_common.list_add_or_assign(articleDataDict["images"], i, curArtImg)
# pubDates magic from "2022-10-23T10:11:18+00:00" to datetime()
curArtPubDate = parsers_common.xpath_to("single", pageTree, '/html/head/meta[@property="article:published_time"]/@content')
curArtPubDate = parsers_datetime.raw_to_datetime(curArtPubDate, "%Y-%m-%dt%H:%M:%S%z")
articleDataDict["pubDates"] = parsers_common.list_add_or_assign(articleDataDict["pubDates"], i, curArtPubDate)
return articleDataDict