-
Notifications
You must be signed in to change notification settings - Fork 1
/
parser_kuma.py
65 lines (52 loc) · 2.92 KB
/
parser_kuma.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Kuma RSS-voo sisendite parsimine
"""
import makereq
import parsers_common
def getArticleListsFromHtml(pageTree, domain, maxPageURLstoVisit):
"""
Meetod uudistesaidi kõigi uudiste nimekirja loomiseks
"""
articleDescriptions = pageTree.xpath('//div[@class="news-list-item-wrapper"]/div[@class="news-list-item-excerpt"]/p/text()')
articleIds = []
articleImages = pageTree.xpath('//div[@class="news-list-media"]/img/@src')
articleImages = parsers_common.domainUrls(domain, articleImages)
articlePubDates = []
articleTitles = pageTree.xpath('//div[@class="news-list-item-wrapper"]/h3/a/text()')
articleUrls = pageTree.xpath('//div[@class="news-list-item-wrapper"]/h3/a/@href')
articleUrls = parsers_common.domainUrls(domain, articleUrls)
articlePubDay = pageTree.xpath('//div[@class="news-list-item-wrapper"]/div[@class="news-list-item-date"]/text()[1]')
articlePubMonth = pageTree.xpath('//div[@class="news-list-item-wrapper"]/div[@class="news-list-item-date"]/span[@class="month"]/text()')
articlePubYear = pageTree.xpath('//div[@class="news-list-item-wrapper"]/div[@class="news-list-item-date"]/text()[2]')
get_article_bodies = True
for i in range(0, len(articleUrls)):
articleUrl = articleUrls[i]
# generate unical id from ArticleUrl
articleIds.append(parsers_common.urlToHash(articleUrl))
if (get_article_bodies is True and i < maxPageURLstoVisit):
# load article into tree
articleTree = makereq.getArticleData(articleUrl)
# descriptions
curArtDescParent = parsers_common.treeExtract(articleTree, '//div[@class="news-single-item"]/div[@class="news-single-content"]') # as a parent
curArtDescChilds = parsers_common.stringify_children(curArtDescParent)
articleDescriptions[i] = curArtDescChilds
# timeformat magic from "13 dets 17" to datetime()
curArtPubDate = parsers_common.treeExtract(articleTree, '//div[@class="news-single-timedata"]/text()')
curArtPubDate = parsers_common.shortMonthsToNumber(curArtPubDate)
curArtPubDate = parsers_common.rawToDatetime(curArtPubDate, "%d %m %y")
articlePubDates.append(curArtPubDate)
else:
if i < len(articlePubYear) and (int(articlePubYear[i].strip()) > 2016):
curYear = articlePubYear[i].strip()
curArtPubDate = articlePubDay[i].strip() + " " + articlePubMonth[i].strip() + " " + curYear
curArtPubDate = parsers_common.rawToDatetime(curArtPubDate, "%d %m %Y")
articlePubDates.append(curArtPubDate)
return {"articleDescriptions": articleDescriptions,
"articleImages": articleImages,
"articleIds": articleIds,
"articlePubDates": articlePubDates,
"articleTitles": articleTitles,
"articleUrls": articleUrls,
}