MultilingualCrawler/PageContent.py at main · dodinhvit23f/MultilingualCrawler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
from bs4 import BeautifulSoup
import Utility
import pdb

def getTextFromTags(html, tag):
    soup = BeautifulSoup(html, "lxml")
    text = ""

    for tag_text in soup.findAll(tag):

        tagText = Utility.formatString(tag_text.text)
        tagText = tagText.replace("\n+", " ")
        tagText = tagText.replace("/.", "")
        tagText = tagText.strip()
        # ký tự lạ xuất hiện tại một số bài báo

        if tagText and not tagText == "":
            text += tagText

    text =  bytes(text, "utf-8").decode('utf-8', 'ignore')
    return text

def getTextFromTagsWithClass(html, tag, class_):
    soup = BeautifulSoup(html, "lxml")
    text = ""

    for tag_text in soup.findAll(tag, class_=class_):

        tagText = Utility.formatString(tag_text.text)
        tagText = tagText.replace("\n+", " ")
        tagText = tagText.replace("/.", "")
        tagText = tagText.strip()
        # ký tự lạ xuất hiện tại một số bài báo

        if tagText and not tagText == "":
            text += tagText

    text =  bytes(text, "utf-8").decode('utf-8', 'ignore')
    return text

def getTextFromTagsWithId(html, tag, id_):
    soup = BeautifulSoup(html, "lxml")
    text = ""

    for tag_text in soup.findAll(tag, {"id":id_} ):
        #
        tagText = Utility.formatString(tag_text.text)
        tagText = tagText.replace("\n+", " ")
        tagText = tagText.replace("/.", "")
        tagText = tagText.strip()
        # ký tự lạ xuất hiện tại một số bài báo

        if tagText and not tagText == "":
            text += tagText

    text = bytes(text, "utf-8").decode('utf-8', 'ignore')
    return text

def getTextFromTags(html, tag):
    soup = BeautifulSoup(html, "lxml")
    text = ""

    for j in soup.findAll(tag):

        text_in_timestamp = Utility.formatString(j.text)
        text_in_timestamp = text_in_timestamp.replace("/.", "")
        text_in_timestamp = text_in_timestamp.strip()
        if text_in_timestamp:
            text += text_in_timestamp

    text = bytes(text, "utf-8").decode('utf-8', 'ignore')
    return text

# từ đây là các hàm để lấy dự liệu các trang web theo từ tên miền
def getVovNewsContent(html):
    """
    :param html: VOV HTML content
    :return:
    """
    return getTextFromTagsWithClass(html, "article", "article")

def getVnanetNewsContet(html):
    return getTextFromTagsWithClass(html, "div", "detail")

def getCanThoNewsContent(html):
    return getTextFromTagsWithId(html, tag="div", id_="newscontents")

def getVietLaoVietNamNewsContent(html):
    return getTextFromTagsWithClass(html, tag="div", class_="post")

def getVietNamPlusNewsContent(html):

    div = getTextFromTagsWithClass(html, "div", "article-body")
    if not div:
        div = getTextFromTagsWithClass(html, "div", "details")
    if not div:
        div = getTextFromTagsWithId(html, "div", "page")
    return div

def getTNUNewsContent(html, lang = "vi"):
    content = ""
    if (lang == "zh" or lang == "en"):
        content = getTextFromTagsWithId(html, tag="div", id="wrapper")
        return content

    return getTextFromTagsWithId(html, tag="div", id="container")

def getQuanDoiNhanDan(html, lang = "vi"):
    if lang == "vi":
        return getTextFromTagsWithId(html = html, tag ="div", id_="dnn_VIEWSINDEX_ctl00_viewhinone")
    if lang == "en":
        return getTextFromTagsWithClass(html, tag="div", class_="NewsNexttop")
    if lang == "zh":
        text = getTextFromTagsWithClass(html, tag="div", class_="detail-post hnoneview")
        if (text != " " or text != ""):
            return text

        return getTextFromTagsWithClass(html, tag="div", class_="col-sm-8")

    if lang == "km":
        text = getTextFromTagsWithClass(html, tag="div", class_="detail-post hnoneview")
        if(text != " " or text != ""):
            return text
        return getTextFromTagsWithClass(html, tag="div", class_="bgqdnd")

    if lang == "lo":
        return getTextFromTagsWithId(html=html, tag="div", id_="dnn_NewsView_Main_ctl00_viewhinone")

def getNhanDanNewsContent(html, lang = "vi"):
    if lang != "vi":
        return getTextFromTagsWithClass(html, "div", "col-sm-8")

    return getTextFromTagsWithClass(html, "div", "detail-page")

def getBCCNewContent(html):
   return getTextFromTags(html, "main")

def getTapChiCongSanConntent(html, lang = "vi"):

    if lang == "vi":
        return getTextFromTagsWithClass(html, "div", "ContentDetail")

    return getTextFromTagsWithClass(html, "div", "journal-content-article")

#print(getQuanDoiNhanDan(requests.get("https://kh.qdnd.vn/preview/pid/27/newid/513765").content, lang= "km"))