MultilingualCrawler/SaveFile.py at main · dodinhvit23f/MultilingualCrawler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
import datetime
import os
import Utility
import AlignmentNews
def saveSentences (src_text, tgt_text, file_path, src_lang_, tgt_lang_):
    time = datetime.datetime.now()

    if src_text and tgt_text:

        if not src_text or not tgt_text:
            print("empty file: " + file_path)
        else:
            f = open(file_path+"-" + src_lang_ + "-" + tgt_lang_ + ".txt", "w", encoding='utf-8')

            # code này để tạm thời
            lim_src = len(src_text)
            lim_tgt = len(tgt_text)
            lim = lim_tgt if lim_tgt >= lim_src else lim_src
            text_src = ""
            text_tgt = ""
            for index in range(0, lim):

                if (index < lim_src):
                    text_src = src_text[index]

                if (index < lim_tgt):
                    text_tgt = tgt_text[index]
                # if tgt_text[index] != "" and
                f.write(text_src + "\t" + text_tgt + "\n")

                text_src = ""
                text_tgt = ""
            f.close()

def saveDocument (src_text, tgt_text, file_path, src_lang_, tgt_lang_):
    file_path = file_path.replace("*","")
    if src_text and tgt_text:
        f = open(file_path + ".{}.txt".format(src_lang_), "w", encoding='utf-8')
        f.write(src_text + "\n")
        f.close()

        f = open(file_path + ".{}.txt".format(tgt_lang_), "w", encoding='utf-8')
        f.write(tgt_text + "\n")
        f.close()
    return

def save_data(src_text, file_path):
    if not src_text:
        print("empty file: ")
    else:
        if os.path.isfile(file_path + ".txt"):
            return

        f = open(file_path + ".txt", "w", encoding='utf-8')

        # code này để tạm thời
        lim_src = len(src_text)

        f.write(src_text + "\n")

        f.close()

    return

def saveJsonFile(file_path, link_dict):
    f = open(file_path, "w", encoding="utf-8")
    f.write(Utility.objectToJson(link_dict))
    f.close()

def saveTranslatedTitle(crawl_folder,src_lang, tgt_lang, list_tgt_title, translated = False):
    list_translate = AlignmentNews.translate(src_lang, tgt_lang, list_tgt_title)

    if not translated:
        f = open(crawl_folder + "/link/title.txt", "w", encoding="utf-8")
        for x, y in zip(list_translate, list_tgt_title):
            f.write(Utility.objectToJson({"vi": x, tgt_lang: y}))
            f.write("\n")
        f.close()
        return

    f = open(crawl_folder + "/link/title.txt", "a", encoding="utf-8")
    for x, y in zip(list_translate, list_tgt_title):
        f.write(Utility.objectToJson({"vi": x, tgt_lang: y}))
        f.write("\n")
    f.close()

def openTranslatedTitle(crawl_folder, tgt_link):
    list_trans = list()
    f = open(crawl_folder + "/link/title.txt", "r", encoding="utf-8")
    for line in f:
        list_trans.append(Utility.stringJsonToOject(line.strip()))
    f.close()

    for link in tgt_link:
        start = 0
        for title in list_trans:
            if (title['lo'] == link['title']):
                link['title'] = title['vi']
                break
            start = start + 1
        del (list_trans[start])

def loadJsonFile(file_path):

    f = open(file_path, "r", encoding="utf-8")
    for line in f:
        src_link = Utility.stringJsonToOject(line)
    f.close()

    return src_link