no message

Author: rory0807

moumoubaimifan/wxCrawler/articles.py

@@ -0,0 +1,66 @@
+# articles.py
+import html
+import requests
+import utils
+
+from urllib.parse import urlsplit
+
+class Articles(object):
+    """文章信息"""
+
+    def __init__(self, appmsg_token, cookie):
+        # 具有时效性
+        self.appmsg_token = appmsg_token
+
+        self.headers = {
+            "User-Agent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0Chrome/57.0.2987.132 MQQBrowser/6.2 Mobile",
+            "Cookie": cookie
+        }
+
+        self.data = {
+            "is_only_read": "1",
+            "is_temp_url": "0",
+            "appmsg_type": "9",
+        }
+
+
+    def read_like_nums(self, article_url):
+        """获取数据"""
+        appmsgstat = self.get_appmsgext(article_url)["appmsgstat"]
+        return appmsgstat["read_num"], appmsgstat["old_like_num"], appmsgstat["like_num"]
+
+    def get_params(self, article_url):
+        """
+        获取到文章url上的请求参数
+        :param article_url: 文章 url
+        :return:
+        """
+        # url转义处理
+        article_url = html.unescape(article_url)
+        """获取文章链接的参数"""
+        url_params = utils.str_to_dict(urlsplit(article_url).query, "&", "=")
+        return url_params
+
+    def get_appmsgext(self, article_url):
+        """
+        请求阅读数
+        :param article_url: 文章 url
+        :return:
+        """
+        url_params = self.get_params(article_url)
+
+        appmsgext_url = "https://mp.weixin.qq.com/mp/getappmsgext?appmsg_token={}&x5=0".format(self.appmsg_token)
+        self.data.update(url_params)
+
+        appmsgext_json = requests.post(
+            appmsgext_url, headers=self.headers, data=self.data).json()
+
+        if "appmsgstat" not in appmsgext_json.keys():
+            raise Exception(appmsgext_json)
+        return appmsgext_json
+
+
+if __name__ == '__main__':
+    info = Articles('1068_XQoMoGGBYG8Tf8k23jfdBr2H_LNekAAlDDUe2aG13TN2fer8xOSMyrLV6s-yWESt8qg5I2fJr1r9n5Y5', 'rewardsn=; wxtokenkey=777; wxuin=1681274216; devicetype=android-29; version=27001037; lang=zh_CN; pass_ticket=H9Osk2CMhrlH34mQ3w2PLv/RAVoiDxweAdyGh/Woa1qwGy2jGATJ6hhg7syTQ9nk; wap_sid2=COjq2KEGEnBPTHRVOHlYV2U4dnRqaWZqRXBqaWl3Xy1saXVWYllIVjAzdlM1VkNDNHgxeWpHOG9pckdkREMwTFEwYmNWMl9FZWtRU3pRRnhDS0pyV1BaZUVMWXN1ZWN0WnZ6aHFXdVBnbVhTY21BYnBSUXNCQUFBMLLAjfgFOA1AAQ==')
+    a, b,c = info.read_like_nums('http://mp.weixin.qq.com/s?__biz=MzU1NDk2MzQyNg==&mid=2247486254&idx=1&sn=c3a47f4bf72b1ca85c99190597e0c190&chksm=fbdad3a3ccad5ab55f6ef1f4d5b8f97887b4a344c67f9186d5802a209693de582aac6429a91c&scene=27#wechat_redirect')
+    print(a, b, c)

moumoubaimifan/wxCrawler/cookie.txt

@@ -0,0 +1,2 @@
+https://mp.weixin.qq.com/mp/getappmsgext?f=json&mock=&uin=777&key=777&pass_ticket=H9Osk2CMhrlH34mQ3w2PLv/RAVoiDxweAdyGh/Woa1qwGy2jGATJ6hhg7syTQ9nk&wxtoken=777&devicetype=android-29&clientversion=27001037&__biz=MzU1NDk2MzQyNg==&enterid=1594059276&appmsg_token=1068_ar7adip424A4GEBXj3ICCnwArRb2kU4Y5Y5m9QrePtCDqEng3qndGHMimpnfnqU7wbKfavW3dYDqknmJ&x5=0&f=json
+MultiDictView[['rewardsn', ''], ['wxtokenkey', '777'], ['wxuin', '1681274216'], ['devicetype', 'android-29'], ['version', '27001037'], ['lang', 'zh_CN'], ['pass_ticket', 'H9Osk2CMhrlH34mQ3w2PLv/RAVoiDxweAdyGh/Woa1qwGy2jGATJ6hhg7syTQ9nk'], ['wap_sid2', 'COjq2KEGEnBPTHRVOHlYV2U4dnRqaWZqRXBqaWkydmtSVUtDSjNVYVQteFRnd0N5V2RQQkMxUWdOZktORlZoLU5pTEpOZndCTWdTRHNYbktjRnpZa09nOTBoUWZTajFzX1VWaFRxNEZWbUdwbDM3VjNtWXNCQUFBMNfUjfgFOA1AAQ==']]

moumoubaimifan/wxCrawler/main.py

@@ -0,0 +1,13 @@
+# coding:utf-8
+# main.py
+from read_cookie import ReadCookie
+from wxCrawler import WxCrawler
+
+"""程序启动类"""
+if __name__ == '__main__':
+    cookie = ReadCookie('E:/python/cookie.txt')
+
+    cookie.write_cookie()
+    appmsg_token, biz, cookie_str = cookie.parse_cookie()
+    wx = WxCrawler(appmsg_token, biz, cookie_str)
+    wx.run()

moumoubaimifan/wxCrawler/read_cookie.py

@@ -0,0 +1,47 @@
+# read_cookie.py
+import re
+import os
+
+class ReadCookie(object):
+    """
+    启动write_cookie.py 和 解析cookie文件，
+    """
+
+    def __init__(self, outfile):
+        self.outfile = outfile
+
+    def parse_cookie(self):
+        """
+        解析cookie
+        :return: appmsg_token, biz, cookie_str·
+        """
+        f = open(self.outfile)
+        lines = f.readlines()
+        appmsg_token_string = re.findall("appmsg_token.+?&", lines[0])
+        biz_string = re.findall('__biz.+?&', lines[0])
+        appmsg_token = appmsg_token_string[0].split("=")[1][:-1]
+        biz = biz_string[0].split("__biz=")[1][:-1]
+
+        cookie_str = '; '.join(lines[1][15:-2].split('], [')).replace('\'','').replace(', ', '=')
+        return appmsg_token, biz, cookie_str
+
+    def write_cookie(self):
+        """
+        启动 write_cookie。py
+        :return:
+        """
+
+        #当前文件路径
+        path = os.path.split(os.path.realpath(__file__))[0]
+        # mitmdump -s 执行脚本 -w 保存到文件 本命令
+        command = "mitmdump -s {}/write_cookie.py -w {} mp.weixin.qq.com/mp/getappmsgext".format(
+            path, self.outfile)
+
+        os.system(command)
+
+
+if __name__ == '__main__':
+    rc = ReadCookie('cookie.txt')
+    rc.write_cookie()
+    appmsg_token, biz, cookie_str = rc.parse_cookie()
+    print("appmsg_token：" + appmsg_token , "\nbiz：" + biz, "\ncookie："+cookie_str)

moumoubaimifan/wxCrawler/utils.py

@@ -0,0 +1,11 @@
+# utils.py
+# 工具模块，将字符串变成字典
+def str_to_dict(s, join_symbol="\n", split_symbol=":"):
+    s_list = s.split(join_symbol)
+    data = dict()
+    for item in s_list:
+        item = item.strip()
+        if item:
+            k, v = item.split(split_symbol, 1)
+            data[k] = v.strip()
+    return data

moumoubaimifan/wxCrawler/write_cookie.py

@@ -0,0 +1,34 @@
+# coding: utf-8
+import urllib
+import sys
+
+from mitmproxy import http
+
+# command: mitmdump -s write_cookie.py -w outfile mp.weixin.qq.com/mp/getappmsgext
+
+class WriterCookie:
+    """
+    mitmproxy的监听脚本，写入cookie和url到文件
+    """
+
+    def __init__(self, outfile):
+        self.f = open(outfile, "w")
+
+    def response(self, flow: http.HTTPFlow) -> None:
+        """
+        完整的response响应
+        :param flow: flow实例，
+        """
+        # 获取url
+        url = urllib.parse.unquote(flow.request.url)
+
+        # 将url和cookie写入文件
+        if "mp.weixin.qq.com/mp/getappmsgext" in url:
+            self.f.write(url + '\n')
+            self.f.write(str(flow.request.cookies))
+            self.f.close()
+            # 退出
+            exit()
+
+# 第四个命令中的参数
+addons = [WriterCookie(sys.argv[4])]

moumoubaimifan/wxCrawler/wxCrawler.py

@@ -0,0 +1,83 @@
+# coding:utf-8
+# wxCrawler.py
+import os
+import time
+
+import requests
+import json
+import urllib3
+
+import utils
+from articles import Articles
+
+
+class WxCrawler(object):
+    """翻页内容抓取"""
+    urllib3.disable_warnings()
+
+    def __init__(self, appmsg_token, biz, cookie, begin_page_index = 0, end_page_index = 100):
+        # 起始页数
+        self.begin_page_index = begin_page_index
+        # 结束页数
+        self.end_page_index = end_page_index
+        # 抓了多少条了
+        self.num = 1
+
+        self.appmsg_token = appmsg_token
+        self.biz = biz
+        self.headers = {
+            "User-Agent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0Chrome/57.0.2987.132 MQQBrowser/6.2 Mobile",
+            "Cookie": cookie
+        }
+        self.cookie = cookie
+
+    def article_list(self, context):
+        articles = json.loads(context).get('general_msg_list')
+        return json.loads(articles)
+
+    def run(self):
+
+        # 翻页地址
+        page_url = "https://mp.weixin.qq.com/mp/profile_ext?action=getmsg&__biz={}&f=json&offset={}&count=10&is_ok=1&scene=&uin=777&key=777&pass_ticket={}&wxtoken=&appmsg_token=" + self.appmsg_token + "&x5=0f=json"
+        # 将 cookie 字典化
+        wx_dict = utils.str_to_dict(self.cookie, join_symbol='; ', split_symbol='=')
+        # 请求地址
+        response = requests.get(page_url.format(self.biz, self.begin_page_index * 10, wx_dict['pass_ticket']), headers=self.headers, verify=False)
+        # 将文章列表字典化
+        articles = self.article_list(response.text)
+        info = Articles(self.appmsg_token, self.cookie)
+
+        result = []
+        for a in articles['list']:
+            if 'app_msg_ext_info' in a.keys() and '' != a.get('app_msg_ext_info').get('content_url', ''):
+
+                read_num, old_like_num, like_num = info.read_like_nums(a.get('app_msg_ext_info').get('content_url'))
+                result.append(str(self.num) + '条,' + a.get('app_msg_ext_info').get('title') + ',' + str(read_num) + ',' + str(old_like_num) + ',' + str(like_num))
+                time.sleep(2)
+
+            if 'app_msg_ext_info' in a.keys():
+                for m in a.get('app_msg_ext_info').get('multi_app_msg_item_list', []):
+                    read_num, old_like_num, like_num = info.read_like_nums(m.get('content_url'))
+                    result.append(str(self.num) + '条的副条,' + m.get('title') + ',' + str(read_num) + ',' + str(old_like_num) + ',' + str(like_num))
+
+                    time.sleep(3)
+
+            self.num = self.num + 1
+
+        self.write_file(result)
+
+        self.is_exit_or_continue()
+        # 递归调用
+        self.run()
+
+    def write_file(self, result):
+        with open('微信公众号.csv', 'a') as f:
+            for row in result:
+                f.write(row + '\n')
+
+    def is_exit_or_continue(self):
+        self.begin_page_index = self.begin_page_index + 1
+
+        if self.begin_page_index > self.end_page_index:
+            print('公众号导出结束，共导出了' + str(self.end_page_index) + '页')
+            os.exit()