python mitmdump抓包與redis訂閱消息
阿新 • • 發佈：2017-05-31
import ole function col chan render androi reg 開始
本實例實現需求
django項目，後端采用python mitmdump 擴展腳本“sdk_log.py”實時抓取與過濾4399SDK 客戶端日誌，並且使用redis發布。
前端使用websocket連接，訂閱某頻道信息，實時輸出對應遊戲的客戶端日誌到頁面中。
開發環境
win7，python3,
安裝redis_server
參考 在windows x64上部署使用Redis
安裝python redis
python3 -m pip install redis
安裝python mitmproxy
python3 -m pip install mitmproxy
代碼實現
一、客戶端日誌抓包處理腳本 sdk_log.py:
#!/usr/bin/env python # -*- coding: utf-8 -*- from mitmproxy import http import urllib import re import json import logging import redis # 對日誌的輸出格式及方式做相關配置 logging.basicConfig(level=logging.DEBUG, format=‘%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s‘, datefmt=‘%a, %d %b %Y %H:%M:%S‘, filename=‘sdk.log‘, filemode=‘a‘) # 過濾目標host的日誌 host_filter = [ ‘udpdcs.4399sy.com‘, ‘sdkdcs.4399sy.com‘, ‘dpdcs.4399sy.com.hk‘, ‘dpdcs.4399en.com‘, ‘dpdcs.4399th.com‘, ‘dpdcs.4399sy.ru‘, ‘dpdcs.moregame.vn‘, ] # 大陸聯運操作名（從path中獲取,如activity_open.php） udpdcs_action_from_path = { ‘init_info‘: ‘初始化日誌‘, ‘activity_open‘: ‘打開遊戲日誌‘, ‘activity_before_login‘: ‘登錄界面前日誌‘, ‘user_login‘: ‘登錄日誌‘, ‘user_server_login‘: ‘選服日誌‘, ‘user_online‘: ‘在線日誌‘, ‘enter_game‘: ‘進入遊戲日誌‘, ‘share‘: ‘share‘, ‘share_log‘: ‘share_log‘, ‘photo_share‘: ‘photo_share‘, ‘event‘: ‘event‘ } # 從path中獲取操作類型，如activity_open.php path_filter = { # ‘load_start_before_login‘: u‘登錄前加載開始日誌‘, # ‘load_finish_before_login‘: u‘登錄前加載結束日誌‘, # ‘click_enter‘: u‘進入遊戲日誌‘, # ‘enter_game‘:u‘進入遊戲‘, # ‘get_user_server_login‘: u‘選服日誌‘, # ‘user_create_role‘: u‘創角日誌‘, # ‘role_login‘: u‘角色登錄日誌‘, # ‘enter_success‘: u‘成功進入遊戲日誌‘, # ‘role_level‘: u‘角色升級日誌‘, # ‘exit_success‘: u‘退出遊戲日誌‘, } # 大陸sy操作名(從body中獲取) sdkdcs_action_from_data = { ‘open_game‘: ‘打開遊戲日誌‘, ‘open_login‘: ‘登錄界面前日誌‘, ‘select_server‘: ‘選服日誌‘, ‘create_role‘: ‘創角日誌‘, ‘role_level_change‘: ‘等級日誌‘, } pool = redis.ConnectionPool(host=‘127.0.0.1‘, port=6379, db=1) r = redis.StrictRedis(connection_pool=pool) def response(flow): host = flow.request.host method = flow.request.method body = flow.request.data url = urllib.parse.unquote(flow.request.url) path = flow.request.path_components querystring = flow.request._get_query() gameId = None # 大陸聯運日誌，Android使用GET方法,iOS使用POST方法 if host == "udpdcs.4399sy.com": # GET 方式 （聯運Android） if method == "GET": # 從path中獲取操作類型 action_type = path[0].rstrip(".php") action_name = udpdcs_action_from_path.get(action_type, "udpdcs_action_from_path[%s]" % action_type) # 從URL參數data中獲取主要sdk請求數據 data = {} for eachp in querystring: if eachp[0] == "data": data = eachp[1] try: data_send = json.loads(data) gameId = data_send.get(‘gameId‘) print("gameId:", gameId) log_msg = "大陸聯運[%s]:\n-->GET URL:%s" % (action_name, url) logging.info(log_msg) except Exception as e: log_msg = "大陸聯運[%s]:\n-->GET 方法中的data內容轉換json格式失敗\n-->GET URL：%s" % (action_name, url) logging.error(log_msg) # 從URL參數中匹配data參數失敗 if not data: log_msg = "大陸聯運[%s]:\n-->GET 方法中的data內容獲取失敗，\n-->GET URL：%s" % (action_name, url) logging.warning(log_msg) # POST 方式(聯運iOS) else: # 從path中獲取操作類型 action_type = path[0].rstrip(".php") action_name = udpdcs_action_from_path.get(action_type, "udpdcs_action_from_path[%s]" % action_type) # 從POST BODY 中匹配‘data=(.*)\‘$‘ 獲得主要sdk請求數據 raw_data = str(body.content) data_patten = re.compile(‘data=(.*)\‘$‘, re.S) data_search = re.search(data_patten, raw_data) if data_search: data_send = urllib.parse.unquote(data_search.group(1)) data_send = data_send.replace("\n", "").replace("\t", "").replace(" ", "") try: data_send = json.loads( data_send) # eg. data_send={"did":"83F6B45E-6DA9-4B78-9DC0-000278B44F84","gameId":"1483512079389590" ...} gameId = data_send.get(‘gameId‘) log_msg = "大陸聯運[%s]:\n-->POST URL:%s,\n-->POST DATA:%s" % (action_name, url, data_send) logging.info(log_msg) except Exception as e: log_msg = "大陸聯運[%s]:\n-->POST 方法中的POST DATA 內容轉換json格式失敗!\n-->POST URL:%s\n-->POST DATA:%s" % ( action_name, url, data_send) logging.error(log_msg) # 從POST BODY 中匹配‘data=(.*)\‘$‘ 數據失敗 else: log_msg = "大陸聯運[%s]:\n-->POST 方法中的BODY data=內容獲取失敗!\n-->POST URL：%s\n-->POST DATA：%s" % ( action_name, url, urllib.parse.unquote(raw_data)) logging.warning(log_msg) print(log_msg) # 大陸sy日誌，Android，iOS使用POST方法 if host == "sdkdcs.4399sy.com": if method == "POST": # 從POST BODY 中匹配‘data=(.*)\‘$‘ 獲得主要sdk請求數據 raw_data = str(body.content) data_patten = re.compile(‘data=(.*)\‘$‘, re.S) data_search = re.search(data_patten, raw_data) if data_search: data_send = urllib.parse.unquote(data_search.group(1)) data_send = data_send.replace("\n", "").replace("\t", "").replace(" ", "") try: data_send = json.loads(data_send) # eg. data_send={"data":{"common":{},"open_game":{}}} data = data_send.get(‘data‘) if data: key_list = list(data.keys()) if "common" in key_list and len(key_list) == 2: gameId = data.get("common").get("gameId") print("gameId:", gameId) # 獲取操作類型 key_list.remove("common") action_type = key_list.pop(0) action_name = sdkdcs_action_from_data.get(action_type, "sdkdcs_action_from_data[%s]" % action_type) log_msg = "大陸sy[%s]:\n-->POST URL:%s \n-->POST DATA:%s" % (action_name, url, data_send) logging.info(log_msg) else: log_msg = "大陸sy[]:\n-->BODY 字典中沒有common這個key\n-->POST URL:%\n-->POST DATA：%s" % (url, data_send) logging.warning(log_msg) else: log_msg = "大陸sy[]:\n-->BODY 字典中沒有data這個key \n-->POST URL:% \n-->POST DATA：%s" % (url, data_send) logging.warning(log_msg) except json.decoder.JSONDecodeError as e: log_msg = "大陸sy[]:\n-->POST 方法中的POST DATA 內容轉換json格式失敗!\n-->POST URL:%s \n-->POST DATA:%s" % (url, data_send) logging.error(log_msg) except Exception as e: print(e) # 從POST BODY 中匹配‘data=(.*)\‘$‘ 數據失敗 else: log_msg = "大陸sy[]:\n-->POST 方法中的BODY data=內容獲取失敗!\n-->POST URL：%s\n-->POST DATA：%s" % ( url, urllib.parse.unquote(raw_data)) logging.warning(log_msg) else: log_msg = "大陸sy[]:\n-->大陸日誌使用了GET方式？\n-->GET URL：%s" % url logging.warning(log_msg) print(log_msg) # redis 發布 if gameId: r.publish(gameId, log_msg)
- 啟動抓包腳本
在cmd中輸入命令
mitmdump -s sdk_log.py ~u abc.com
正確啟動後如下
E:\workspace_python\tmp>mitmdump -s sdk_log.py ~u abc.com Loading script: sdk_log.py Proxy server listening at http://0.0.0.0:8080 192.168.1.103:39247: clientconnect
- 手機連接代理
手機連上與電腦相同局域網wifi，並設置代理。如電腦端ip為192.168.1.104，則設置代理為 192.168.1.104:8080 ,端口可以在mitmdump中添加參數修改，默認為8080
安裝證書：手機訪問mitm.it 下載安裝對應證書即可。
- 啟動手機遊戲
啟動任意一個四三九九遊戲，觀察控制臺日誌輸出，本實例以在4399sy.com中下載的安卓“翻滾球球”為例子。
192.168.1.103:40586: clientconnect 大陸聯運[初始化日誌]: -->GET 方法中的data內容獲取失敗， -->GET URL：http://udpdcs.4399sy.com/init_info.php?time=1496146854&flag=ee16ef51b6aee287a4f87be08aee2d6e gameId: 1461722512884260 大陸聯運[打開遊戲日誌]: -->GET URL:http://udpdcs.4399sy.com/activity_open.php?time=1496146854&flag=cab238218ccf3c3e7f8cf103d389e621&data={"eventId":"0","ip":"0","did":"861744030244058","appVersion":"1.4.4.0","sdkVersion":"2.6.9.3","platformId":"274","gameId":"1461722512884260","areaId":"0","serverId":"0","os":"android","osVersion":"5.1","device":"OPPO+R9m","deviceType":"android","screen":"1920*1080","mno":"中國電信","nm":"WIFI","eventTime":"0","channel":"4399_cz","channelOld":"4399_cz","channelSy":"113","sim":"0","kts":"23c023c0acefd77a11ca131f092bf85d","pkgName":"com.jingmo.ball3d"} gameId: 1461722512884260 大陸sy[打開遊戲日誌]: -->POST URL:http://sdkdcs.4399sy.com/?time=1496146854&flag=92aae983b4afc6b7236e29cbe19411f1 -->POST DATA:{‘data‘: {‘open_game‘: [{‘ip‘: ‘0‘, ‘appVersion‘: ‘1.4.4.0‘, ‘sdkVersion‘: ‘3.7.20.0‘, ‘channelId‘: ‘113‘, ‘nm‘: ‘WIFI‘, ‘retry‘: 1, ‘roleLevel‘: ‘0‘, ‘roleName‘: ‘‘, ‘nickname‘: ‘‘, ‘serverId‘: ‘0‘, ‘uid‘: ‘0‘, ‘eventTime‘: ‘1496146854‘, ‘msgId‘: ‘0‘, ‘ic‘: ‘MMHxiazowfGNkOjA=‘}], ‘common‘: {‘eventId‘: ‘0‘, ‘did‘: ‘861744030244058‘, ‘gameId‘: ‘1461722512884260‘, ‘os‘: ‘android‘, ‘osVersion‘: ‘5.1‘, ‘device‘: ‘OPPO+R9m‘, ‘deviceType‘: ‘android‘, ‘screen‘: ‘1920*1080‘, ‘mno‘: ‘中國電信‘, ‘areaId‘: ‘1‘}}} 192.168.1.103:41319: clientconnect
二、安裝dwebsocket
下載dwebsocket https://github.com/duanhongyi/dwebsocket 後，進行安裝
python setup.py install
三、django項目編寫
- 創建項目"dj_websocket",創建app"demo"
django-admin startproject dj_websocket
cd dj_websocket
django-admin startapp demo
- dj_websocket/url.py
from django.conf.urls import url from demo import views as v urlpatterns = [ # url(r‘^admin/‘, admin.site.urls), url(r‘^index/‘, v.index), url(r‘^echo$‘, v.echo), ]
- templates/index.html
<!DOCTYPE html> <html> <head> <title>django-websocket</title> <script src="http://code.jquery.com/jquery-1.11.1.min.js"></script> <script type="text/javascript">//<![CDATA[ $(function () { $(‘#connect_websocket‘).click(function () { if (window.s) { window.s.close() } /*創建socket連接*/ var socket = new WebSocket("ws://" + window.location.host + "/echo"); socket.onopen = function () { console.log(‘WebSocket open‘);//成功連接上Websocket window.s.send($(‘#message‘).val());//通過websocket發送數據 }; socket.onmessage = function (e) { console.log(‘[SDK]: ‘ + e.data);//打印出服務端返回過來的數據 $(‘#messagecontainer‘).append(‘<p>‘ + e.data + ‘</p>‘); }; // Call onopen directly if socket is already open if (socket.readyState == WebSocket.OPEN) socket.onopen(); window.s = socket; }); $(‘#send_message‘).click(function () { //如果未連接到websocket if (!window.s) { alert("websocket未連接."); } else { window.s.send($(‘#message‘).val());//通過websocket發送數據 } }); $(‘#close_websocket‘).click(function () { if (window.s) { window.s.close();//關閉websocket console.log(‘websocket已關閉‘); } }); }); //]]></script> </head> <body> <br> <input type="text" id="message" value="1461722512884260"/> <button type="button" id="connect_websocket">連接 websocket</button> <button type="button" id="send_message" style="display: none">發送 message</button> <button type="button" id="close_websocket" style="display: none">關閉 websocket</button> <h1>SDK 客戶端實時日誌</h1> <div id="messagecontainer" style="word-break: break-all"> </div> </body> </html>
- demo/views.py
# -*- coding: utf-8 -*- from django.shortcuts import render from dwebsocket.decorators import accept_websocket from django.http import HttpResponse import redis def index(request): return render(request, ‘index.html‘) @accept_websocket def echo(request): if not request.is_websocket(): # 判斷是不是websocket連接 try: # 如果是普通的http方法 message = request.GET[‘message‘] return HttpResponse(message) except: return render(request, ‘index.html‘) else: for message in request.websocket: print("gameId:", message) pool = redis.ConnectionPool(host=‘127.0.0.1‘, port=6379, db=1) r = redis.StrictRedis(connection_pool=pool) p = r.pubsub() p.subscribe(message) for item in p.listen(): # request.websocket.send(json.dumps(item)) # 發送消息到客戶端 if item[‘type‘] == ‘message‘: data = item[‘data‘] print("sdk_log:", data) request.websocket.send(data) if item[‘data‘] == ‘over‘: break;
- 運行django後，訪問頁面localhost:8000
python manage.py runserver
- 點擊頁面中的按鈕”連接 websocket“後，控制臺輸出”WebSocket open“
- 啟動手機中的遊戲“翻滾球球”，則頁面中實時輸出抓包記錄（所訂閱頻道根據輸入框中的gameId值）
python mitmdump抓包與redis訂閱消息