大致思路：

1.利用tornado提供的websocket功能與瀏覽器建立長連線，讀取實時日誌並輸出到瀏覽器
2.寫一個實時讀取日誌的指令碼，利用saltstack遠端執行，並把實時日誌發往redis中。
3.tornado讀取redis中的資訊，發往瀏覽器。

此過程用到了redis的釋出和訂閱功能。

先看一下tornado中是如何處理的：

import os
import sys
import tornado.websocket
import tornado.web
import tornado.ioloop
import redis
import salt.client

from
 
 tornado import gen
from tornado.escape import to_unicode

from logs.utility import get_last_lines
from logs import settings


class SubWebSocket(tornado.websocket.WebSocketHandler):
    """
    此handler處理遠端日誌檢視
    """
    def open(self, *args, **kwargs):
        print("opened")

    @gen.coroutine
 

    def on_message(self, message):
        # 主機名，要檢視的日誌路徑，執行指令碼的命令這些資訊從瀏覽器傳過來
        hostname, log_path, cmd = message.split("||")
        local = salt.client.LocalClient()
        r = redis.StrictRedis(host=settings.REDIS_HOST, port=settings.REDIS_PORT,
                              password=settings.REDIS_PASSWD, db=5
 
)
        # 訂閱頻道，伺服器和日誌路徑確定一個頻道
        key = settings.LOG_KEY.format(server=hostname.strip(), log_path=log_path.strip())
        channel = r.pubsub()
        channel.subscribe(key)
        # 非同步方式執行命令，遠端執行指令碼
        local.cmd_async(hostname, "cmd.run", [cmd])
        try:
            while True:
                data = channel.get_message()
                if not data:
                    # 如果讀取不到訊息，間隔一定時間，避免無謂的CPU消耗
                    yield gen.sleep(0.05)
                    continue
                if data["type"] == "message":
                    line = format_line(data["data"])
                    self.write_message(line)
        except tornado.websocket.WebSocketClosedError:
            self.close()

    def on_close(self):
        global FLAG
        FLAG = False
        print("closed")


def format_line(line):
    line = to_unicode(line)
    if "INFO" in line:
        color = "#46A3FF"
    elif "WARN" in line:
        color = "#FFFF37"
    elif "ERROR" in line:
        color = "red"
    elif "CRITICAL" in line:
        color = "red"
    else:
        color = "#FFFFFF"

    return "<span style='color:{}'>{}</span>".format(color, line)


class EchoWebSocket(tornado.websocket.WebSocketHandler):
    def open(self):
        print("WebSocket opened")

    @gen.coroutine
    def on_message(self, message):
        log = message
        print "log file: ", log

        try:
            with open(log, 'r') as f:
                for line in get_last_lines(f):
                    line1 = format_line(line)
                    self.write_message(line1)
                while True:
                    line = f.readline()
                    if not line:
                        yield gen.sleep(0.05)
                        continue
                    self.write_message(format_line(line.strip()))
        except tornado.websocket.WebSocketClosedError as e:
            print e
            self.close()

    # def check_origin(self, origin):
    #     print origin, self.request.headers.get("Host")
    #     # super(EchoWebSocket, self).check_origin()
    #     return True

    def on_close(self):
        print("WebSocket closed")


class Application(tornado.web.Application):
    def __init__(self):
        handlers = [
            (r'/log/', MainHandler),  # 提供瀏覽頁面，頁面中的JS與伺服器建立連線
            (r'/log/local', EchoWebSocket),  # 處理本地日誌實時檢視，比較簡單
            (r'/log/remote', SubWebSocket),  # 處理遠端日誌實時檢視，稍微複雜
        ]
        settings = {
            "debug": True,
            "template_path": os.path.join(os.path.dirname(__file__), "templates"),
            "static_path": os.path.join(os.path.dirname(__file__), "static"),
        }
        super(Application, self).__init__(handlers, **settings)


class MainHandler(tornado.web.RequestHandler):
    def get(self):
        # 要檢視的日誌路徑
        log = self.get_argument("log", None)
        # hostname實際上是saltstack中這臺機器對應的minion id
        hostname = self.get_argument("hostname", None)
        # 本地日誌還是遠端日誌
        type = self.get_argument("type", "local")
        # 執行讀取實時日誌的指令碼，引數比較多，後面會有
        cmd = self.get_argument("cmd", "")
        context = {
            "log": log,
            "hostname": hostname,
            "type": type,
            "cmd": cmd,
        }
        self.render("index.html", **context)

配置檔案中主要記錄了redis伺服器的地址等資訊

# encoding: utf-8

LOG_KEY = "logs:{server}:{log_path}"

LOG_NAME = "catalina.out"
TAIL_LINE_NUM = 20

REDIS_HOST = "127.0.0.1"
REDIS_PORT = "6379"
REDIS_PASSWD = None
REDIS_EXPIRE = 300

try:
    from local_settings import *
except ImportError:
    pass

index.html的內容如下：

<html>
<head>
<link href="{{ static_url('public/css/public.css') }}" rel="stylesheet" />
<link href="{{ static_url('kylin/css/style.css') }}" rel="stylesheet" />
</head>
<body style="background:#000000">
<div style="margin-left:10px;">
    <pre id="id-content">
    </pre>
    <div id="id-bottom"></div>
    <input type="hidden" id="id-log" value="{{ log }}" />
    <input type="hidden" id="id-type" value="{{ type }}" />
    <input type="hidden" id="id-hostname" value="{{ hostname }}" />
    <input type="hidden" id="id-cmd" value="{{ cmd }}" />
    <div class="btns btns_big">
        <button type="button" class="query_btn cancle" id="id-stop">Stop</button>
        <button type="button" class="query_btn commit" id="id-start">Start</button>
    </div>
</div>
<script type="text/javascript" src="{{ static_url('js/jquery-1.11.3.min.js') }}"></script>
<script type="text/javascript">
    var log_name = $("#id-log").val();
    var type = $("#id-type").val();
    var hostname = $("#id-hostname").val();
    var cmd = $("#id-cmd").val();
    // 初始化websocket物件
    var ws = new WebSocket("ws://{{ request.host }}/log/" + type);
    ws.onopen = function(){
        if (type === "local"){
            ws.send(log_name);
        } else {
            // 建立連線後把相關資訊發往伺服器，對應上面的SubWebSocket
            ws.send(hostname + "||" + log_name + "||" + cmd);
        }
    };
    var get_message = function(evt){
        $("#id-content").append(evt.data + "\n");
        document.getElementById("id-bottom").scrollIntoView()
    };
    ws.onmessage = get_message;
    // 兩個按鈕控制日誌的輸出，如果看到需要的日誌資訊，可以暫停日誌的輸出，
    // 之後可以繼續啟動日誌的輸出
    $("#id-stop").click(function(){
        ws.onmessage = function(){};
    })
    $("#id-start").click(function(){
        ws.onmessage = get_message;
    })
</script>
</body>
</html>

這個tornado僅僅是提供了實時日誌的服務，實際專案使用的是django，django中要做的其實很簡單，提供log_name,hostname,type,cmd等四個引數。下面看一個例項：

class LogView(KylinView):
    # 實時讀取日誌的指令碼，事先使用saltstack批量傳到各臺伺服器上
    client_path = "/tmp/logtail.py"

    def get(self, request):
        minion_id = request.GET.get("minion_id")
        context = {
            "minion_id": minion_id,
            "tail_log_url": settings.TAIL_LOG_URL,
        }
        return render(request, "cmdb/log_view.html", context)

    def post(self, request):
        minion_id = request.POST.get("minion_id")
        log_path = request.POST.get("log_path")
        if not log_path:
            return JsonResponse({"success": False, "message": "請填寫日誌路徑"})
        try:
            # 制定一開始讀取的行數
            line_count = request.POST.get("line_count")
        except (TypeError, ValueError):
            return JsonResponse({"success": False, "message": "請輸入正確的行數"})
        local = salt.client.LocalClient()
        # 確保saltstack能連通並且日誌檔案存在
        ret = local.cmd(minion_id, "file.file_exists", [log_path])
        if minion_id not in ret:
            return JsonResponse({"success": False, "message": "伺服器無法連通"})
        if not ret[minion_id]:
            return JsonResponse({"success": False, "message": "日誌檔案不存在"})
        # 組成命令的各個引數，redis資訊需要和tornado配置檔案中的redis資訊一致
        cmd = "{} {} {} {} {} {} {} {}".format(
            settings.PYTHON_BIN, self.client_path, minion_id, log_path, line_count, settings.REDIS_HOST,
            settings.REDIS_PORT, settings.REDIS_PASSWD)
        # settings.TAIL_LOG_URL是tornado中MainHandler對應的url,把其它幾個
        # 引數組合成最終的URL，直接訪問這個URL就可以在瀏覽器中實時讀取日誌了。
        url = "{}?type=remote&log={}&hostname={}&cmd={}".format(
            settings.TAIL_LOG_URL, log_path, minion_id, cmd)
        # 這一步的操作確保同一個日誌檔案只有一個指令碼在讀取，避免日誌資訊重複，這一步
        # 也很重要，必不可少
        local.cmd(minion_id, "cmd.run",
                  ["kill `ps aux|grep logtail.py|grep %s|grep -v grep|awk '{print $2}'`" % (log_path,)])
        return JsonResponse({"success": True, "url": url})

下面來看看logtail.py的實現：

# encoding: utf-8
from __future__ import unicode_literals, division

import math
import time
import sys
import socket
import signal
import redis

FLAG = True


def get_last_lines(f, num=10):
    """讀取檔案的最後幾行
    """
    size = 1000
    try:
        f.seek(-size, 2)
    except IOError:  # 檔案內容不足size
        f.seek(0)
        return f.readlines()[-num:]

    data = f.read()
    lines = data.splitlines()
    n = len(lines)
    while n < num:
        size *= int(math.ceil(num / n))
        try:
            f.seek(-size, 2)
        except IOError:
            f.seek(0)
            return f.readlines()[-num:]
        data = f.read()
        lines = data.splitlines()
        n = len(lines)

    return lines[-num:]


def process_line(r, channel, line):
    r.publish(channel, line.strip())


def sig_handler(signum, frame):
    global FLAG
    FLAG = False


# 收到退出訊號後，以比較優雅的方式終止指令碼
signal.signal(signal.SIGTERM, sig_handler)
# 為了避免日誌輸出過多，瀏覽器承受不住，設定5分鐘後腳本自動停止
signal.signal(signal.SIGALRM, sig_handler)
signal.alarm(300)


def get_hostname():
    return socket.gethostname()


def force_str(s):
    if isinstance(s, unicode):
        s = s.encode("utf-8")
    return s


def tail():
    password = sys.argv[6]
    if password == "None":
        password = None
    r = redis.StrictRedis(host=sys.argv[4], port=sys.argv[5], password=password, db=5)
    log_path = sys.argv[2]
    line_count = int(sys.argv[3])
    # 往redis頻道傳送實時日誌
    channel = "logs:{hostname}:{log_path}".format(hostname=sys.argv[1], log_path=log_path)

    with open(log_path, 'r') as f:
        last_lines = get_last_lines(f, line_count)
        for line in last_lines:
            process_line(r, channel, force_str(line))
        try:
            while FLAG:  # 通過訊號控制這個變數，實現優雅退出迴圈
                line = f.readline()
                if not line:
                    time.sleep(0.05)
                    continue
                process_line(r, channel, line)
        except KeyboardInterrupt:
            pass
    print("Exiting...")

if __name__ == "__main__":
    if len(sys.argv) < 6:
        print "Usage: %s minion_id log_path host port redis_pass"
        exit(1)

    tail()