問題描述

在微軟雲環境中，使用python SDK連線儲存賬號（Storage Account）需要計算Blob大小？雖然Azure提供了一個專用工具Azure Storage Explorer可以統計出Blob的大小：

但是它也是隻能一個Blob Container一個的統計，如果Container數量巨大，這將是一個繁瑣的工作。而作為開發者，應該讓程式碼來幫助完成。下文使用最快上手的Python程式碼來計算Blob中容量的大小。

完整程式碼

import os, uuid, datetime, threading

import logging

from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient, __version__

def calculateBlob(connect_string, count):

    try:

        blob_service_client = BlobServiceClient.from_connection_string(connect_string)

    except Exception as e:

        messages = str(count) + "Connect_String Error, Messages:" + e.args.__str__()

        print(messages)

        logging.info(messages)

    else:

        all_containers = blob_service_client.list_containers()

        for c in all_containers:

            count_name = c.name

            print(count_name)

            if count_name not in blobSize_Total:

                blobSize_Total[count_name] = 0

            if count_name not in blobSize_Daily:

                blobSize_Daily[count_name] = 0

            container_client = blob_service_client.get_container_client(count_name)

            generator = container_client.list_blobs()

            total_size_container = 0

            daily_size_container = 0

            for blob in generator:

                total_size_container += blob.size

                blob_create_time = blob.creation_time.strftime("%Y%m%d")

                if blob_create_time != now_date:

                    continue

                else:

                    # Calculate BlobSize in this month

                    daily_size_container += blob.size

                    # blobSize_Daily[count_name] += blob.size  # /(1024*1024)  # content_length - bytes

            blobSize_Total[count_name] += total_size_container / (1024 * 1024)

            blobSize_Daily[count_name] += daily_size_container / (1024 * 1024)

    return None

if __name__ == '__main__':

    # connect string

    Connection_String_List ="DefaultEndpointsProtocol=https;AccountName=<storagename>;AccountKey=<key>;EndpointSuffix=core.chinacloudapi.cn"

    # for i in Connection_String:

    start = datetime.datetime.now()

    print(start)

    # 定義全域性變數 - blobSize_Daily & blobSize_Total

    blobSize_Daily = {}

    blobSize_Total = {}

    now_date = datetime.datetime.now().strftime("%Y%m%d")

    print("開始計算")

    calculateBlob(Connection_String_List, 1)

    print("計算完成")

    print("統計當前新增大小")

    print(blobSize_Daily)

    print("統計Blob總大小")

    print(blobSize_Total)

    end = datetime.datetime.now()

    print(end)

如執行是沒有Azure blob模組，可以使用 pip install azure-storage-blob 安裝。以上程式碼執行結果如下：

如果有多個Storage Account，可以考慮加入多執行緒的方式來執行，在程式碼中增加一個myThread類，然後在 __main__ 中把 calculateBlob(Connection_String_List, 1) 執行替換為 many_thread(Connection_String_List) 即可。

class myThread(threading.Thread):

    def __init__(self, threadID, name, connection_string):

        threading.Thread.__init__(self)

        self.threadID = threadID

        self.name = name

        self.connection_string = connection_string

    def run(self):

        print("開始執行緒：" + self.name)

        calculateBlob(self.connection_string, self.threadID)

        print("退出執行緒：" + self.name)

def many_thread(Connection_String_List):

    threads = []

    for i in range(len(Connection_String_List)):  # 迴圈建立32個執行緒

        t = myThread(i, "Thread-" + str(i), Connection_String_List[i])

        threads.append(t)

    for t in threads:  # 迴圈啟動32個執行緒 - 對應64個storage account

        t.start()

    for t in threads:

        t.join()

遇見問題

在多執行緒執行時，可能會遇見問題：("Connection broken: ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None)", ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None))，出現此問題大都是由於客戶端使用了已經斷開的連線導致所導致的。所以一定要仔細除錯多執行緒關閉程式碼。是否是把還需要執行的執行緒給關閉了。導致了以上的錯誤訊息。

附錄一：多執行緒計算Blob的完整程式碼

import os, uuid, datetime, threading

import logging

from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient, __version__

def calculateBlob(connect_string, count):

    try:

        blob_service_client = BlobServiceClient.from_connection_string(connect_string)

    except Exception as e:

        messages = str(count) + "Connect_String Error, Messages:" + e.args.__str__()

        print(messages)

        logging.info(messages)

    else:

        all_containers = blob_service_client.list_containers()

        for c in all_containers:

            count_name = c.name

            print(count_name)

            if count_name not in blobSize_Total:

                blobSize_Total[count_name] = 0

            if count_name not in blobSize_Daily:

                blobSize_Daily[count_name] = 0

            container_client = blob_service_client.get_container_client(count_name)

            generator = container_client.list_blobs()

            total_size_container = 0

            daily_size_container = 0

            for blob in generator:

                total_size_container += blob.size

                blob_create_time = blob.creation_time.strftime("%Y%m%d")

                if blob_create_time != now_date:

                    continue

                else:

                    # Calculate BlobSize in this month

                    daily_size_container += blob.size

                    # blobSize_Daily[count_name] += blob.size  # /(1024*1024)  # content_length - bytes

            blobSize_Total[count_name] += total_size_container / (1024 * 1024)

            blobSize_Daily[count_name] += daily_size_container / (1024 * 1024)

    return None

class myThread(threading.Thread):

    def __init__(self, threadID, name, connection_string):

        threading.Thread.__init__(self)

        self.threadID = threadID

        self.name = name

        self.connection_string = connection_string

    def run(self):

        print("開始執行緒：" + self.name)

        calculateBlob(self.connection_string, self.threadID)

        print("退出執行緒：" + self.name)

def many_thread(Connection_String_List):

    threads = []

    for i in range(len(Connection_String_List)):  # 迴圈建立32個執行緒

        t = myThread(i, "Thread-" + str(i), Connection_String_List[i])

        threads.append(t)

    for t in threads:  # 迴圈啟動32個執行緒 - 對應64個storage account

        t.start()

    for t in threads:

        t.join()

if __name__ == '__main__':

    # connect string

    Connection_String_List =  ['DefaultEndpointsProtocol=https;AccountName=<your storage account 1>;AccountKey=<Key 1>;EndpointSuffix=core.chinacloudapi.cn', 'DefaultEndpointsProtocol=https;AccountName=<your storage account 2>;AccountKey=<Key 2>;EndpointSuffix=core.chinacloudapi.cn']

    # for i in Connection_String:

    start = datetime.datetime.now()

    print(start)

    # 定義全域性變數 - blobSize_Daily & blobSize_Total

    blobSize_Daily = {}

    blobSize_Total = {}

    now_date = datetime.datetime.now().strftime("%Y%m%d")

    many_thread(Connection_String_List)

    print("Main Thread End")

    print(blobSize_Daily)

    print(blobSize_Total)

    end = datetime.datetime.now()

    print(end)