'''

遍歷資料夾

如果檔名是.cpp .h

    如果原來的編碼不是utf-8,將檔案編碼格式改成utf-8

'''

import os,sys

import chardet

def convert( filename, out_enc="UTF8" ):

    try:

        fp = open(filename,'rb+')

        content = fp.read()

        coding = chardet.detect(content)['encoding'] #獲取encoding的值[編碼格式]

        if coding != 'utf-8':

            new_content = content.decode(coding,"ignore").encode(out_enc)

            fp.seek(0)

            fp.write(new_content)

            fp.close()

    except IOError:

        print( " error")

def explore(dir):

    #遍歷目錄

    for root, dirs, files in os.walk(dir):

        for file in files:

            if '.cpp' in file or '.h' in file:

                path = os.path.join(root, file)

                convert(path)

fiePath = r'E:\Code'

def main():

    explore(fiePath)

if __name__ == "__main__":

    main()

import zipfile

import tarfile

import gzip

import os

from time import ctime

from multiprocessing import Pool

from multiprocessing import cpu_count

dayZipsPath = r'.'             

quarterZipsPath = r'./tmp'     

zipPassWord = b'password'        

mrFilePath = r'./data'          

def unzipDayFile():

    for file_name in os.listdir(dayZipsPath):

        if os.path.splitext(file_name)[1] == '.zip':

            print( file_name)

            file_zip = zipfile.ZipFile(file_name, 'r')

            file_zip.extractall(path = quarterZipsPath, pwd = zipPassWord)

            file_zip.close()

            #os.remove(file_name)

def untarDayFile():

    for file_name in os.listdir(dayZipsPath):

        if '.tar.gz' in file_name:

            print( file_name)

            file_tar = tarfile.open(file_name)

            file_tar.extractall(path = quarterZipsPath)

            file_tar.close()

            #os.remove(file_name)

def unzip(zipsList):

    for file_name in zipsList:

        if os.path.splitext(file_name)[1] == '.zip':

            zipFileName = quarterZipsPath +'/'+ file_name

            file_zip = zipfile.ZipFile(zipFileName, 'r')

            file_zip.extractall(path = mrFilePath, pwd = zipPassWord)

            file_zip.close()

            os.remove(zipFileName)

if __name__ == '__main__':

    print('Begin:%s' % ctime())

    #獲取CPU核個數

    cpuNum = cpu_count()

    print(cpuNum)

    unzipDayFile()

    untarDayFile()

    #多程序解壓，大大縮短處理時間

    quarterZipsList = list(os.listdir(quarterZipsPath))

    zipFileNum = len(quarterZipsList)

    print("total zip files num:%d" % (zipFileNum))

    print("begin unzip:%s" % ctime())

    p = Pool()

    for i in range(cpuNum):

        beginPos = int(i*zipFileNum/cpuNum)

        endPos = min(int((i+1)*zipFileNum/cpuNum),zipFileNum)

        print("proc %d - %d" % (beginPos, endPos))

        p.apply_async(unzip,args=(quarterZipsList[beginPos:endPos],))

    print("waiting for unzip quarter mr data ...")

    p.close()

    p.join()

    print("end unzip:%s" % ctime())  

    print( "End:%s" % ctime())