1. 程式人生 > >pandas 讀取大檔案

pandas 讀取大檔案

from matplotlib import pyplot  
import scipy as sp  
import numpy as np  
from matplotlib import pylab  
from pandas import Series, DataFrame
import pandas as pd
filename = 'data.txt'
data = pd.read_csv(filename, sep='::', names=list('uirt'), index_col=False, usecols=range(len('uirt')),engine = 'python'
, iterator=True) loop = True chunkSize = 100000 chunks = [] while loop: try: chunk = data.get_chunk(chunkSize) chunks.append(chunk) except StopIteration: loop = False print "Iteration is stopped." data = pd.concat(chunks, ignore_index=True)