C# Http多執行緒下載、斷點續傳
阿新 • • 發佈:2019-02-03
using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; using System.Net.Sockets; using System.Text; using System.Threading; using System.Threading.Tasks; namespace FengBan { /// /// 多執行緒下載、斷點下載的管理類 /// class DFGManager : IDisposable { private string dfgFilename; private int size; private FileStream writer; private SortedSet rset = new SortedSet(new RangeCompare()); /// /// 多執行緒下載、斷點下載的管理類 /// /// internal DFGManager(string dfgFilename) { this.dfgFilename = dfgFilename; if (File.Exists(dfgFilename)) { ReadDFGFile(); File.Delete(dfgFilename); } writer = new FileStream(dfgFilename, FileMode.OpenOrCreate); WriteDFGFile(); } /// /// 獲取或設定要下載的檔案大小 /// public int Size { get { return size; } set { size = value; byte[] head = new byte[16]; byte[] bsize = BitConverter.GetBytes(size); int i; for (i = 0; i < bsize.Count(); i++) { head[i] = bsize[i]; } writer.Seek(0, SeekOrigin.Begin); writer.Write(head, 0, 16); writer.Flush(); } } /// /// 釋放檔案段,該段已經下載完成 /// /// public void Free(Range range) { lock (rset) { var find = from r in rset where r.left == range.left && r.status == 1 select r; if (find.Count() == 1) { var ele = find.First(); ele.status = 2; range.status = 2; if (ele.right < range.right) { ele.right = range.right; } else { range.right = ele.right; } MergeRange(); writer.Write(BitConverter.GetBytes(range.left), 0, 4); writer.Write(BitConverter.GetBytes(range.right), 0, 4); writer.Flush(); } else if (find.Count() == 0) { throw new Exception("找不到釋放的塊"); } else { throw new Exception("多個符合條件的塊"); } } } /// /// 分配檔案段 /// /// 期望分配的大小 /// 實際分配的大小 public Range Alloc(int expectSize = 102400) { lock (rset) { Range range = new Range(); if (rset.Count == 0) { range.left = 0; range.right = Math.Min(expectSize, this.size); range.status = 1; rset.Add(range); } else if (rset.Count == 1) { if (rset.First().left > 0) { range.left = 0; range.right = rset.First().left; range.status = 1; rset.Add(range); } else if (rset.Last().right < size) { range.left = rset.Last().right; range.right = Math.Min(size, range.left + expectSize); range.status = 1; rset.Add(range); } else { range = null; } } else { if (rset.First().left > 0) { range.left = 0; range.right = rset.First().left; range.status = 1; rset.Add(range); } else if (rset.Last().right < size) { range.left = rset.Last().right; range.right = Math.Min(size, range.left + expectSize); range.status = 1; rset.Add(range); } else { for (int i = 1; i < rset.Count; i++) { Range l = rset.ElementAt(i - 1); Range r = rset.ElementAt(i); if (l.right < r.left) { range.left = l.right; range.right = r.left; range.status = 1; rset.Add(range); return range; } } range = null; } } return range; } } private void ReadDFGFile() { StreamReader reader = new StreamReader(dfgFilename); byte[] head = new byte[16];/*head佔4個int*/ reader.BaseStream.Read(head, 0, 16); size = BitConverter.ToInt32(head, 0); byte[] buff = new byte[8]; int rd = 0; do { rd = reader.BaseStream.Read(buff, 0, 8); if (rd == 0) { break; } Range range = new Range(); range.left = BitConverter.ToInt32(buff, 0); range.right = BitConverter.ToInt32(buff, 4); range.status = 2; rset.Add(range); } while (true); reader.Close(); MergeRange(); } private void WriteDFGFile() { byte[] head = new byte[16]; byte[] bsize = BitConverter.GetBytes(size); int i; for (i = 0; i < bsize.Count(); i++) { head[i] = bsize[i]; } writer.Write(head, 0, 16); for (i = 0; i < rset.Count; i++) { Range r = rset.ElementAt(i); if (r.status == 2) { writer.Write(BitConverter.GetBytes(r.left), 0, 4); writer.Write(BitConverter.GetBytes(r.right), 0, 4); } } writer.Flush(); } private void MergeRange() { int i; if (rset.Count >= 2) { Range last = rset.First(); for (i = 1; i < rset.Count && rset.Count >= 2; i++) { Range cur = rset.ElementAt(i); if (cur.left <= last.right && cur.status == 2 && last.status == 2) { last.right = cur.right; rset.Remove(cur); i--; } else { last = cur; } } } } /// /// 釋放資源 /// public void Dispose() { if (writer != null) { writer.Close(); writer = null; } } ~DFGManager() { Dispose(); } } class Range { public int left; public int right; public int status; } class RangeCompare : IComparer { public int Compare(Range x, Range y) { return (x.left - y.left); } } } using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; using System.Net.Sockets; using System.Text; using System.Threading; using System.Threading.Tasks; using System.Text.RegularExpressions; namespace FengBan { class Program { static void Main() { MultiThreadDownload downloader = new MultiThreadDownload(); downloader.Download("http://dlsw.baidu.com/sw-search-sp/soft/3a/12350/QQ_V7.0.14275.0_setup.1426647314.exe", "qq.exe"); } } /// /// 多執行緒下載類 /// class MultiThreadDownload { /*dfg檔案記錄下載好的檔案段*/ private const string suffix = ".dfg"; private string site;/*檔案下載地址*/ private string outname;/*下載的檔名稱*/ private DFGManager dfgManager;/*檔案分段下載管理物件*/ private FileStream writer;/*多個下載子執行緒共用一個輸出流*/ /*通過currentDownload和lastDownload來實現限速*/ private int currentDownload; private int lastDownload; private int shouldSleep;/*限速時應該休眠的時長,單位毫秒*/ private System.Timers.Timer timer;/*限速計時器*/ /*上網代理*/ public string ProxyAddressAndPort { get; set; } public string ProxyUserName { get; set; } public string ProxyPassword { get; set; } public MultiThreadDownload(int ThreadCount = 5) { this.ThreadCount = ThreadCount; timer = new System.Timers.Timer(1000); timer.Elapsed += CheckDownloadSpeed; timer.AutoReset = true; } /// /// 下載檔案的函式,僅支援http協議下載 /// /// 檔案下載地址 /// 輸出檔名 public void Download(string site, string outname) { string realname = outname; outname += ".down"; this.site = site; this.outname = outname; try { /*根據需要設定代理*/ if (ProxyEnable) { ICredentials cred = new NetworkCredential(ProxyUserName, ProxyPassword); WebProxy p = new WebProxy(ProxyAddressAndPort, true, null, cred); WebRequest.DefaultWebProxy = p; } /*獲取要下載的檔案的資訊,主要是獲取檔案大小*/ HttpWebRequest request = (HttpWebRequest)WebRequest.Create(site); request.UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36"; WebResponse response = request.GetResponse(); Stream responseStream = response.GetResponseStream(); long contentLength = response.ContentLength; Console.WriteLine("要下載的檔案大小:{0}", contentLength); responseStream.Close(); response.Close(); /*建立管理物件,負責給每一個執行緒分配下載的檔案段*/ dfgManager = new DFGManager(outname + suffix); dfgManager.Size = Convert.ToInt32(contentLength); currentDownload = 0; lastDownload = 0; writer = new FileStream(outname, FileMode.OpenOrCreate); ManualResetEvent[] _ManualEvents = new ManualResetEvent[ThreadCount]; //開啟執行緒池下載 for (int i = 0; i < ThreadCount; i++) { _ManualEvents[i] = new ManualResetEvent(false); ThreadPool.QueueUserWorkItem(new WaitCallback(SDown), _ManualEvents[i]); } WaitHandle.WaitAll(_ManualEvents);/*等待子執行緒下載結束*/ Console.WriteLine("全部下載子執行緒結束"); //下載完成,釋放資源 writer.Close(); dfgManager.Dispose(); /*收尾*/ if (File.Exists(realname)) { File.Delete(realname); } File.Move(outname, realname); File.Delete(outname + suffix); } finally { } } private void CheckDownloadSpeed(object sender, System.Timers.ElapsedEventArgs e) { int curSpeed = (currentDownload - lastDownload); Console.WriteLine("當前速度 {0:f} kb/s", ((double)curSpeed) / 1000); lastDownload = currentDownload; //如果當前速度超過設定的速度,則應該執行限速動作 //未實現 } /// /// 設定代理 /// /// 代理伺服器地址和埠 /// 代理伺服器使用者名稱 /// 代理伺服器密碼 public void SetProxy(string ProxyAddressAndPort, string Username, string Password) { this.ProxyAddressAndPort = ProxyAddressAndPort; this.ProxyUserName = Username; this.ProxyPassword = Password; this.ProxyEnable = true; } /*是否使用代理*/ public bool ProxyEnable { get; set; } /// /// 設定或獲取多執行緒下載的數量 /// public int ThreadCount { set; get; } /*多執行緒下載的執行緒函式*/ private void SDown(object param) { Range range; byte[] buff = null; /*從dfgManager中獲取要下載的檔案段*/ while ((range = dfgManager.Alloc()) != null) { HttpWebRequest request = (HttpWebRequest)WebRequest.Create(site); request.AddRange(range.left, range.right); request.UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36"; WebResponse response = request.GetResponse(); int contentLength = (int)response.ContentLength; //分配下載的快取空間 if (buff == null || buff.Count() < contentLength) { buff = new byte[contentLength]; } //使用while迴圈讀取全部位元組到buff中 int count = 0; while (count < contentLength) { count += response.GetResponseStream().Read(buff, count, (int)contentLength - count); } //真正從伺服器下載下來的檔案段 Range realRange = new Range(); realRange.left = range.left; realRange.right = Math.Min(range.left + count, range.right); dfgManager.Free(realRange); response.Close(); //將buff寫入檔案 writer.Seek(realRange.left, SeekOrigin.Begin); writer.Write(buff, 0, realRange.right - realRange.left); writer.Flush(); int realDown = realRange.right - realRange.left; currentDownload += realDown; Console.WriteLine("本次分段下載大小{0}", realDown); if (LimitSpeed > 0 && shouldSleep > 10) { Console.WriteLine("休眠{0}", shouldSleep); Thread.Sleep(shouldSleep); } } Console.WriteLine("執行緒執行結束"); /*通知主執行緒執行完成*/ ManualResetEvent e = (ManualResetEvent)param; e.Set(); } private int xiansu; /// /// 表示是否限速,0表示不限速,單位 b/s /// public int LimitSpeed { get { return xiansu; } set { xiansu = value; if (value > 0) { timer.Enabled = true; } } } } }