Java查詢一次性查詢幾十萬，幾百萬資料解決辦法。

很早的時候寫工具用的一個辦法，當時是用來把百萬資料打包成rar檔案。

所以用了個笨辦法。 希望高手指導一下，有什麼好方法沒有啊。

以下是查詢資料庫。按批次查詢

1. publicstaticvoid getMonthDataList() {
2. ResultSet rs = null;
3. Statement stat = null;
4. Connection conn = null;
5. List<DataBean> list = new ArrayList<DataBean>();
6. try {
7. conn = createConnection();
8. if(conn!=null){
9. SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
10. SimpleDateFormat timesdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
11. String nowDate = sdf.format(new Date());
12. Config.lasttimetext = timesdf.format(
    new Date());
13. String lastDate = sdf.format(CreateData.addDaysForDate(new Date(), 30));
14. stat = conn.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE,ResultSet.CONCUR_UPDATABLE);
15. int lastrow = 0;
16. int datanum = 0;
17. String countsql = "SELECT count(a.id) FROM trip_special_flight a"
    +
18. " where a.dpt_date >= to_date('"+nowDate+"','yyyy-mm-dd') " +
19. "and a.dpt_date <= to_date('"+lastDate+"','yyyy-mm-dd') and rownum>"+lastrow+" order by a.get_time desc";
20. rs = stat.executeQuery(countsql);
21. while (rs.next()) {
22. datanum = rs.getInt(1);
23. }
24. int onerun = 10000;
25. int runnum = datanum%onerun==0?(datanum/onerun):(datanum/onerun)+1;
26. for(int r =0;r<runnum;r++){
27. System.out.println("getMonthDataList--"+datanum+" 開始查詢第"+(r+1)+"批資料");
28. String sql = "SELECT * FROM (SELECT rownum rn, a.dpt_code, a.arr_code,a.dpt_date,a.airways,a.flight," +
29. "a.cabin,a.price FROM trip_special_flight a" +
30. " where a.dpt_date >= to_date('"+nowDate+"','yyyy-mm-dd') " +
31. "and a.dpt_date <= to_date('"+lastDate+"','yyyy-mm-dd') order by rownum asc) WHERE rn > "+lastrow;
32. stat.setMaxRows(onerun);
33. stat.setFetchSize(1000);
34. rs = stat.executeQuery(sql);
35. String text = "";
36. int i = 1;
37. while (rs.next()) {
38. text += rs.getString(2)+"|"+rs.getString(3)+"|"+rs.getDate(4)+"|"+rs.getString(5)+"|"+rs.getString(6)+"|"+rs.getString(7)+"|"+rs.getString(8)+"||";
39. if(i%1000==0){
40. FileUtil.appendToFile(Config.tempdatafile, text);
41. text = "";
42. }
43. i++;
44. }
45. if(text.length()>10){
46. FileUtil.appendToFile(Config.tempdatafile, text);
47. }
48. lastrow+=onerun;
49. }
50. }
51. } catch (Exception e) {
52. e.printStackTrace();
53. } finally {
54. closeAll(rs, stat, conn);
55. }
56. }

-----java一次性查詢幾十萬，幾百萬資料解決辦法

存入臨時檔案之後，再用讀取大量資料檔案方法。

設定快取大小BUFFER_SIZE ，Config.tempdatafile是檔案地址。

來源部落格 http://yijianfengvip.blog.163.com/blog/static/175273432201191354043148/

1. package com.yjf.util;
2. import java.io.File;
3. import java.io.RandomAccessFile;
4. import java.nio.MappedByteBuffer;
5. import java.nio.channels.FileChannel;
6. publicclass Test {
7. publicstaticvoid main(String[] args) throws Exception {
8. finalint BUFFER_SIZE = 0x300000; // 緩衝區為3M
9. File f = new File(Config.tempdatafile);
10. // 來源部落格http://yijianfengvip.blog.163.com/blog/static/175273432201191354043148/
11. int len = 0;
12. Long start = System.currentTimeMillis();
13. for (int z = 8; z >0; z--) {
14. MappedByteBuffer inputBuffer = new RandomAccessFile(f, "r")
15. .getChannel().map(FileChannel.MapMode.READ_ONLY,
16. f.length() * (z-1) / 8, f.length() * 1 / 8);
17. byte[] dst = newbyte[BUFFER_SIZE];// 每次讀出3M的內容
18. for (int offset = 0; offset < inputBuffer.capacity(); offset += BUFFER_SIZE) {
19. if (inputBuffer.capacity() - offset >= BUFFER_SIZE) {
20. for (int i = 0; i < BUFFER_SIZE; i++)
21. dst[i] = inputBuffer.get(offset + i);
22. } else {
23. for (int i = 0; i < inputBuffer.capacity() - offset; i++)
24. dst[i] = inputBuffer.get(offset + i);
25. }
26. int length = (inputBuffer.capacity() % BUFFER_SIZE == 0) ? BUFFER_SIZE
27. : inputBuffer.capacity() % BUFFER_SIZE;
28. len += new String(dst, 0, length).length();
29. System.out.println(new String(dst, 0, length).length()+"-"+(z-1)+"-"+(8-z+1));
30. }
31. }
32. System.out.println(len);
33. long end = System.currentTimeMillis();
34. System.out.println("讀取檔案檔案花費：" + (end - start) + "毫秒");
35. }
36. }

讀取大量資料檔案方法。