android查詢幾十萬條資料的調研(一)
此次調研分兩步走吧, 先從測試的角度看(一), 再從原始碼的角度看(二)(待續).
下面的是測試程式碼:
SQLiteUtils.java
package com.johnsoft.app.temp;
import android.content.Context;
import android.database.Cursor;
import android.database.sqlite.SQLiteDatabase;
import android.database.sqlite.SQLiteOpenHelper;
/**
* @author John Kenrinus Lee
* @version 2016-05-16
*/
public class SQLiteUtils {
private static SQLiteOpenHelper sHelper;
public static void loopInsert(Context context) {
final SQLiteOpenHelper helper = getSQLiteHelper(context);
final SQLiteDatabase db = helper.getWritableDatabase();
db.beginTransaction();
String[] args = new String[2];
for (int i = 0; i < 200000; ++i) {
args[0] = "person" + i;
args[1] = String.valueOf(i);
db.execSQL("insert into person (name,age) values (?,?);", args);
}
db.setTransactionSuccessful();
db.endTransaction();
System.err.println("Insert OOOOOOOOOOOOOOKKKKKKKKKKKKKK" );
}
public static void fullQuery(Context context) {
final long start = System.currentTimeMillis();
final SQLiteOpenHelper helper = getSQLiteHelper(context);
final SQLiteDatabase db = helper.getReadableDatabase();
final Cursor cursor = db.rawQuery("select * from person", null);
long count = 0;
int size = 0;
for (cursor.moveToFirst(); !cursor.isAfterLast(); cursor.moveToNext()) {
count += cursor.getString(1).length();
++size;
}
cursor.close();
System.err.println("Size: " + size);
System.err.println("Query count ccccccccccccccc " + count);
System.err.println("total: " + (System.currentTimeMillis() - start));
}
public static void segQuery(Context context) {
final long start = System.currentTimeMillis();
final SQLiteOpenHelper helper = getSQLiteHelper(context);
final SQLiteDatabase db = helper.getReadableDatabase();
Cursor cursor = db.rawQuery("select count(*) from person", null);
final int size;
cursor.moveToFirst();
if (!cursor.isAfterLast()) {
size = cursor.getInt(0);
} else {
System.err.println("Query failed !!!!!!!!!!!!!!!!!!!!!!!");
return;
}
cursor.close();
int offset = 0;
final int length = size / 100;
long count = 0;
boolean isNotLast = true;
int arg0, arg1;
while(isNotLast) {
if (offset + length > size) {
arg0 = size - offset;
if (arg0 <= 0) {
break;
} else {
isNotLast = false;
}
} else {
arg0 = length;
}
arg1 = offset;
offset += length;
System.err.println("args0: " + arg0 + ", args1: " + arg1);
cursor = db.rawQuery("select * from person limit " + arg0 + " offset " + arg1 + ";", null);
for (cursor.moveToFirst(); !cursor.isAfterLast(); cursor.moveToNext()) {
count += cursor.getString(1).length();
}
cursor.close();
}
System.err.println("Query count ccccccccccccccc " + count);
System.err.println("total: " + (System.currentTimeMillis() - start));
}
private static SQLiteOpenHelper getSQLiteHelper(Context context) {
if (sHelper == null) {
synchronized(SQLiteUtils.class) {
if (sHelper == null) {
sHelper = new SQLiteOpenHelper(context, "qu.db", null, 1) {
@Override
public void onCreate(SQLiteDatabase db) {
db.execSQL("create table if not exists person ("
+ "id integer primary key,"
+ "name text not null,"
+ "age text);");
}
@Override
public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
}
};
}
}
}
return sHelper;
}
}
MainActivity.java
package com.johnsoft.app.temp;
import android.os.Bundle;
import android.support.v7.app.AppCompatActivity;
import android.view.View;
public class MainActivity extends AppCompatActivity {
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
final View insert = findViewById(R.id.insert);
final View full = findViewById(R.id.full);
final View seg = findViewById(R.id.seg);
if (insert != null) {
insert.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
new Thread("insert") {
@Override
public void run() {
SQLiteUtils.loopInsert(getApplicationContext());
}
}.start();
}
});
}
if (full != null) {
full.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
new Thread("full") {
@Override
public void run() {
SQLiteUtils.fullQuery(getApplicationContext());
}
}.start();
}
});
}
if (seg != null) {
seg.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
new Thread("seg") {
@Override
public void run() {
SQLiteUtils.segQuery(getApplicationContext());
}
}.start();
}
});
}
}
}
activity_main.xml
<?xml version="1.0" encoding="utf-8"?>
<RelativeLayout
xmlns:android="http://schemas.android.com/apk/res/android"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:paddingLeft="@dimen/activity_horizontal_margin"
android:paddingRight="@dimen/activity_horizontal_margin"
android:paddingTop="@dimen/activity_vertical_margin"
android:paddingBottom="@dimen/activity_vertical_margin">
<Button android:layout_width="wrap_content" android:layout_height="wrap_content"
android:id="@+id/insert" android:text="Insert"
android:layout_alignParentTop="true" android:layout_marginTop="10dp"
/>
<Button android:layout_width="wrap_content" android:layout_height="wrap_content"
android:id="@+id/full" android:text="Full Query"
android:layout_below="@id/insert" android:layout_marginTop="20dp"
/>
<Button android:layout_width="wrap_content" android:layout_height="wrap_content"
android:id="@+id/seg" android:text="Seg Query"
android:layout_below="@id/full" android:layout_marginTop="20dp"
/>
</RelativeLayout>
測試結果:[fullQuery]
05-16 00:01:35.845 11175-11468/com.johnsoft.app.temp W/CursorWindow: Window is full: requested allocation 404 bytes, free space 114 bytes, window size 2097152 bytes
05-16 00:01:37.933 11175-11468/com.johnsoft.app.temp W/CursorWindow: Window is full: requested allocation 6 bytes, free space 0 bytes, window size 2097152 bytes
05-16 00:01:39.519 11175-11468/com.johnsoft.app.temp W/CursorWindow: Window is full: requested allocation 6 bytes, free space 0 bytes, window size 2097152 bytes
05-16 00:01:41.150 11175-11468/com.johnsoft.app.temp W/CursorWindow: Window is full: requested allocation 36 bytes, free space 16 bytes, window size 2097152 bytes
05-16 00:01:42.812 11175-11468/com.johnsoft.app.temp W/CursorWindow: Window is full: requested allocation 404 bytes, free space 114 bytes, window size 2097152 bytes
05-16 00:01:44.489 11175-11468/com.johnsoft.app.temp W/CursorWindow: Window is full: requested allocation 6 bytes, free space 0 bytes, window size 2097152 bytes
05-16 00:01:46.176 11175-11468/com.johnsoft.app.temp W/CursorWindow: Window is full: requested allocation 6 bytes, free space 0 bytes, window size 2097152 bytes
05-16 00:01:47.849 11175-11468/com.johnsoft.app.temp W/CursorWindow: Window is full: requested allocation 6 bytes, free space 0 bytes, window size 2097152 bytes
05-16 00:01:49.582 11175-11468/com.johnsoft.app.temp W/CursorWindow: Window is full: requested allocation 404 bytes, free space 114 bytes, window size 2097152 bytes
05-16 00:01:51.345 11175-11468/com.johnsoft.app.temp W/CursorWindow: Window is full: requested allocation 36 bytes, free space 18 bytes, window size 2097152 bytes
05-16 00:01:53.092 11175-11468/com.johnsoft.app.temp W/CursorWindow: Window is full: requested allocation 6 bytes, free space 0 bytes, window size 2097152 bytes
05-16 00:01:54.825 11175-11468/com.johnsoft.app.temp W/CursorWindow: Window is full: requested allocation 6 bytes, free space 0 bytes, window size 2097152 bytes
05-16 00:01:56.633 11175-11468/com.johnsoft.app.temp W/CursorWindow: Window is full: requested allocation 404 bytes, free space 114 bytes, window size 2097152 bytes
05-16 00:01:58.474 11175-11468/com.johnsoft.app.temp W/CursorWindow: Window is full: requested allocation 36 bytes, free space 26 bytes, window size 2097152 bytes
05-16 00:02:00.324 11175-11468/com.johnsoft.app.temp W/CursorWindow: Window is full: requested allocation 6 bytes, free space 0 bytes, window size 2097152 bytes
05-16 00:02:02.186 11175-11468/com.johnsoft.app.temp W/CursorWindow: Window is full: requested allocation 6 bytes, free space 0 bytes, window size 2097152 bytes
05-16 00:02:03.927 11175-11468/com.johnsoft.app.tempx W/System.err: Size: 400000
05-16 00:02:03.927 11175-11468/com.johnsoft.app.tempx W/System.err: Query count ccccccccccccccc 4355560
05-16 00:02:03.927 11175-11468/com.johnsoft.app.tempx W/System.err: total: 29344
測試結果[segQuery]:
05-16 00:04:52.782 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 0
05-16 00:04:52.849 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 4000
05-16 00:04:52.912 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 8000
05-16 00:04:52.972 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 12000
05-16 00:04:53.032 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 16000
05-16 00:04:53.100 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 20000
05-16 00:04:53.162 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 24000
05-16 00:04:53.217 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 28000
05-16 00:04:53.267 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 32000
05-16 00:04:53.322 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 36000
05-16 00:04:53.378 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 40000
05-16 00:04:53.436 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 44000
05-16 00:04:53.489 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 48000
05-16 00:04:53.544 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 52000
05-16 00:04:53.599 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 56000
05-16 00:04:53.661 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 60000
05-16 00:04:53.715 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 64000
05-16 00:04:53.769 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 68000
05-16 00:04:53.825 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 72000
05-16 00:04:53.881 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 76000
05-16 00:04:53.943 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 80000
05-16 00:04:54.005 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 84000
05-16 00:04:54.063 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 88000
05-16 00:04:54.121 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 92000
05-16 00:04:54.180 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 96000
05-16 00:04:54.247 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 100000
05-16 00:04:54.306 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 104000
05-16 00:04:54.364 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 108000
05-16 00:04:54.428 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 112000
05-16 00:04:54.495 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 116000
05-16 00:04:54.555 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 120000
05-16 00:04:54.616 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 124000
05-16 00:04:54.678 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 128000
05-16 00:04:54.741 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 132000
05-16 00:04:54.810 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 136000
05-16 00:04:54.872 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 140000
05-16 00:04:54.940 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 144000
05-16 00:04:55.015 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 148000
05-16 00:04:55.081 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 152000
05-16 00:04:55.149 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 156000
05-16 00:04:55.219 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 160000
05-16 00:04:55.287 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 164000
05-16 00:04:55.369 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 168000
05-16 00:04:55.449 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 172000
05-16 00:04:55.526 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 176000
05-16 00:04:55.605 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 180000
05-16 00:04:55.681 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 184000
05-16 00:04:55.759 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 188000
05-16 00:04:55.847 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 192000
05-16 00:04:55.935 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 196000
05-16 00:04:56.051 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 200000
05-16 00:04:56.145 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 204000
05-16 00:04:56.229 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 208000
05-16 00:04:56.315 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 212000
05-16 00:04:56.409 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 216000
05-16 00:04:56.498 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 220000
05-16 00:04:56.583 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 224000
05-16 00:04:56.686 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 228000
05-16 00:04:56.773 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 232000
05-16 00:04:56.863 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 236000
05-16 00:04:56.964 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 240000
05-16 00:04:57.056 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 244000
05-16 00:04:57.154 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 248000
05-16 00:04:57.250 11175-13623/com.johnsoft.app.temp W/System.err: args0: 4000, args1: 252000
05-16 00:04:
相關推薦
android查詢幾十萬條資料的調研(一)
此次調研分兩步走吧, 先從測試的角度看(一), 再從原始碼的角度看(二)(待續). 下面的是測試程式碼:
SQLiteUtils.java
package com.johnsoft.app.temp;
import android.content.Context;
import
Excel匯出幾十萬條資料
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Microsoft.Office.Interop.Excel;
using System.Data;
namespace E
一張表中不同行的資料(join聯合)查詢到同一條資料中
$list = Db::query('select t.phone_number,t.group_name,t1.friend_number F1,t2.friend_number F2,(t2.friend_number-t1.friend_number) f from g
sql insert into 一次性插入多條資料 從一張表中查詢到的資料插入到另一張表
--插入多條資料使用DEFAULT關鍵字(第二種方法,不要將預設列名寫出,在UNION後面加上all,最後一行不加)
------------------------------------------------------------------------------
如何處理幾十萬條併發資料
資源洩漏。在 .NET 2.0 之前的版本中, ReaderWriterLock 類會造成核心物件洩露。這些物件只有在程序終止後才能再次回收。幸運的是,.NET 2.0 修正了這個 Bug 。
此外,ReaderWriterLock 還有個令人擔心的危險的非原子性操作。它就是 UpgradeTo
Java一次性查詢幾十萬 幾百萬資料解決辦法
Java查詢一次性查詢幾十萬,幾百萬資料解決辦法。
很早的時候寫工具用的一個辦法,當時是用來把百萬資料打包成rar檔案。
所以用了個笨辦法。 希望高手指導一下,有什麼好方法沒有啊。
以下是查詢資料庫。按批次查詢
publicstaticvoid getMonthDataList() {
mysql 一次插入幾萬條資料應該怎麼做優化
對於一些資料量較大的系統,資料庫面臨的問題除了查詢效率低下,還有就是資料入庫時間長。特別像報表系統,每天花費在資料匯入上的時間可能會長達幾個小時或十幾個小時之久。因此,優化資料庫插入效能是很有意義的。經過對MySQL InnoDB的一些效能測試,發現一些可以提高insert效
Android 簡單幾行程式碼實現搖一搖功能
1、activity 實現加速度監聽類 。。。。implements SensorEventListener
public class MainActivity extends AppCompatActivity implements SensorEventListene
mysql 資料庫查詢最後兩條資料
有一個mysql資料庫的表,要查詢他的最後兩條資料的結果例如以下:
這是原表中的內容:
idname 1 ad 2 jk 3 tre
4 hgv
這是查詢的最後兩條的資料的內容(為最新的插入的資料):
批量修改,每500條資料更新一次
// 商品資訊,每500條批量更新一次
if(null != updateItemList && updateItemList.size() > 0){
Map
高效能分散式查詢五千萬條資料3秒查詢完畢
package com.dinglin;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
mysql儲存過程插入十萬條資料
用儲存過程建立一個類似這樣的表,從0到99999的五位數,不足五位數的補足
delimiter // #定義識別符號為雙斜槓
drop procedure if exists init; #如果
查詢幾個表資料,根據存在情況,插入到另一個表中
INSERT INTO now_pay_cust_stat (
customer_id,
n_p_custid,
sum_offer_order,
sum_offer_p
面試題--如何渲染幾萬條資料並不卡住介面
根據題意,如果我們一次性渲染重新整理幾萬條資料,頁面會卡頓,因此只能分批渲染,既然知道原理我們就可以使用setInterval和setTimeout、requestAnimationFrame來實現定時
oracle 幾十萬的資料單表多個欄位快速更新的方法
merge into 要修改的表名 別名1
using (select 要修改的欄位1,要修改的欄位2, 關聯的欄位 from 表名) 別名2
on (別名1.關聯欄位 = 別名2. 關聯欄位)
when matched then update set
別
查詢前多少條資料
查詢前N條記錄
SELECT TOP 10 * FROM 訂單明細 ORDER BY 數量 DESC
查詢前n%條記錄
SELECT top 10 PERCENT * FROM 訂單明細 ORDER BY 數量
查詢前n%條記錄(包含最後並
MyBatis批量插入幾千條資料慎用foreach
近日,專案中有一個耗時較長的Job存在CPU佔用過高的問題,經排查發現,主要時間消耗在往MyBatis中批量插入資料。mapper configuration是用foreach迴圈做的,差不多是這樣。(由於專案保密,以下程式碼均為自己手寫的demo程式碼)
<
sql分組(orderBy、GroupBy)獲取每組前一(幾)條資料
sql資料庫實現分組並取每組的前1(幾)條資料
測試資料準備工作:
根據某一個欄位分組取最大(小)值所在行的資料:
建立表並且插入資料
CREATE table Test_orderByOrGroupBy_tb(Name nvarchar(50),Val int,Describe n
Oracle刪除重複記錄只保留一條資料的幾種方法
1、查詢表中多餘的重複記錄,重複記錄是根據單個欄位(peopleId)來判斷
SELECT
*
FROM
people
WHERE
peopleid IN (
SELECT
peopleid
FROM
people
GROUP BY
peopleid
查詢某張表中時間最近的一條資料
SELECT
a1.id
FROM
a a1
LEFT JOIN b b1 ON a1.cid = b1.cid