1. 程式人生 > >【資料庫效能測試實戰】測試不同分頁儲存過程在10w,100w以及1000w資料量下面的表現

【資料庫效能測試實戰】測試不同分頁儲存過程在10w,100w以及1000w資料量下面的表現

前言

資料庫的效能與每一行程式碼息息相關,所以,每次寫程式碼可以考慮一下在不同級別的資料量下面測試一下效能。
本文參考了:
Postgresql生成大量測試資料

以及

準備測試用資料

此次測試我們將分別用10w,100w以及1000w級別的表來測試,下面先建立一波資料表。

-- 測試用資料表,分別為10w,100w以及1000w級別。表結構基本一致。
create table tbl_test_10w (
id serial primary key,
"name" varchar(200) null,
"groupid" integer null,
create_time timestamp(0) without time zone
);
insert into tbl_test_10w (name,groupid,create_time) 
select md5(n::varchar||random()::varchar) as "name", 
(floor(RANDOM()*(100))::int%15) as groupid,
('2018-5-1'::date + trunc(random()*100)::integer +' 00:22:22'::time + (trunc(random()*3600*24)||' second')::interval) as create_time 
 from generate_series(0,100000) n;
select count(*) from tbl_test_10w;

-- 100w資料量

create table tbl_test_100w (
id serial primary key,
"name" varchar(200) null,
"groupid" integer null,
create_time timestamp(0) without time zone
);
insert into tbl_test_100w (name,groupid,create_time) 
select md5(n::varchar||random()::varchar) as "name", 
(floor(RANDOM()*(100))::int%15) as groupid,
('2018-5-1'::date + trunc(random()*100)::integer +' 00:22:22'::time + (trunc(random()*3600*24)||' second')::interval) as create_time 
 from generate_series(0,1000000) n;
select count(*) from tbl_test_100w;
-- 1000w資料量

create table tbl_test_1000w (
id serial primary key,
"name" varchar(200) null,
"groupid" integer null,
create_time timestamp(0) without time zone
);
insert into tbl_test_1000w (name,groupid,create_time) 
select md5(n::varchar||random()::varchar) as "name", 
(floor(RANDOM()*(100))::int%15) as groupid,
('2018-5-1'::date + trunc(random()*100)::integer +' 00:22:22'::time + (trunc(random()*3600*24)||' second')::interval) as create_time 
 from generate_series(0,10000000) n;
select count(*) from tbl_test_1000w;

接下來,看看這些記錄佔用空間:

select pg_size_pretty(pg_relation_size('tbl_test_1000w'));

在這裡插入圖片描述

效能測試工具
待會需要用到:top free vmstat iostat【可能會用】等工具,用的是centos7系統,
還有,可以參考:

效能測試

先說一下渣機,阿里雲上面的配置:

在這裡插入圖片描述

然後再說明一下測試程式碼,首先,全域性配置 config:

-- 這是全域性配置

create or replace function config(

)
returns jsonb
language 'plpgsql'
IMMUTABLE
as $body$

declare job_manager_config constant jsonb := '{
"pager":{
		"pageSize":20
}

}';

BEGIN

return job_manager_config;

END;
$body$;

-- 快速獲取配置項。

create or replace function config(
variadic path_array text[]
)
returns text
LANGUAGE 'plpgsql'
IMMUTABLE
as $body$

begin 
return jsonb_extract_path_text(config(),variadic path_array);
end;
$body$;

針對系統的自定義型別:

/***
系統基本型別。
***/
-- 2018-10-28
drop type sys_types_pager_outline;
create  type sys_type_pager_outline as (
    "pageIndex" integer,
    "pageSize" integer,
    "total" integer,
    "totalPages" integer,
		"beginIndex" integer,
    "state" boolean,
    "stateCode" integer,    
    "message" varchar(600)
);
drop type sys_type_operation_outline;
create type sys_type_operation_outline as (
    "state" boolean,
    "stateCode" integer,    
    "message" varchar(600)
);

分頁偏移量計算:

/****系統基本函式**/

-- 計算分頁的偏移還有總頁碼等資訊。
create or replace function sys_func_cal_pager(
in pageIndex integer,
in pageSize integer,
in totalSize integer
)
returns sys_type_pager_outline
as
$body$
declare pagerOutLine sys_type_pager_outline; 

begin

-- 初始化分頁基本資料。
pagerOutLine."pageIndex":=1;
pagerOutLine."pageSize":=config('pager','pageSize');
pagerOutLine."total":=0;
pagerOutLine."totalPages":=0;
pagerOutLine."beginIndex":=0;

if pageIndex > 0 then
pagerOutLine."pageIndex":=pageIndex;
end if;

if pageSize > 0 then 
pagerOutLine."pageSize"=pageSize;
end if;

if totalSize > 0 then
pagerOutLine."total":=totalSize;

-- 計算總頁碼
if pagerOutLine."total" % pagerOutLine."pageSize" = 0  then
pagerOutLine."totalPages" := pagerOutLine."total"/pagerOutLine."pageSize";
else
pagerOutLine."totalPages" := (pagerOutLine."total"-(pagerOutLine."total"%pagerOutLine."pageSize"))/pagerOutLine."pageSize"+1;
end if;
-- 計算當前頁碼是不是超出正常水平
if pagerOutLine."pageIndex" > pagerOutLine."totalPages" then
pagerOutLine."pageIndex" := pagerOutLine."totalPages";
end if;

-- 計算offset
pagerOutLine."beginIndex"=(pagerOutLine."pageIndex"-1) * pagerOutLine."pageSize";


else 

pagerOutLine."total":=0;
pagerOutLine."totalPages":=0;
pagerOutLine."pageIndex":=1;
pagerOutLine."beginIndex":=0;
end if;


return pagerOutLine;

end;
$body$ language plpgsql volatile;

好了,接下來才是正主,幾個測試用分頁儲存過程:

/*********為了可以測試各個數量級的效能,請屆時將tbl_test_1000w這個名稱換成100w以及10w級別的。*************/

/***
*
* 測試用分頁儲存過程,其中利用了臨時表儲存中間記錄集
*
***/
CREATE OR REPLACE FUNCTION public.test_case_search_pager_using_temp_table(
	in pageindex integer,
	in pagesize integer,
	cnd_eq_id integer,
	cnd_min_groupid integer,
	cnd_max_groupid integer,
	cnd_begin_create_time timestamp without time zone,
	cnd_end_create_time timestamp without time zone,
	outline_cursor refcursor,
	records_cursor refcursor)
    RETURNS setof refcursor  
AS $BODY$
declare pagerOutLine sys_type_pager_outline;
declare totalSize integer;
declare tmpStr varchar(400);
begin




-- 根據條件查詢列表,然後存放到臨時表裡面去。

create temp table "temp_tbl4pager"
on commit drop 
as select * from  
(
select * from tbl_test_1000w 
where 1=1
and
case when cnd_eq_id is not null then "id"=cnd_eq_id else 1=1 end
and 
case when cnd_min_groupid is not null then "groupid">cnd_min_groupid else 1=1 end

and 
case when cnd_max_groupid is not null then "groupid"<cnd_max_groupid else 1=1 end

and
case when cnd_begin_create_time is not null then "create_time">cnd_begin_create_time else 1=1 end
and
case when cnd_end_create_time is not null then "create_time"< cnd_end_create_time else 1=1 end
) tbl_middle;

select count(*) into totalSize from temp_tbl4pager;


pagerOutLine:=sys_func_cal_pager(pageIndex,pageSize,totalSize);
-- raise notice '好了,看看pagerOutline的資料:';
-- raise notice '看看pagerOutLine裡面的資料:%,真奇怪,那麼,totalSize 是:%',pagerOutLine,totalSize;
-- 定義返回的狀態。
pagerOutLine."state":=true;
pagerOutLine."stateCode":=0;
pagerOutLine."message":='';

-- 返回相關資料。
open outline_cursor for select pagerOutLine;
open records_cursor for select * from temp_tbl4pager limit pagerOutLine."pageSize" offset pagerOutLine."beginIndex";


-- 返回資料
return next outline_cursor;
return next records_cursor;

end;

$BODY$ LANGUAGE plpgsql volatile;


/***
*
* 測試用分頁儲存過程,其中使用了cte
*
***/
CREATE OR REPLACE FUNCTION public.test_case_search_pager_using_cte(
	in pageindex integer,
	in pagesize integer,
	cnd_eq_id integer,
	cnd_min_groupid integer,
	cnd_max_groupid integer,
	cnd_begin_create_time timestamp without time zone,
	cnd_end_create_time timestamp without time zone,
	outline_cursor refcursor,
	records_cursor refcursor)
    RETURNS setof refcursor  
AS $BODY$
declare pagerOutLine sys_type_pager_outline;
declare totalSize integer;
declare tmpStr varchar(400);
begin



-- 首先獲取記錄總數
with tbl_middle as (
select * from tbl_test_1000w 
where 1=1
and
case when cnd_eq_id is not null then "id"=cnd_eq_id else 1=1 end
and 
case when cnd_min_groupid is not null then "groupid">cnd_min_groupid else 1=1 end

and 
case when cnd_max_groupid is not null then "groupid"<cnd_max_groupid else 1=1 end

and
case when cnd_begin_create_time is not null then "create_time">cnd_begin_create_time else 1=1 end
and
case when cnd_end_create_time is not null then "create_time"< cnd_end_create_time else 1=1 end
) 
select count(*) into totalSize from tbl_middle;


-- 然後計算beginIndex,totalPages
pagerOutLine:=sys_func_cal_pager(pageIndex,pageSize,totalSize);
-- raise notice '好了,看看pagerOutline的資料:';
-- raise notice '看看pagerOutLine裡面的資料:%,真奇怪,那麼,totalSize 是:%',pagerOutLine,totalSize;
-- 定義返回的狀態。
pagerOutLine."state":=true;
pagerOutLine."stateCode":=0;
pagerOutLine."message":='';

-- 返回相關資料。
open outline_cursor for select pagerOutLine;
open records_cursor for select * from 

(
select * from tbl_test_1000w 
where 1=1
and
case when cnd_eq_id is not null then "id"=cnd_eq_id else 1=1 end
and 
case when cnd_min_groupid is not null then "groupid">cnd_min_groupid else 1=1 end

and 
case when cnd_max_groupid is not null then "groupid"<cnd_max_groupid else 1=1 end

and
case when cnd_begin_create_time is not null then "create_time">cnd_begin_create_time else 1=1 end
and
case when cnd_end_create_time is not null then "create_time"< cnd_end_create_time else 1=1 end
)  tbl_middle
 limit pagerOutLine."pageSize" offset pagerOutLine."beginIndex";


-- 返回資料
return next outline_cursor;
return next records_cursor;

end;

$BODY$ LANGUAGE plpgsql volatile;



/***
*
* 測試用分頁儲存過程,原始版本,不用cte等等特性。
*
***/
CREATE OR REPLACE FUNCTION public.test_case_search_pager_origin(
	in pageindex integer,
	in pagesize integer,
	cnd_eq_id integer,
	cnd_min_groupid integer,
	cnd_max_groupid integer,
	cnd_begin_create_time timestamp without time zone,
	cnd_end_create_time timestamp without time zone,
	outline_cursor refcursor,
	records_cursor refcursor)
    RETURNS setof refcursor  
AS $BODY$
declare pagerOutLine sys_type_pager_outline;
declare totalSize integer;
declare tmpStr varchar(400);
begin



-- 首先獲取記錄總數

select count(*) into totalSize from 
(
select * from tbl_test_1000w 
where 1=1
and
case when cnd_eq_id is not null then "id"=cnd_eq_id else 1=1 end
and 
case when cnd_min_groupid is not null then "groupid">cnd_min_groupid else 1=1 end

and 
case when cnd_max_groupid is not null then "groupid"<cnd_max_groupid else 1=1 end

and
case when cnd_begin_create_time is not null then "create_time">cnd_begin_create_time else 1=1 end
and
case when cnd_end_create_time is not null then "create_time"< cnd_end_create_time else 1=1 end
) tbl_middle
;


-- 然後計算beginIndex,totalPages
pagerOutLine:=sys_func_cal_pager(pageIndex,pageSize,totalSize);
-- raise notice '好了,看看pagerOutline的資料:';
-- raise notice '看看pagerOutLine裡面的資料:%,真奇怪,那麼,totalSize 是:%',pagerOutLine,totalSize;
-- 定義返回的狀態。
pagerOutLine."state":=true;
pagerOutLine."stateCode":=0;
pagerOutLine."message":='';

-- 返回相關資料。
open outline_cursor for select pagerOutLine;
open records_cursor for select * from 

(
select * from tbl_test_1000w 
where 1=1
and
case when cnd_eq_id is not null then "id"=cnd_eq_id else 1=1 end
and 
case when cnd_min_groupid is not null then "groupid">cnd_min_groupid else 1=1 end

and 
case when cnd_max_groupid is not null then "groupid"<cnd_max_groupid else 1=1 end

and
case when cnd_begin_create_time is not null then "create_time">cnd_begin_create_time else 1=1 end
and
case when cnd_end_create_time is not null then "create_time"< cnd_end_create_time else 1=1 end
)  tbl_middle
 limit pagerOutLine."pageSize" offset pagerOutLine."beginIndex";


-- 返回資料
return next outline_cursor;
return next records_cursor;

end;

$BODY$ LANGUAGE plpgsql volatile;







先測試1000w條記錄的資料庫的讀取時間…

首先,用兩個ssh連線到阿里雲伺服器。。嗯,每個視窗分別準備一個命令:

-- 每一秒繪製一次記憶體使用情況,持續60次。採用mb單位顯示。
vmstat 1 60 -S M

以及:

-- 列出所有postgres使用者持有的程序,每次檢測間隔1s,批次執行模式方便監測,對了 -c表示要顯示完整的命令。
top -b -c  -d 1   -U postgres

如下圖:
在這裡插入圖片描述

在這裡插入圖片描述

然後準備一段呼叫程式碼:

begin;

select test_case_search_pager_using_temp_table(
	2, -- pageIndex
	399, -- pageSize
	null, -- equal id
	7,-- min group id
	19, -- max group id 
	null,-- begin create time
	null,-- end create time
	'outline_crs_temp_table',
	'records_crs_temp_table');
fetch all in outline_crs_temp_table;
fetch all in records_crs_temp_table;
end;

然後,先執行top和vmstat進行監測,再執行sql程式碼,你會發現如下結果:

在這裡插入圖片描述

1000w的排序足足用了5秒多,搞不好會上6秒的。。。嗯。。。。
然後,檢視監測情況:

有一段時間,記憶體是急劇消耗的,最低空閒記憶體降到115mb ,
而cpu方面的 監測圖例有:

在這裡插入圖片描述

又多又雜的跟蹤,很難用肉眼來判斷。。

而我們注意到,20522這個pid似乎是用得最多,壓根就是我們執行的pid,於是,我們可以直接改改命令,新增pid指定,方便監控。
在這裡插入圖片描述

改為:

-- 注意,-U -p這些沒辦法同時使用。。
top -b -c  -d 1    -p 20522

然後再嘗試一次。。

得到結果:

在這裡插入圖片描述

procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
 r  b   swpd   free   buff  cache   si   so    bi    bo   in   cs us sy id wa st
 4  0      0   1060      7    400    0    0    12    46    9   10  2  1 96  0  0
 0  0      0   1060      7    400    0    0     0    40  369 1008  1  0 99  0  0
 0  0      0   1059      7    400    0    0     0    76  388 1042  2  2 96  0  0
14  0      0    971      7    489    0    0 65048   240  783 1033 18  7 74  1  0
 1  0      0    612      7    848    0    0 252456 58000 1781  738 70 29  0  1  0
 2  0      0    296      7   1164    0    0 222092 114824 1634  815 62 25  0 13  0
 4  0      0    130      7   1330    0    0 62176 114832 1571  834 73 21  0  6  0
 3  0      0    103      1   1172    0    0     4    40 1400  727 65 35  0  0  0
 1  0      0   1094      1    371    0    0    24   192 1161  823 29 51 19  1  0
 4  0      0   1090      1    375    0    0  4712    52  458 1083  2  1 85 12  0

而top的結果是:

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  405988 133680 120628 S  0.0  3.4   0:18.31 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle

top - 23:30:16 up 47 days, 12:01,  4 users,  load average: 0.10, 0.72, 0.68
Tasks:   1 total,   0 running,   1 sleeping,   0 stopped,   0 zombie
%Cpu(s):  1.0 us,  1.0 sy,  0.0 ni, 98.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem :  3882052 total,  1085684 free,  2378848 used,   417520 buff/cache
KiB Swap:        0 total,        0 free,        0 used.  1107460 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  405988 133680 120628 S  0.0  3.4   0:18.31 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle

top - 23:30:17 up 47 days, 12:01,  4 users,  load average: 0.09, 0.71, 0.68
Tasks:   1 total,   1 running,   0 sleeping,   0 stopped,   0 zombie
%Cpu(s):  5.9 us,  2.9 sy,  0.0 ni, 91.2 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem :  3882052 total,  1085684 free,  2378848 used,   417520 buff/cache
KiB Swap:        0 total,        0 free,        0 used.  1107460 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  405988 133940 120888 R  4.0  3.5   0:18.35 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT

top - 23:30:18 up 47 days, 12:01,  4 users,  load average: 0.09, 0.71, 0.68
Tasks:   1 total,   1 running,   0 sleeping,   0 stopped,   0 zombie
%Cpu(s): 69.7 us, 28.3 sy,  0.0 ni,  0.0 id,  2.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem :  3882052 total,   697940 free,  2378356 used,   805756 buff/cache
KiB Swap:        0 total,        0 free,        0 used.  1106284 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  405988 133940 120888 R 96.0  3.5   0:19.31 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT

top - 23:30:19 up 47 days, 12:01,  4 users,  load average: 0.09, 0.71, 0.68
Tasks:   1 total,   1 running,   0 sleeping,   0 stopped,   0 zombie
%Cpu(s): 66.0 us, 26.0 sy,  0.0 ni,  0.0 id,  8.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem :  3882052 total,   697940 free,  2378356 used,   805756 buff/cache
KiB Swap:        0 total,        0 free,        0 used.  1106284 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  405988 133940 120888 R 88.1  3.5   0:20.20 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT

top - 23:30:20 up 47 days, 12:01,  4 users,  load average: 0.09, 0.71, 0.68
Tasks:   1 total,   1 running,   0 sleeping,   0 stopped,   0 zombie
%Cpu(s): 71.0 us, 23.0 sy,  0.0 ni,  0.0 id,  6.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem :  3882052 total,   697940 free,  2378356 used,   805756 buff/cache
KiB Swap:        0 total,        0 free,        0 used.  1106284 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  405988 133940 120888 R 88.0  3.5   0:21.08 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT

top - 23:30:21 up 47 days, 12:01,  4 users,  load average: 0.09, 0.71, 0.68
Tasks:   1 total,   1 running,   0 sleeping,   0 stopped,   0 zombie
%Cpu(s): 67.6 us, 27.5 sy,  0.0 ni,  0.0 id,  4.9 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem :  3882052 total,   125756 free,  2444088 used,  1312208 buff/cache
KiB Swap:        0 total,        0 free,        0 used.  1037680 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  478612 199908 120892 R 91.1  5.1   0:22.00 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT

top - 23:30:22 up 47 days, 12:01,  4 users,  load average: 0.96, 0.88, 0.73
Tasks:   1 total,   0 running,   1 sleeping,   0 stopped,   0 zombie
%Cpu(s): 38.4 us, 60.6 sy,  0.0 ni,  1.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem :  3882052 total,   125756 free,  2444088 used,  1312208 buff/cache
KiB Swap:        0 total,        0 free,        0 used.  1037680 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  405972 133928 120892 S 88.0  3.4   0:22.88 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle

top - 23:30:23 up 47 days, 12:01,  4 users,  load average: 0.96, 0.88, 0.73
Tasks:   1 total,   0 running,   1 sleeping,   0 stopped,   0 zombie
%Cpu(s):  2.0 us,  1.0 sy,  0.0 ni, 84.2 id, 12.9 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem :  3882052 total,   125756 free,  2444088 used,  1312208 buff/cache
KiB Swap:        0 total,        0 free,        0 used.  1037680 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  405972 133928 120892 S  0.0  3.4   0:22.88 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle

嗯,臨時表的耗時是5秒以上。。。

下面來看看cte的,兩次查詢,不用臨時表,看看效率。

測試程式碼:

begin;
-- select test_case_search_pager_using_temp_table(
select test_case_search_pager_using_temp_table(
	2, -- pageIndex
	399, -- pageSize
	null, -- equal id
	7,-- min group id
	19, -- max group id 
	null,-- begin create time
	null,-- end create time
	'outline_crs_temp_table',
	'records_crs_temp_table');
fetch all in outline_crs_temp_table;
fetch all in records_crs_temp_table;
end;


其他跟之前的一樣。

可以看到:

在這裡插入圖片描述

還是5秒多,那麼看看其他結果:

procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
 r  b   swpd   free   buff  cache   si   so    bi    bo   in   cs us sy id wa st
 5  0      0   1125      0    342    0    0    13    47    9    0  2  1 96  0  0
 0  0      0   1124      0    342    0    0     0   116  419 1000  4  2 93  1  0
 0  0      0   1124      0    342    0    0     0    40  341  976  1  1 98  0  0
 0  0      0   1124      0    342    0    0     0    72  374 1013  2  1 97  0  0
 0  0      0   1124      0    342    0    0     0    40  361  994  1  0 99  0  0
 3  1      0   1041      0    426    0    0 61536    56  712  992 16  6 78  0  0
 1  0      0    670      0    797    0    0 261788 53388 1708  731 71 28  0  1  0
 1  0      0    321      0   1146    0    0 242284 106792 1746  804 69 27  0  4  0
 1  0      0    167      0   1300    0    0 63980 113120 1334  934 58 19  0 23  0
 2  0      0    145      0   1105    0    0     0    92 1362  743 65 35  0  0  0
 0  0      0   1129      0    337    0    0   256    48 1057  789 27 45 27  0  0
 0  0      0   1130      0    337    0    0     0    40  350  966  2  0 98  0  0
 0  0      0   1129      0    337    0    0     0   140  349  954  2  1 97  0  0
 0  0      0   1130      0    337    0    0   668    40  373  986  1  0 98  1  0
 0  0      0   1129      0    338    0    0   248    40  372 1007  2  1 97  0  0
 0  0      0   1129      0    338    0    0     0    40  331  958  1  0 99  0  0



  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  406156 135212 121984 S  0.0  3.5   0:55.59 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle

top - 23:52:44 up 47 days, 12:23,  4 users,  load average: 0.05, 0.46, 0.62
Tasks:   1 total,   0 running,   1 sleeping,   0 stopped,   0 zombie
%Cpu(s):  1.0 us,  0.0 sy,  0.0 ni, 99.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem :  3882052 total,  1151460 free,  2379272 used,   351320 buff/cache
KiB Swap:        0 total,        0 free,        0 used.  1120732 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  406156 135212 121984 S  0.0  3.5   0:55.59 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle

top - 23:52:45 up 47 days, 12:23,  4 users,  load average: 0.05, 0.46, 0.62
Tasks:   1 total,   1 running,   0 sleeping,   0 stopped,   0 zombie
%Cpu(s): 15.0 us,  6.0 sy,  0.0 ni, 79.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem :  3882052 total,  1151460 free,  2379272 used,   351320 buff/cache
KiB Swap:        0 total,        0 free,        0 used.  1120732 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  406156 135212 121984 R 18.0  3.5   0:55.77 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT

top - 23:52:46 up 47 days, 12:24,  4 users,  load average: 0.05, 0.46, 0.62
Tasks:   1 total,   1 running,   0 sleeping,   0 stopped,   0 zombie
%Cpu(s): 71.0 us, 28.0 sy,  0.0 ni,  0.0 id,  1.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem :  3882052 total,   693076 free,  2378228 used,   810748 buff/cache
KiB Swap:        0 total,        0 free,        0 used.  1106416 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  406156 135212 121984 R 97.0  3.5   0:56.75 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT

top - 23:52:47 up 47 days, 12:24,  4 users,  load average: 0.21, 0.48, 0.63
Tasks:   1 total,   1 running,   0 sleeping,   0 stopped,   0 zombie
%Cpu(s): 69.0 us, 27.0 sy,  0.0 ni,  0.0 id,  4.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem :  3882052 total,   693076 free,  2378228 used,   810748 buff/cache
KiB Swap:        0 total,        0 free,        0 used.  1106416 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  406156 135212 121984 R 93.0  3.5   0:57.68 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT

top - 23:52:48 up 47 days, 12:24,  4 users,  load average: 0.21, 0.48, 0.63
Tasks:   1 total,   1 running,   0 sleeping,   0 stopped,   0 zombie
%Cpu(s): 59.0 us, 18.0 sy,  0.0 ni,  0.0 id, 23.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem :  3882052 total,   693076 free,  2378228 used,   810748 buff/cache
KiB Swap:        0 total,        0 free,        0 used.  1106416 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  406156 135212 121984 R 73.3  3.5   0:58.42 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT

top - 23:52:49 up 47 days, 12:24,  4 users,  load average: 0.21, 0.48, 0.63
Tasks:   1 total,   1 running,   0 sleeping,   0 stopped,   0 zombie
%Cpu(s): 65.3 us, 34.7 sy,  0.0 ni,  0.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem :  3882052 total,   101368 free,  2591208 used,  1189476 buff/cache
KiB Swap:        0 total,        0 free,        0 used.   890824 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  626256 347796 122140 R 97.0  9.0   0:59.39 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT

top - 23:52:50 up 47 days, 12:24,  4 users,  load average: 0.21, 0.48, 0.63
Tasks:   1 total,   0 running,   1 sleeping,   0 stopped,   0 zombie
%Cpu(s): 28.3 us, 46.5 sy,  0.0 ni, 25.3 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem :  3882052 total,   101368 free,  2591208 used,  1189476 buff/cache
KiB Swap:        0 total,        0 free,        0 used.   890824 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  406156 135368 122140 S 65.3  3.5   1:00.05 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle

top - 23:52:51 up 47 days, 12:24,  4 users,  load average: 0.21, 0.48, 0.63
Tasks:   1 total,   0 running,   1 sleeping,   0 stopped,   0 zombie
%Cpu(s):  2.0 us,  0.0 sy,  0.0 ni, 98.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem :  3882052 total,   101368 free,  2591208 used,  1189476 buff/cache
KiB Swap:        0 total,        0 free,        0 used.   890824 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  406156 135368 122140 S  0.0  3.5   1:00.05 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle

差距沒多大啊。。

下面看看不用cte的。。

begin;
-- select test_case_search_pager_using_temp_table(
-- select test_case_search_pager_using_temp_table(
select test_case_search_pager_origin(
	2, -- pageIndex
	399, -- pageSize
	null, -- equal id
	7,-- min group id
	19, -- max group id 
	null,-- begin create time
	null,-- end create time
	'outline_crs_temp_table',
	'records_crs_temp_table');
fetch all in outline_crs_temp_table;
fetch all in records_crs_temp_table;
end;

然後測試:
在這裡插入圖片描述

procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
 r  b   swpd   free   buff  cache   si   so    bi    bo   in   cs us sy id wa st
 4  0      0    456      4   1006    0    0    13    47    9    0  2  1 96  0  0
 0  0      0    455      4   1006    0    0     0    40  389 1066  1  1 98  0  0
 0  0      0    456      4   1006    0    0     0    52  368 1022  1  1 98  0  0
 2  0      0    456      4   1006    0    0     0    40  291  924  1  1 98  0  0
 0  0      0    456      4   1006    0    0     0   100  346  991  1  0 99  0  0
 0  0      0    456      4   1006    0    0     0    40  304  942  1  1 98  0  0
 0  0      0    455      4   1006    0    0     0    40  361  976  2  1 97  0  0
 3  0      0    452      4   1008    0    0  1892     0 1230 3788 48 32 17  3  0
 1  0      0    451      4   1008    0    0     0    80 1420 4396 60 40  0  0  0
 1  0      0    452      4   1008    0    0     0   108 1406 4228 62 38  0  0  0
 9  0      0    446      4   1009    0    0     0   112 1341 3003 68 32  0  0  0
13  0      0    447      4   1009    0    0     0    40 1272 1069 83 17  0  0  0
 3  0      0    452      4   1008    0    0     0    40 1369 3571 64 36  0  0  0
 2  0      0    452      4   1008    0    0     0    40 1390 4355 59 41  0  0  0
 1  0      0    453      4   1008    0    0     0    40 1406 4523 56 44  0  0  0
 4  0      0    452      4   1008    0    0   148   176 1406 4359 58 42  0  0  0


  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  409836 139740 125028 S  0.0  3.6   1:01.77 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle

top - 23:59:31 up 47 days, 12:30,  4 users,  load average: 1.72, 0.58, 0.56
Tasks:   1 total,   1 running,   0 sleeping,   0 stopped,   0 zombie
%Cpu(s): 80.8 us, 19.2 sy,  0.0 ni,  0.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem :  3882052 total,   463120 free,  2381500 used,  1037432 buff/cache
KiB Swap:        0 total,        0 free,        0 used.  1104724 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  410172 139740 125028 R 16.0  3.6   1:01.93 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT

top - 23:59:32 up 47 days, 12:30,  4 users,  load average: 1.90, 0.63, 0.58
Tasks:   1 total,   0 running,   1 sleeping,   0 stopped,   0 zombie
%Cpu(s): 79.2 us, 19.8 sy,  0.0 ni,  0.0 id,  0.0 wa,  0.0 hi,  1.0 si,  0.0 st
KiB Mem :  3882052 total,   463096 free,  2381364 used,  1037592 buff/cache
KiB Swap:        0 total,        0 free,        0 used.  1104708 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  409836 140176 125464 S 14.0  3.6   1:02.07 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle

top - 23:59:33 up 47 days, 12:30,  4 users,  load average: 1.90, 0.63, 0.58
Tasks:   1 total,   0 running,   1 sleeping,   0 stopped,   0 zombie
%Cpu(s): 58.0 us, 40.0 sy,  0.0 ni,  0.0 id,  0.0 wa,  0.0 hi,  2.0 si,  0.0 st
KiB Mem :  3882052 total,   463096 free,  2381364 used,  1037592 buff/cache
KiB Swap:        0 total,        0 free,        0 used.  1104708 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  409836 140176 125464 S  1.0  3.6   1:02.08 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle

這裡我先解釋一下為什麼記憶體會突然之間降低了這麼多一直沒見恢復的。。。是不是洩露了?
不是的。。因為剛才中間第三個儲存過程報錯了,結果空閒記憶體一下子降了這麼多。。我也很無奈的。

報錯時候執行的在這裡:

procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
 r  b   swpd   free   buff  cache   si   so    bi    bo   in   cs us sy id wa st
 4  0      0   1056      3    407    0    0    13    47    9    0  2  1 96  0  0
 0  0      0   1056      3    407    0    0     0    40  359 1004  1  0 99  0  0
 0  0      0   1056      3    407    0    0     0    68  349  994  2  1 97  0  0
 0  0      0   1056      3    407    0    0     0    40  337  979  1  0 99  0  0
10  5      0    866      4    592    0    0 189020   104 1017 1185 27 10 54  9  0
 4  3      0    494      4    964    0    0 380656    60 1727 1230 72 17  0 11  0
 0  0      0    457      4   1005    0    0 42972    40 1048  957 53  7 35  5  0
 0  0      0    457      4   1005    0    0     0    56  347  987  2  1 97  0  0
 0  0      0    457      4   1005    0    0     0    40  316  949  1  1 98  0  0
 0  0      0    457      4   1005    0    0     0   120  319  958  1  0 99  0  0
 0  0      0    457      4   1005    0    0     0    40  323  969  1  1 98  0  0
 0  0      0    457      4   1005    0    0     0    40  324  956  2  0 98  0  0
 0  0      0    457      4   1005    0    0     0    76  309  947  1  1 98  0  0
 0  0      0    457      4   1005    0    0     0    40  320  955  1  2 97  0  0
 0  0      0    457      4   1005    0    0     0    96  324  963  0  0 100  0  0
 0  0      0    458      4   1005    0    0     0    40  343  978  2  1 97  0  0
 0  0      0    458      4   1005    0    0     0    40  348  973  1  1 98  0  0
 0  0      0    458      4   1005    0    0     0    40  349  989  1  0 99  0  0

從一千多降到460m左右。。不見恢復。。

好了,說說第三次的結論。。
1.7s。。。。額。。是臨時表還有cte的三分一消耗。。而且,記憶體不減,cpu消耗也不高。這是開玩笑嘛?

下面來調整搜尋條件還有先進行一下分析看看結果。
在這裡插入圖片描述

好了,調整後還是分析還是認為1.7s,那麼執行實際情況,監測:

begin;
-- select test_case_search_pager_using_temp_table(
-- select test_case_search_pager_using_temp_table(
select test_case_search_pager_origin(
	7, -- pageIndex
	319, -- pageSize
	null, -- equal id
	1,-- min group id
	11, -- max group id 
	null,-- begin create time
	null,-- end create time
	'outline_crs_temp_table',
	'records_crs_temp_table');
fetch all in outline_crs_temp_table;
fetch all in records_crs_temp_table;
end;

結果:

在這裡插入圖片描述

兩秒鐘!結果還是遠遠低於cte還有臨時表!

其他表現:

 r  b   swpd   free   buff  cache   si   so    bi    bo   in   cs us sy id wa st
 6  0      0    445      7   1011    0    0    13    47    9    1  2  1 96  0  0
 1  0      0    445      7   1011    0    0     0    40 1389 4451 62 38  0  0  0
 3  0      0    445      7   1011    0    0     0   148 1393 4545 58 42  0  0  0
 3  0      0    445      7   1011    0    0     0    40 1383 4555 58 42  0  0  0
 2  0      0    445      7   1011    0    0     0    40 1382 4518 58 42  0  0  0
 4  0      0    440      7   1011    0    0     0   116 1256 2036 79 21  0  0  0
 6  0      0    439      7   1011    0    0     0    40 1250 1078 87 13  0  0  0
 2  0      0    445      7   1011    0    0     0   124 1373 4330 62 38  0  0  0
 6  0      0    445      7   1011    0    0     0    92 1392 4516 59 41  0  0  0
 1  0      0    445      7   1011    0    0     0     0 1370 4343 62 38  0  0  0
 4  0      0    445      7   1011    0    0     0    40 1344 2263 83 17  0  0  0
 3  0      0    446      7   1011    0    0     0    40 1383 4305 57 43  0  0  0
 2  0      0    446      7   1011    0    0     0    40 1383 4315 57 43  0  0  0

top - 00:07:18 up 47 days, 12:38,  4 users,  load average: 4.03, 1.81, 1.09
Tasks:   1 total,   0 running,   1 sleeping,   0 stopped,   0 zombie
%Cpu(s): 58.6 us, 40.4 sy,  0.0 ni,  0.0 id,  0.0 wa,  0.0 hi,  1.0 si,  0.0 st
KiB Mem :  3882052 total,   456592 free,  2382092 used,  1043368 buff/cache
KiB Swap:        0 total,        0 free,        0 used.  1102968 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  410084 141056 125984 S  0.0  3.6   1:02.44 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle

top - 00:07:19 up 47 days, 12:38,  4 users,  load average: 4.03, 1.81, 1.09
Tasks:   1 total,   1 running,   0 sleeping,   0 stopped,   0 zombie
%Cpu(s): 72.3 us, 26.7 sy,  0.0 ni,  0.0 id,  0.0 wa,  0.0 hi,  1.0 si,  0.0 st
KiB Mem :  3882052 total,   456592 free,  2382092 used,  1043368 buff/cache
KiB Swap:        0 total,        0 free,        0 used.  1102968 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  410420 141320 126108 R  8.9  3.6   1:02.53 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT

top - 00:07:20 up 47 days, 12:38,  4 users,  load average: 4.03, 1.81, 1.09
Tasks:   1 total,   1 running,   0 sleeping,   0 stopped,   0 zombie
%Cpu(s): 86.0 us, 14.0 sy,  0.0 ni,  0.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem :  3882052 total,   456592 free,  2382092 used,  1043368 buff/cache
KiB Swap:        0 total,        0 free,        0 used.  1102968 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  410420 141320 126108 R 18.0  3.6   1:02.71 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT

top - 00:07:21 up 47 days, 12:38,  4 users,  load average: 4.03, 1.81, 1.09
Tasks:   1 total,   0 running,   1 sleeping,   0 stopped,   0 zombie
%Cpu(s): 68.6 us, 30.4 sy,  0.0 ni,  0.0 id,  0.0 wa,  0.0 hi,  1.0 si,  0.0 st
KiB Mem :  3882052 total,   456460 free,  2382068 used,  1043524 buff/cache
KiB Swap:        0 total,        0 free,        0 used.  1102844 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
20522 postgres  20   0  410084 141396 126184 S  6.9  3.6   1:02.78 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle

top - 00:07:22 up 47 days, 12:38,  4 users,  load average: 4.43, 1.93, 1.14
Tasks:   1 total,   0 running,   1 sleeping,   0 stopped,   0 zombie
%Cpu(s): 59.6 us, 39.4 sy,  0.0 ni,  0.0 id,  0.0 wa,  0.0 hi,  1.0 si,  0.0 st
KiB Mem :  3882052 total,   456460 free,  2382068 used,  1043524 buff/cache
KiB Swap:        0 total,        0 free,        0 used.  1102844 avail Mem 

記憶體消耗很少,不到20%的cpu利用率。。額,我這種渣機器竟然能夠拖動千萬級別表的除了主鍵以外其他地方沒有加索引的任意條件查詢!
這個簡直是逆天了。

而使用臨時表還有cte的。。。感覺就應了一句話: 特性一時爽,全家火葬場。。。

100w及10w級別的已經不想測試了,因為。。。1000w級別的資料也能查詢這麼快了。。嗯,我何必再跟小兒科斤斤計較。。