Hinton關於RBM的程式碼註解之（二）backpropclassify.m
阿新 • • 發佈：2019-01-31
原始碼：http://www.cs.toronto.edu/~hinton/MatlabForSciencePaper.html
這個是關於RBM的微調步驟的程式碼，代價函式是交叉熵
% Version 1.000
%% Code provided by Ruslan Salakhutdinov and Geoff Hinton
%
% Permission is granted for anyone to copy, use, modify, or distribute this
% program and accompanying programs and documents for any purpose, provided
% this copyright notice is retained and prominently displayed, along with
% a note saying that the original programs are available from our
% web page.
% The programs and documents are distributed without any warranty, express or
% implied. As the programs were written for research purposes only, they have
% not been tested to the degree that would be advisable in any important
% application. All use of these programs is entirely at the user's own risk.
% This program fine-tunes an autoencoder with backpropagation.
% Weights of the autoencoder are going to be saved in mnist_weights.mat
% and trainig and test reconstruction errors in mnist_error.mat
% You can also set maxepoch, default value is 200 as in our paper.
maxepoch=200;
fprintf(1,'\nTraining discriminative model on MNIST by minimizing cross entropy error. \n'); %最小化交叉熵
fprintf(1,'60 batches of 1000 cases each. \n');
load mnistvhclassify
load mnisthpclassify
load mnisthp2classify
makebatches; %分包
[numcases numdims numbatches]=size(batchdata);
N=numcases; %每一個包的大小
%%%% PREINITIALIZE WEIGHTS OF THE DISCRIMINATIVE MODEL%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
w1=[vishid; hidrecbiases]; %對應的權重引數及第二層的偏置，矩陣大小是（N+1）*hidden
w2=[hidpen; penrecbiases];
w3=[hidpen2; penrecbiases2];
w_class = 0.1*randn(size(w3,2)+1,10); %隨機生成第四層的列數+1行，10列的矩陣，不同的是這個是exp,,上面的是logistic的;而在backprop中這個是線性的
%%%%%%%%%% END OF PREINITIALIZATIO OF WEIGHTS %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
l1=size(w1,1)-1; %這是輸入可見層的number 28*28
l2=size(w2,1)-1; %第一層隱含層的number 500
l3=size(w3,1)-1; %第二層隱含層的number 500
l4=size(w_class,1)-1; %第三層隱含層的number 2000
l5=10; %label層單元個數
test_err=[];
train_err=[];
for epoch = 1:maxepoch %總共maxepoch200次BP更新引數
%%%%%%%%%%%%%%%%%%%% COMPUTE TRAINING MISCLASSIFICATION ERROR %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
err=0;
err_cr=0;
counter=0;
[numcases numdims numbatches]=size(batchdata);
N=numcases;
for batch = 1:numbatches
data = [batchdata(:,:,batch)];
target = [batchtargets(:,:,batch)];
data = [data ones(N,1)]; %在原始資料後新增一列資料. %加一列是因為偏置，原本是bj+ w1*data，現在變化為[data 1]*[w1;bj]
w1probs = 1./(1 + exp(-data*w1)); w1probs = [w1probs ones(N,1)]; %計算各隱含層為1的概率值
w2probs = 1./(1 + exp(-w1probs*w2)); w2probs = [w2probs ones(N,1)];
w3probs = 1./(1 + exp(-w2probs*w3)); w3probs = [w3probs ones(N,1)];
targetout = exp(w3probs*w_class); %計算最後的輸出值 N行10列 ;這裡不是LOGISTIC的，上面都是logistics
targetout = targetout./repmat(sum(targetout,2),1,10);
[I J]=max(targetout,[],2); %取計算結果每行中的最大值及列標
[I1 J1]=max(target,[],2); %取原先設定目標值的最大值及列標
counter=counter+length(find(J==J1)); %統計正確估計的數目
err_cr = err_cr- sum(sum( target(:,1:end).*log(targetout))) ;%重構的交叉熵的錯誤代價函式
end
train_err(epoch)=(numcases*numbatches-counter);%平均訓練的錯誤次數
train_crerr(epoch)=err_cr/numbatches; %平均每一批的代價函式
%%%%%%%%%%%%%% END OF COMPUTING TRAINING MISCLASSIFICATION ERROR %%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%% COMPUTE TEST MISCLASSIFICATION ERROR %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%同上，自己對應
err=0;err_cr=0;
counter=0;
[testnumcases testnumdims testnumbatches]=size(testbatchdata);
N=testnumcases;
for batch = 1:testnumbatches
data = [testbatchdata(:,:,batch)];
target = [testbatchtargets(:,:,batch)];
data = [data ones(N,1)];
w1probs = 1./(1 + exp(-data*w1)); w1probs = [w1probs ones(N,1)];
w2probs = 1./(1 + exp(-w1probs*w2)); w2probs = [w2probs ones(N,1)];
w3probs = 1./(1 + exp(-w2probs*w3)); w3probs = [w3probs ones(N,1)];
targetout = exp(w3probs*w_class);
targetout = targetout./repmat(sum(targetout,2),1,10);
[I J]=max(targetout,[],2);
[I1 J1]=max(target,[],2);
counter=counter+length(find(J==J1));
err_cr = err_cr- sum(sum( target(:,1:end).*log(targetout))) ;
end
test_err(epoch)=(testnumcases*testnumbatches-counter);
test_crerr(epoch)=err_cr/testnumbatches;
fprintf(1,'Before epoch %d Train # misclassified: %d (from %d). Test # misclassified: %d (from %d) \t \t \n',...
epoch,train_err(epoch),numcases*numbatches,test_err(epoch),testnumcases*testnumbatches);
%%%%%%%%%%%%%% END OF COMPUTING TEST MISCLASSIFICATION ERROR %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
tt=0;
for batch = 1:numbatches/10 %進入微調，先組合10個小批次，原來是100個小批次的
fprintf(1,'epoch %d batch %d\r',epoch,batch);
%%%%%%%%%%% COMBINE 10 MINIBATCHES INTO 1 LARGER MINIBATCH %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%分包，分成1個小包
tt=tt+1;data=[];
targets=[];
for kk=1:10
data=[data
batchdata(:,:,(tt-1)*10+kk)];
targets=[targets
batchtargets(:,:,(tt-1)*10+kk)];
end
%%%%%%%%%%%%%%% PERFORM CONJUGATE GRADIENT WITH 3 LINESEARCHES %%%%%%%%%%%%%%%%%%%%%%%%%%%%%
max_iter=3; %最多三次線性搜尋
if epoch<6 % First update top-level weights holding other weights fixed. 前5次最先更新頂層權重
N = size(data,1);
XX = [data ones(N,1)];
w1probs = 1./(1 + exp(-XX*w1)); w1probs = [w1probs ones(N,1)];
w2probs = 1./(1 + exp(-w1probs*w2)); w2probs = [w2probs ones(N,1)];
w3probs = 1./(1 + exp(-w2probs*w3)); %w3probs = [w3probs ones(N,1)];
VV = [w_class(:)']';%將w_class展開成一列，其實就是最後一層的權重，把它作為共軛梯度函式的變數
Dim = [l4; l5]; %記錄最後兩層的單元節點個數，注L4，L5，代表的值看前面
[X, fX] = minimize(VV,'CG_CLASSIFY_INIT',max_iter,Dim,w3probs,targets);
%BP反向傳播，計算出使用該函式經過最多3次搜尋得到的更新後的權重X，及其梯度fx
w_class = reshape(X,l4+1,l5); %使Xreshape為原來的矩陣大小
else %更新全部權重
VV = [w1(:)' w2(:)' w3(:)' w_class(:)']'; %所有的引數組合成一列，作為共軛梯度的變數
Dim = [l1; l2; l3; l4; l5];
[X, fX] = minimize(VV,'CG_CLASSIFY',max_iter,Dim,data,targets);
%使用GC_CLASSIFY函式作為代價函式，實際上就是上面提到的交叉熵
%以下是將得到的權重更新X轉化會各個對應的原來的矩陣
w1 = reshape(X(1:(l1+1)*l2),l1+1,l2);
xxx = (l1+1)*l2;
w2 = reshape(X(xxx+1:xxx+(l2+1)*l3),l2+1,l3);
xxx = xxx+(l2+1)*l3;
w3 = reshape(X(xxx+1:xxx+(l3+1)*l4),l3+1,l4);
xxx = xxx+(l3+1)*l4;
w_class = reshape(X(xxx+1:xxx+(l4+1)*l5),l4+1,l5);
end
%%%%%%%%%%%%%%% END OF CONJUGATE GRADIENT WITH 3 LINESEARCHES %%%%%%%%%%%%%%%%%%%%%%%%%%%%%
end
save mnistclassify_weights w1 w2 w3 w_class
save mnistclassify_error test_err test_crerr train_err train_crerr;
end
原文地址：http://blog.csdn.net/u014537068/article/details/44925437