Lucene6入門教程（七）多目錄多執行緒索引/檢索
阿新 • • 發佈：2019-01-15
在使用lucene6 的時候，會遇到多目錄多執行緒的索引構建和檢索（查詢）情況。
把不同型別的資料放到同一個索引檔案中，這是不現實的，而且，除了記憶體索引，建立多目錄索引也能提高索引速度，這是因為，一個索引目錄只能有一把鎖，那如果有多個索引目錄，每個索引目錄發一把鎖，N個索引目錄同時進行索引寫操作就有意義了。
而檢索的時候，也需要多執行緒併發來提高搜尋速度。
下邊是程式碼，首先是LuceneUtils工具類，封裝各個物件，是為防止Index和Search過程時候的讀衝突或寫衝突等做準備的。
import java.io.IOException;
import java.nio.file.Paths;
import 然後是一個LuceneManager類
import java.io.IOException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockObtainFailedException;
/**
* Lucene索引讀寫器/查詢器單例獲取工具類
* @author Lanxiaowei
*
*/
public class LuceneManager {
private volatile static LuceneManager singleton; private volatile static IndexWriter writer; private volatile static IndexReader reader;
private volatile static IndexSearcher searcher;
private final Lock writerLock = new ReentrantLock();
//private final Lock readerLock = new ReentrantLock();
//private final Lock searcherLock = new ReentrantLock();
private static ThreadLocal<IndexWriter> writerLocal = new ThreadLocal<IndexWriter>();
private LuceneManager() {}
public static LuceneManager getInstance() {
if (null == singleton) {
synchronized (LuceneManager.class) {
if (null == singleton) {
singleton = new LuceneManager();
}
}
}
return singleton;
}
/**
* 獲取IndexWriter單例物件
* @param dir
* @param config
* @return
*/
public IndexWriter getIndexWriter(Directory dir, IndexWriterConfig config) {
if(null == dir) {
throw new IllegalArgumentException("Directory can not be null.");
}
if(null == config) {
throw new IllegalArgumentException("IndexWriterConfig can not be null.");
}
try {
writerLock.lock();
writer = writerLocal.get();
if(null != writer) {
return writer;
}
if(null == writer){ //如果索引目錄被鎖，則直接拋異常
if(IndexWriter.isLocked(dir)) {
throw new LockObtainFailedException("Directory of index had been locked.");
}
writer = new IndexWriter(dir, config);
writerLocal.set(writer);
}
} catch (LockObtainFailedException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
writerLock.unlock();
}
return writer;
}
/**
* 獲取IndexWriter[可能為Null]
* @return
*/
public IndexWriter getIndexWriter() {
return writer;
}
/**
* 獲取IndexReader物件
* @param dir
* @param enableNRTReader 是否開啟NRTReader
* @return
*/
public IndexReader getIndexReader(Directory dir,boolean enableNRTReader) {
if(null == dir) {
throw new IllegalArgumentException("Directory can not be null.");
}
try {
if(null == reader){
reader = DirectoryReader.open(dir);
} else {
if(enableNRTReader && reader instanceof DirectoryReader) {
//開啟近實時Reader,能立即看到動態新增/刪除的索引變化
reader = DirectoryReader.openIfChanged((DirectoryReader)reader);
}
}
} catch (IOException e) {
e.printStackTrace();
}
return reader;
}
/**
* 獲取IndexReader物件(預設不啟用NETReader)
* @param dir
* @return
*/
public IndexReader getIndexReader(Directory dir) {
return getIndexReader(dir, false);
}
/**
* 獲取IndexSearcher物件
* @param reader IndexReader物件例項
* @param executor 如果你需要開啟多執行緒查詢，請提供ExecutorService物件引數
* @return
*/
public IndexSearcher getIndexSearcher(IndexReader reader,ExecutorService executor) {
if(null == reader) {
throw new IllegalArgumentException("The indexReader can not be null.");
}
if(null == searcher){
searcher = new IndexSearcher(reader);
}
return searcher;
}
/**
* 獲取IndexSearcher物件(不支援多執行緒查詢)
* @param reader IndexReader物件例項
* @return
*/
public IndexSearcher getIndexSearcher(IndexReader reader) {
return getIndexSearcher(reader, null);
}
/**
* 關閉IndexWriter
* @param writer
*/
public void closeIndexWriter(IndexWriter writer) {
if(null != writer) {
try {
writer.close();
writer = null;
writerLocal.remove();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
多執行緒索引的類IndexCreator，防止寫衝突等
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.concurrent.CountDownLatch;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.*;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.FSDirectory;
/**
* 索引建立執行緒
* @author Lanxiaowei
*
*/
public class IndexCreator implements Runnable {
/**需要讀取的檔案存放目錄*/
private String docPath;
/**索引檔案存放目錄*/
private String luceneDir;
private int threadCount;
private final CountDownLatch countDownLatch1;
private final CountDownLatch countDownLatch2;
public IndexCreator(String docPath, String luceneDir,int threadCount,CountDownLatch countDownLatch1,CountDownLatch countDownLatch2) {
super();
this.docPath = docPath;
this.luceneDir = luceneDir;
this.threadCount = threadCount;
this.countDownLatch1 = countDownLatch1;
this.countDownLatch2 = countDownLatch2;
}
public void run() {
IndexWriter writer = null;
try {
countDownLatch1.await();
Analyzer analyzer = LuceneUtils.analyzer;
FSDirectory directory = LuceneUtils.openFSDirectory(luceneDir);
IndexWriterConfig config = new IndexWriterConfig(analyzer);
config.setOpenMode(OpenMode.CREATE_OR_APPEND);
writer = LuceneUtils.getIndexWriter(directory, config);
try {
indexDocs(writer, Paths.get(docPath));
} catch (IOException e) {
e.printStackTrace();
}
} catch (InterruptedException e1) {
e1.printStackTrace();
} finally {
LuceneUtils.closeIndexWriter(writer);
countDownLatch2.countDown();
}
}
/**
*
* @param writer
* 索引寫入器
* @param path
* 檔案路徑
* @throws IOException
*/
public static void indexDocs(final IndexWriter writer, Path path)
throws IOException {
// 如果是目錄，查詢目錄下的檔案
if (Files.isDirectory(path, new LinkOption[0])) {
System.out.println("directory");
Files.walkFileTree(path, new SimpleFileVisitor() {
@Override
public FileVisitResult visitFile(Object file,
BasicFileAttributes attrs) throws IOException {
Path path = (Path)file;
System.out.println(path.getFileName());
indexDoc(writer, path, attrs.lastModifiedTime().toMillis());
return FileVisitResult.CONTINUE;
}
});
} else {
indexDoc(writer, path,
Files.getLastModifiedTime(path, new LinkOption[0])
.toMillis());
}
}
/**
* 讀取檔案建立索引
*
* @param writer
* 索引寫入器
* @param file
* 檔案路徑
* @param lastModified
* 檔案最後一次修改時間
* @throws IOException
*/
public static void indexDoc(IndexWriter writer, Path file, long lastModified)
throws IOException {
InputStream stream = Files.newInputStream(file, new OpenOption[0]);
Document doc = new Document();
Field pathField = new StringField("path", file.toString(),
Field.Store.YES);
doc.add(pathField);
doc.add(new NumericDocValuesField("modified", lastModified));
doc.add(new LongPoint("modified", lastModified));
doc.add(new StoredField("modified", lastModified));
doc.add(new TextField("contents",intputStream2String(stream),Field.Store.YES));
//doc.add(new TextField("contents", new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))));
if (writer.getConfig().getOpenMode() == IndexWriterConfig.OpenMode.CREATE) {
System.out.println("adding " + file);
writer.addDocument(doc);
} else {
System.out.println("updating " + file);
writer.updateDocument(new Term("path", file.toString()), doc);
}
writer.commit();
}
/**
* InputStream轉換成String
* @param is 輸入流物件
* @return
*/
private static String intputStream2String(InputStream is) {
BufferedReader bufferReader = null;
StringBuilder stringBuilder = new StringBuilder();
String line;
try {
bufferReader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8));
while ((line = bufferReader.readLine()) != null) {
stringBuilder.append(line + "\r\n");
}
} catch (IOException e) {
e.printStackTrace();
} finally {
if (bufferReader != null) {
try {
bufferReader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return stringBuilder.toString();
}
}接著便是分頁的類
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.apache.lucene.document.Document;
import org.apache.lucene.search.ScoreDoc;
public class Page<T> {
/**當前第幾頁(從1開始計算)*/
private int currentPage;
/**每頁顯示幾條*/
private int pageSize;
/**總記錄數*/
private int totalRecord;
/**總頁數*/
private int totalPage;
/**分頁資料集合[用泛型T來限定集合元素型別]*/
private Collection<T> items;
/**當前顯示起始索引(從零開始計算)*/
private int startIndex;
/**當前顯示結束索引(從零開始計算)*/
private int endIndex;
/**一組最多顯示幾個頁碼[比如Google一組最多顯示10個頁碼]*/
private int groupSize;
/**左邊偏移量*/
private int leftOffset = 5;
/**右邊偏移量*/
private int rightOffset = 4;
/**當前頁碼範圍*/
private String[] pageRange;
/**分頁資料*/
private List<Document> docList;
/**上一頁最後一個ScoreDoc物件*/
private ScoreDoc afterDoc;
/**上一頁最後一個ScoreDoc物件的Document物件ID*/
private int afterDocId;
public void setRangeIndex() {
int groupSize = getGroupSize();
int totalPage = getTotalPage();
if(totalPage < 2) {
startIndex = 0;
endIndex = totalPage - startIndex;
} else {
int currentPage = getCurrentPage();
if(groupSize >= totalPage) {
startIndex = 0;
endIndex = totalPage - startIndex - 1;
} else {
int leftOffset = getLeftOffset();
int middleOffset = getMiddleOffset();
if(-1 == middleOffset) {
startIndex = 0;
endIndex = groupSize - 1;
} else if(currentPage <= leftOffset) {
startIndex = 0;
endIndex = groupSize - 1;
} else {
startIndex = currentPage - leftOffset - 1;
if(currentPage + rightOffset > totalPage) {
endIndex = totalPage - 1;
} else {
endIndex = currentPage + rightOffset - 1;
}
}
}
}
}
public int getCurrentPage() {
if(currentPage <= 0) {
currentPage = 1;
} else {
int totalPage = getTotalPage();
if(totalPage > 0 && currentPage > getTotalPage()) {
currentPage = totalPage;
}
}
return currentPage;
}
public void setCurrentPage(int currentPage) {
this.currentPage = currentPage;
}
public int getPageSize() {
if(pageSize <= 0) {
pageSize = 10;
}
return pageSize;
}
public void setPageSize(int pageSize) {
this.pageSize = pageSize;
}
public int getTotalRecord() {
return totalRecord;
}
public void setTotalRecord(int totalRecord) {
this.totalRecord = totalRecord;
}
public int getTotalPage() {
int totalRecord = getTotalRecord();
if(totalRecord == 0) {
totalPage = 0;
} else {
int pageSize = getPageSize();
totalPage = totalRecord % pageSize == 0 ? totalRecord / pageSize : (totalRecord / pageSize) + 1;
}
return totalPage;
}
public void setTotalPage(int totalPage) {
this.totalPage = totalPage;
}
public int getStartIndex() {
return startIndex;
}
public void setStartIndex(int startIndex) {
this.startIndex = startIndex;
}
public int getEndIndex() {
return endIndex;
}
public void setEndIndex(int endIndex) {
this.endIndex = endIndex;
}
public int getGroupSize() {
if(groupSize <= 0) {
groupSize = 10;
}
return groupSize;
}
public void setGroupSize(int groupSize) {
this.groupSize = groupSize;
}
public int getLeftOffset() {
leftOffset = getGroupSize() / 2;
return leftOffset;
}
public void setLeftOffset(int leftOffset) {
this.leftOffset = leftOffset;
}
public int getRightOffset() {
int groupSize = getGroupSize();
if(groupSize % 2 == 0) {
rightOffset = (groupSize / 2) - 1;
} else {
rightOffset = groupSize / 2;
}
return rightOffset;
}
public void setRightOffset(int rightOffset) {
this.rightOffset = rightOffset;
}
/**中心位置索引[從1開始計算]*/
public int getMiddleOffset() {
int groupSize = getGroupSize();
int totalPage = getTotalPage();
if(groupSize >= totalPage) {
return -1;
}
return getLeftOffset() + 1;
}
public String[] getPageRange() {
setRangeIndex();
int size = endIndex - startIndex + 1;
if(size <= 0) {
return new String[0];
}
if(totalPage == 1) {
return new String[] {"1"};
}
pageRange = new String[size];
for(int i=0; i < size; i++) {
pageRange[i] = (startIndex + i + 1) + "";
}
return pageRange;
}
public void setPageRange(String[] pageRange) {
this.pageRange = pageRange;
}
public Collection<T> getItems() {
return items;
}
public void setItems(Collection<T> items) {
this.items = items;
}
public List<Document> getDocList() {
return docList;
}
public void setDocList(List<Document> docList) {
this.docList = docList;
}
public ScoreDoc getAfterDoc() {
setAfterDocId(afterDocId);
return afterDoc;
}
public void setAfterDoc(ScoreDoc afterDoc) {
this.afterDoc = afterDoc;
}
public int getAfterDocId() {
return afterDocId;
}
public void setAfterDocId(int afterDocId) {
this.afterDocId = afterDocId;
if(null == afterDoc) {
this.afterDoc = new ScoreDoc(afterDocId, 1.0f);
}
}
public Page() {}
public Page(int currentPage, int pageSize) {
this.currentPage = currentPage;
this.pageSize = pageSize;
}
public Page(int currentPage, int pageSize, Collection<T> items) {
this.currentPage = currentPage;
this.pageSize = pageSi