Java 讀取任意shapefile的所有欄位，並插入到MongoDB資料庫（Spring Boot）
阿新 • • 發佈：2018-11-20
文章目錄
Java 讀取任意shapefile的所有欄位，並插入到MongoDB資料庫（Spring Boot）
- 亮點：動態獲取欄位資訊，動態建立資料庫表資訊，支援任意shapefile資料
- pom
<repositories>
<repository>
<id>maven2-repository.dev.java.net</id>
<name>Java.net repository</name>
<url>http://download.java.net/maven/2</url>
</ <dependency>
<groupId>org.geotools</groupId>
<artifactId>gt-shapefile</artifactId>
<version>18.4</version>
</dependency>
<dependency>
<groupId>org.geotools</groupId>
<artifactId>gt-geojson</artifactId>
<version>18.4</version>
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongodb-driver</artifactId>
<version>3.6.2</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.8.2</version>
</dependency>
1. 統一返回結果封裝
/**
* @program: tool
* @description:
* @author: Mr.superbeyone
* @create: 2018-10-18 16:37
**/
@Data
public class JsonResult<T> {
private Integer code;
private String msg;
private T data;
//使用 lombok @Data註解 或者 Get Set方法 ...
}
2. shp檔案資料實體封裝
/**
* @program: tool
* @description:
* @author: Mr.superbeyone
* @create: 2018-10-16 12:12
**/
@Data
public class ShapeModel implements Serializable {
private String id;
private String type;
private String geometry;
private String properties;
//使用 lombok @Data註解 或者 Get Set方法 ...
}
3. 核心程式碼
/**
* 讀取shp檔案
*
* @param filePath 讀取的檔案路徑
* @param collection mongoDB collection
* @return
*/
private JsonResult<Set<String>> readShapeFile(String filePath, MongoCollection<Document> collection) {
JsonResult<Set<String>> result = new JsonResult<>();
Set<String> set = new HashSet<>();
File folder = new File(filePath);
if (!folder.isDirectory()) {
if (folder.toString().endsWith(".shp")) {
try {
return getShapeFile(folder, collection);
} catch (Exception e) {
e.printStackTrace();
}
} else {
System.out.println("選擇的檔案字尾名不是.shp");
return null;
}
} else {
File[] files = folder.listFiles();
if (files.length <= 0) {
System.out.println("目錄檔案為空");
return null;
}
for (File file : files) {
if (!file.toString().endsWith(".shp")) {
continue;
}
try {
result = getShapeFile(file, collection);
Set fields = result.getData();
set.addAll(fields);
} catch (Exception e) {
e.printStackTrace();
}
}
result.setData(set);
}
return result;
}
private JsonResult<Set<String>> getShapeFile(File file, MongoCollection<Document> collection) {
JsonResult<Set<String>> result = new JsonResult();
HashSet<String> fields = new HashSet<>();
int number = 0;
int index = 0;
try {
ShapefileDataStore shapefileDataStore = new ShapefileDataStore(file.toURI().toURL());
shapefileDataStore.setCharset(Charset.forName("utf-8"));
ContentFeatureSource featureSource = shapefileDataStore.getFeatureSource();
SimpleFeatureIterator featureIterator = featureSource.getFeatures().features();
ArrayList<Document> documents = new ArrayList<>();
Document document = null;
while (featureIterator.hasNext()) {
number++;
SimpleFeature feature = featureIterator.next();
Collection<? extends Property> properties = feature.getValue();
//feature轉GeoJSON
FeatureJSON featureJSON = new FeatureJSON();
StringWriter writer = new StringWriter();
featureJSON.writeFeature(feature, writer);
//TODO: 判斷writer.toString().length() 否則可能會拋下列異常
//org.bson.BsonSerializationException: Payload document size of is larger than maximum of 16777216
ShapeModel shapeModel = JSON.parseObject(writer.toString(), ShapeModel.class);
document = new Document();
if (properties.size() < 0) {
continue;
}
if (index == 0) {
for (Property property : properties) {
if (property.getType().getClass() == org.geotools.feature.type.AttributeTypeImpl.class) {
fields.add(property.getName().toString());
} else {
//空間欄位
fields.add("#" + property.getName() + "#");
}
}
result.setData(fields);
result.setCode(ResultCodeEnum.GET_UPDATE_FILE_FIELDS_SUCCESS.getCode());
result.setMsg(ResultCodeEnum.GET_UPDATE_FILE_FIELDS_SUCCESS.getMsg());
//TODO:建立分片資料庫
//createShardDatabase(fields);
}
for (Property property : properties) {
if (property.getType().getClass() == org.geotools.feature.type.AttributeTypeImpl.class) {
if (property.getValue() == null) {
document.put(property.getName().toString(), "");
} else {
document.put(property.getName().toString(), property.getValue().toString());
}
} else {
//空間欄位
document.put(property.getName().toString(), shapeModel.getGeometry());
}
}
index++;
//逐條插入
//collection.insertOne(document);
documents.add(document);
//每1000條資料插入一次
if (number % 1000 == 0) {
collection.insertMany(documents);
documents = new ArrayList<>();
}
//最後插入不足1000條的資料
if (!featureIterator.hasNext()) {
collection.insertMany(documents);
}
}
} catch (Exception e) {
//出現異常
result.setMsg("插入第" + (index + 1) + "條時出現錯誤");
result.setCode(4000);
} finally {
featureIterator.close();
}
return result;
}
}
3. 可能出現的異常
org.bson.BsonSerializationException: Payload document size of is larger than maximum of 16777216
3.1 異常產生原因
- 當前插入的資料大於16M，會丟擲此異常
3.2 解決方案
- 判斷所要插入的資料的大小，如果大於16M，可以選擇跳過，或者停止插入
4 擴充套件
4.1 建立資料庫分片
public void createShardDatabase(Set<String> fields) {
//TODO:使用 databaseName 生成策略
String databaseName = "super";
//TODO:使用 collectionName 生成策略
String collectionName = "beyone";
List<String> set = new ArrayList<>();
Map<String, Object> map = new HashMap<String, Object>();
if (fields.size() < 0) {
return;
}
for (String field : fields) {
//空間欄位
if (field.startsWith("#") && field.endsWith("#")) {
String splitField = field.split("#")[1];
map.put(splitField, "hashed");
} else {
set.add(field);
}
}
//mongoDBConfig 是通過注入獲得的
/**
* @Autowired
* MongoDBConfig mongoDBConfig;
*/
MongoClient client = new MongoClient(mongoDBConfig.getHost(), mongoDBConfig.getPort());
MongoDatabase adminDB = client.getDatabase(mongoDBConfig.getDatabase());
Document enableShardDoc = new Document();
enableShardDoc.put("enablesharding", databaseName);
Document result = adminDB.runCommand(enableShardDoc);
Object obj = result.get("ok");
if (null == obj) {
}
if ("1.0".equals(obj.toString())) {
Document shardKeyDoc = new Document();
String coll = databaseName + "." + collectionName;
shardKeyDoc.put("shardcollection", coll);
Document t = new Document();
t.putAll(map);
shardKeyDoc.put("key", t);
result = adminDB.runCommand(shardKeyDoc);
obj = result.get("ok");
if (null == obj) {
//logger.info("error");
}
if ("1.0".equals(obj.toString())) {
//logger.info("success");
} else {
//logger.info("error");
}
} else {
//logger.info("error");
}
}