使用應用程序(Java/Python)訪問MaxCompute Lightning進行數據開發
很多開發者希望利用Lightning的特性來開發數據應用,本文將結合示例來介紹Java和Python如何連接訪問Lightning進行應用開發(參考時需要替換為您項目所在region的Endpoint及用戶認證信息)。
一、Java使用JDBC訪問Lightning
示例如下:
import java.sql.*;
public class Main {
private static Connection connection; public static void main(String[] args) throws SQLException { String url = "jdbc:postgresql://lightning.cn-shanghai.maxcompute.aliyun.com:443/your_project_name?prepareThreshold=0&sslmode=require"; String accessId = "<your_maxcompute_access_id>"; String accessKey = "<your_maxcompute_access_key>"; String sql = "select * from dual"; try { Connection conn = getCon(url, accessId, accessKey); Statement st = conn.createStatement(); System.out.println("Send Lightning query"); ResultSet rs = st.executeQuery(sql); while (rs.next()) { System.out.println(rs.getString(1)+ "\t"); } System.out.println("End Lightning query"); conn.close(); } catch (SQLException e) { e.printStackTrace(); } } public static Connection getCon(String lightningsHost, String lightningUser, String lightningPwd) { try { if (connection == null || connection.isClosed()) { try { Class.forName("org.postgresql.Driver").newInstance(); DriverManager.setLoginTimeout(1); connection = DriverManager.getConnection(lightningsHost, lightningUser, lightningPwd); } catch (Exception ex) { ex.printStackTrace(); } } } catch (Exception ex) { ex.printStackTrace(); } return connection; }
}
二、Java使用druid訪問Lightning
1.pom依賴
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
<version>1.0.23</version>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>9.3-1101-jdbc4</version>
2.spring配置
<bean id="LightningDataSource" class="com.alibaba.druid.pool.DruidDataSource" init-method="init" destroy-method="close"> <property name="url" value="jdbc:postgresql://lightning.cn-shanghai.maxcompute.aliyun.com:443/project_name?prepareThreshold=0&sslmode=require”/> <!--替換成自己project所在region的Endpoint—> <property name="username" value=“訪問用戶的Access Key ID"/> <property name="password" value="訪問用戶的Access Key Secret"/> <property name="driverClassName" value="org.postgresql.Driver"/> <property name="dbType" value="postgresql"/> <property name="initialSize" value="1" /> <property name="minIdle" value="1" /> <property name="maxActive" value="5" /> <!—Lightning服務每個project的連接數限制20,所以不要配置過大,按需配置,否則容易出現query_wait_timeout錯誤 --> <!--以下兩個配置,檢測連接有效性,修復偶爾出現create connection holder error錯誤 --> <property name="testWhileIdle" value="true" /> <property name="validationQuery" value="SELECT 1" /> </bean>
<bean class="com.xxx.xxx.LightningProvider">
<property name="druidDataSource" ref="LightningDataSource"/>
</bean>
3.代碼訪問
public class LightningProvider {
DruidDataSource druidDataSource;
/**
* 執行sql
* @param sql
* @return
* @throws Exception
*/
public void execute(String sql) throws SQLException {
DruidPooledConnection connection = null ;
Statement st = null;
try{
connection = druidDataSource.getConnection();
st = connection.createStatement();
ResultSet resultSet = st.executeQuery(sql);
//對返回值的解析和處理的代碼
//按行處理,每行的數據放到一個map中
ResultSetMetaData metaData = resultSet.getMetaData();
int columnCount = metaData.getColumnCount();
List<LinkedHashMap> rows = Lists.newArrayList();
while(resultSet.next()){
LinkedHashMap map = Maps.newLinkedHashMap();
for(int i=1;i<=columnCount;i++){
String label = resultSet.getMetaData().getColumnLabel(i);
map.put(label,resultSet.getString(i));
}
rows.add(map);
}
}catch (Exception e){
e.printStackTrace();
}finally {
try {
if(st!=null) {
st.close();
}
} catch (SQLException e) {
e.printStackTrace();
}
try {
if(connection!=null) {
connection.close();
}
} catch (SQLException e) {
e.printStackTrace();
}
}
}
}
三、Python使用pyscopg2訪問Lightning
示例如下:
#!/usr/bin/env python
coding=utf-8
import psycopg2
import sys
def query_lightning(lightning_conf, sql):
"""Query data through Lightning by sql
Args:
lightning_conf: a map contains settings of ‘dbname‘, ‘user‘, ‘password‘, ‘host‘, ‘port‘
sql: query submit to Lightning
Returns:
result: the query result in format of list of rows
"""
result = None
conn = None
conn_str = None
try:
conn_str = ("dbname={dbname} "
"user={user} "
"password={password} "
"host={host} "
"port={port}").format(**lightning_conf)
except Exception, e:
print >> sys.stderr, ("Invalid Lightning‘ configuration "
"{}".format(e))
sys.exit(1)
try:
conn = psycopg2.connect(conn_str)
conn.set_session(autocommit=True) # This will disable transaction
# started with keyword BEGIN,
# which is currently not
# supported by Lightning’ public service
cur = conn.cursor()
# execute Lightning‘ query
cur.execute(sql)
# get result
result = cur.fetchall()
except Exception, e:
print >> sys.stderr, ("Failed to query data through "
"Lightning: {}".format(e))
finally:
if conn:
conn.close()
return result
if name == "main":
step1. setup configuration
lightning_conf = {
"dbname": “your_project_name”,
"user": "<your_maxcompute_access_id>",
"password": "<your_maxcompute_access_key>",
"host": "lightning.cn-shanghai.maxcompute.aliyun.com", #your region lightning endpoint
"port": 443
}
# step2. issue a query
result = query_lightning(lightning_conf, "select * from test”)
# step3. print result
if result:
for i in xrange(0, len(result)):
print "Got %d row from Lightning:%s" % (i + 1, result[i])
四、Python使用ODBC訪問Lightning
您需要現在電腦上安裝並和配置odbc驅動。代碼示例如下:
import pyodbc
conn_str = (
"DRIVER={PostgreSQL Unicode};"
"DATABASE=your_project_name;"
"UID=your_maxcompute_access_id;"
"PWD=your_maxcompute_access_key;"
"SERVER=lightning.cn-shanghai.maxcompute.aliyun.com;" #your region lightning endpoint
"PORT=443;"
)
conn = pyodbc.connect(conn_str)
crsr = conn.execute("SELECT * from test”)
row = crsr.fetchone()
print(row)
crsr.close()
conn.close()
由於Lightning提供了PostgreSQL兼容的接口,您可以像開發PostgreSQL的應用一樣開發Lightning應用程序。
使用應用程序(Java/Python)訪問MaxCompute Lightning進行數據開發