利用 Go 實現簡單程式替代 NFS 這個古董
相信現在稍微能說得過去的網站，後臺伺服器至少有倆臺，所以在面對使用者檔案上傳等功能的處理上都藉助了類似nfs、fastdfs等網路檔案系統來解決這類問題。單位之前一直在使用nfs，因為它足夠簡單有效。但是前段時間安全測發來一個測試報告，需要升級nfs才能解決。因此做了一波升級,開發運維都做確實有點累的。後來有天休假，安全側的哥們在處理故障的時候，修改完配置後之前重啟了伺服器(也是醉了)，導致很多無服務上掛載的nfs都無效了，然後產生了連鎖反應導致核心業務無法正常使用，然後悲催我剛和家人點的一桌菜都沒上我就去救火了。
事後重新梳理了下業務流程，然後決定放棄nfs和想依賴的一些業務任務。打算用go重寫一個類似的功能以方便所以人都零基礎維護。因此寫了GoUploadRysnc，其實原來很簡單，當用戶上傳檔案的時候有java和python做完處理校驗後以http的放上上傳到Go中,Go中在指定伺服器上儲存後並返回給Java和Python，同時利用Go的協程同步到其他伺服器上。然後在這些儲存檔案的伺服器上進行後續的業務任務。
使用方法
編譯 通過buildXXX方式即可編譯相應平臺的可執行檔案
配置 配置使用json方式，簡單明瞭
{
{
//配置監地址和埠
"addr" : "0.0.0.0:9090" ,
//檔案儲存路徑
"path" : "c:/var/upload/wwww/" ,
//檔名長度
"fileNameLength" : 11 ,
"rysncAddr" :[
//同步地址
"http://localhost:9091/rsync"
]
}
啟動
UploadRysnc -conf conf/server1.conf > run.log
執行
下面是執行部分執行日誌
2019/03/17 10:40:00 Server is starting:0.0.0.0:9090
2019/03/17 10:40:00 Server UploadPath:c:/var/upload/wwww/
2019/03/17 10:40:00 Server Rysnc Addr:http://localhost:9091/rsync
2019/03/17 10:40:10 [::1]:49743 uploadfile [server1.conf][server1.conf] > c:/var/upload/wwww/banner/LwhSfU1nh6w.conf
2019/03/17 10:40:31 Clientrsyncfile Error http://localhost:9091/rsync c:/var/upload/wwww/banner/LwhSfU1nh6w.conf
GO程式碼實現
因為英語比較渣的很，很多gg的官方文件看懂，因此go也是學的很凌亂，程式碼比較凌亂，歡迎往死裡拍。
GitHub地址: https://github.com/0opslab/GoUploadRysnc
package main
//
// @instruction
// 利用go實現的HTTP版的檔案上傳，上傳介面以json方式返回
// @上傳方式
// 通用的http檔案上傳方式
// @實現原理
//
// @配置json
// {
// "addr":"0.0.0.0:9090",
// "path":"c:/var/upload/wwww/",
// "fileNameLength":11,
// "rysncAddr":[
// "http://localhost:9091/rsync",
// "http://localhost:9092/rsync"
// ]
// }
// @說明
// 到新建資料夾下的可以同http head欄位path新增目錄(目錄名需要BASE64)
import (
"bytes"
"crypto/rand"
"encoding/base64"
"encoding/json"
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"mime/multipart"
"net/http"
"os"
"path"
"path/filepath"
"regexp"
"strings"
)
type ServerConfig struct {
//監聽地址和埠
ADDR string `json:'ADDR'`
//檔案寫入路徑
PATH string `json:'PATH'`
//檔名隨機長度
FILENAMELENGTH int `json:'FILENAMELENGTH'`
//同步的地址
RYSNCADDR [] string `json:'RYSNCADDR'`
}
var conf = ServerConfig{}
func main () {
confile := flag.String( "conf" , "" , "the configuration file" )
flag.Parse()
if *confile == "" {
fmt.Println( "Please specify the configuration file" )
return
}
file, _ := os.Open(*confile)
defer file.Close()
decoder := json.NewDecoder(file)
err := decoder.Decode(&conf)
if err != nil {
fmt.Println( "Error:" , err)
return
}
//@TODO-FORTEST
//if err := json.Unmarshal([]byte(jsonstr), &conf); err != nil {
// panic("ErrorConfig")
//}
log.Println( "Server is starting:" + conf.ADDR)
log.Println( "Server UploadPath:" + conf.PATH)
log.Print( "Server Rysnc Addr:" + strings.Replace(strings.Trim(fmt.Sprint(conf.RYSNCADDR), "[]" ), " " , "," , -1 ))
http.HandleFunc( "/upload" , UploadHandler)
http.HandleFunc( "/rsync" , RsyncHandler)
if err := http.ListenAndServe(conf.ADDR, nil ); err != nil {
fmt.Println( "Server starting error" )
}
}
func RandomFile (path string , suffix string ) ( string , error) {
if (!IsFileExist(path)) {
err := os.MkdirAll(path, os.ModePerm)
return "" , err
}
for {
dstFile := path + NewLenChars(conf.FILENAMELENGTH) + suffix
if (!IsFileExist(dstFile)) {
return dstFile, nil
}
}
}
func IsFileExist (filename string ) bool {
if _, err := os.Stat(filename); os.IsNotExist(err) {
return false
}
return true
}
func NewLenChars (length int ) string {
if length == 0 {
return ""
}
var chars = [] byte ( "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" )
clen := len (chars)
if clen < 2 || clen > 256 {
panic ( "Wrong charset length for NewLenChars()" )
}
maxrb := 255 - ( 256 % clen)
b := make ([] byte , length)
r := make ([] byte , length+(length/ 4 ))
i := 0
for {
if _, err := rand.Read(r); err != nil {
panic ( "Error reading random bytes: " + err.Error())
}
for _, rb := range r {
c := int (rb)
if c > maxrb {
continue
}
b[i] = chars[c%clen]
i++
if i == length {
return string (b)
}
}
}
}
func getCurrentIP (r http.Request) ( string ) {
ip := r.Header.Get( "X-Real-IP" )
if ip == "" {
return r.RemoteAddr
}
return ip
}
func RsyncHandler (w http.ResponseWriter, r *http.Request) {
file, header, err := r.FormFile( "rsyncfile" )
defer file.Close()
if err != nil {
log.Println(fmt.Sprintf( "%s rsyncfile %s %s " , getCurrentIP(*r), header.Filename, "FormParseError" ))
res := fmt.Sprintf( "{'code':'error'}" )
w.Header().Add( "Content-Type" , "application/json;charset:utf-8;" )
fmt.Fprintf(w, res)
return
}
dstFile := conf.PATH + header.Filename
if IsFileExist(dstFile) {
log.Println(fmt.Sprintf( "%s rsyncfile %s %s " , getCurrentIP(*r), header.Filename, "FileExists" ))
res := fmt.Sprintf( "{'code':'error'}" )
w.Header().Add( "Content-Type" , "application/json;charset:utf-8;" )
fmt.Fprintf(w, res)
return
}
cur, err := os.Create(dstFile);
defer cur.Close()
if err != nil {
log.Println(fmt.Sprintf( "%s rsyncfile %s %s " , getCurrentIP(*r), header.Filename, "CreateError" ))
res := fmt.Sprintf( "{'code':'error'}" )
w.Header().Add( "Content-Type" , "application/json;charset:utf-8;" )
fmt.Fprintf(w, res)
return
}
res := fmt.Sprintf( "{'code':'error'}" )
loginfo := ""
_, erro := io.Copy(cur, file)
if erro != nil {
loginfo = fmt.Sprintf( "%s rsyncfile %s %s" , getCurrentIP(*r), header.Filename, "WriteError" )
} else {
loginfo = fmt.Sprintf( "%s rsyncfile %s %s" , getCurrentIP(*r), header.Filename, "RysncSuccess" )
res = fmt.Sprintf( "{'code':'success'}" )
}
log.Println(loginfo)
w.Header().Add( "Content-Type" , "application/json;charset:utf-8;" )
fmt.Fprintf(w, res)
}
func UploadHandler (w http.ResponseWriter, r *http.Request) {
// 實現多檔案接收
//上傳結果以以json格式返回
uploadPath := r.Header.Get( "Path" )
basePath := conf.PATH
re2, _ := regexp.Compile( "\\.{2,}" )
re3, _ := regexp.Compile( "/{2,}" )
if uploadPath != "" {
if decodeBytes, err := base64.StdEncoding.DecodeString(uploadPath); err == nil {
ppath := string (decodeBytes)
ppath = re3.ReplaceAllString(re2.ReplaceAllString(ppath, "" ), "/" )
uploadPath = ppath
basePath += "/" + ppath
}
}
if (!strings.HasSuffix(basePath, "/" )) {
basePath += "/"
}
basePath = re3.ReplaceAllString(basePath, "/" )
bastPathLen := len (conf.PATH) - 1
reader, err := r.MultipartReader()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
s := ""
res := "success"
for {
part, err := reader.NextPart()
if err == io.EOF {
break
}
if newfile, err := RandomFile(basePath, path.Ext(part.FileName())); err == nil {
if part.FileName() != "" {
dst, _ := os.Create(newfile)
defer dst.Close()
io.Copy(dst, part)
newFileName := string ([] byte (newfile)[bastPathLen:])
log.Println(fmt.Sprintf( "%s uploadfile [%s][%s] > %s" , getCurrentIP(*r),
part.FormName(), part.FileName(), newfile))
s += fmt.Sprintf( "%s@%s:'%s'," , part.FormName(), part.FileName(), newFileName)
for _, v := range conf.RYSNCADDR {
go Rsync(v, uploadPath, newfile)
}
}
} else {
log.Println(fmt.Sprintf( "%s uploadfile [%s][%s] CreateDestinationFileError" , getCurrentIP(*r),
part.FormName(), part.FileName(), newfile))
s += fmt.Sprintf( "%s@%s:'%s'," , part.FormName(), part.FileName())
res = "error"
}
}
w.Header().Add( "Content-Type" , "application/json;charset:utf-8;" )
fmt.Fprintf(w, fmt.Sprintf( "{'code':'%s',results:{%s}}" , res, strings.Trim(s, "," )))
}
func Rsync (url string , dstPath string , files string ) {
bodyBuffer := &bytes.Buffer{}
bodyWriter := multipart.NewWriter(bodyBuffer)
_, fileName := filepath.Split(files)
fileWriter, _ := bodyWriter.CreateFormFile( "rsyncfile" , fileName)
file, _ := os.Open(files)
defer file.Close()
io.Copy(fileWriter, file)
contentType := bodyWriter.FormDataContentType()
bodyWriter.Close()
if req, err := http.NewRequest( "POST" , url, bodyBuffer); err == nil {
req.Header.Set( "Content-Type" , contentType)
if dstPath != "" {
req.Header.Set( "Path" , base64.StdEncoding.EncodeToString([] byte (dstPath)))
}
if resp, errsp := http.DefaultClient.Do(req); errsp == nil {
resp_body, _ := ioutil.ReadAll(resp.Body)
log.Println(fmt.Sprintf( "Clientrsyncfile %s %s " , resp.Status, string (resp_body)))
} else {
log.Println(fmt.Sprintf( "Clientrsyncfile Error %s %s " , url, files))
}
} else {
log.Println(fmt.Sprintf( "Clientrsyncfile Error %s %s " , url, files))
}
}