根據上一篇下載的數據，現在每天更新day數據

需要保證股票當日新上市下載，退市保留，停牌復牌識別、是否發生復權（決定前復權價格是否變化）

數據更新

1、大盤指數數據（399001, 399005, 399006, 399300，999999）

## Python
import pandas as pd
import numpy as np
import tushare as ts
import os
import shutil

os.getcwd()
os.chdir("C:/Users/~~~/Desktop/數據")   #修改當前工作目錄

## 尋找上一數據日期
last_data_date = max([int(i[4:]) for i in os.listdir("ts大盤數據")])
## 所有交易日期
all_date = [int(i.replace("-", "")) for i in ts.trade_cal()["calendarDate"][ts.trade_cal()["isOpen"] == 1]]

## 尋找上一交易日期
def func(x, lst):
    i = 0
    while(lst[i] <= x):
        i += 1
    return lst[i-1]
last_data_day = str(func(last_data_date, all_date))

## 更新到新日期，創建新文件夾
new_date = "20171221
 
"
os.mkdir("ts大盤數據/大盤數據"+new_date)
shutil.copyfile("ts大盤數據/大盤數據"+str(last_data_date)+"/"+"stock_info.csv","ts大盤數據/大盤數據"+new_date+"/"+"stock_info.csv")

## 獲取數據起始與終止日期
start_date = last_data_day[0:4]+"-"+last_data_day[4:6]+"-"+last_data_day[6:8]
end_date = new_date[0:4]+"-"+new_date[4:6]+"-"+new_date[6:8]

## 大盤補全
## 399001, 399005, 399006, 399300
dapan_code = ["399001", "399005", "399006", "399300"]
for i in dapan_code:
    old_dapan = pd.read_csv("ts大盤數據/大盤數據"+str(last_data_date)+"/"+i+".csv", engine=‘python‘)
    
    new_dapan = ts.get_h_data(i, start=start_date, end=end_date,  index=True, pause=4)
    
    new_dapan.sort_index(inplace=True)
    new_dapan[["open", "high", "close", "low"]] = new_dapan[["open", "high", "close", "low"]]*10000
    new_dapan["date"] = new_dapan.index
    new_dapan["date"] = new_dapan["date"].astype(str).apply(lambda x:x.replace(‘-‘, ‘‘))
#     new_dapan.index = list(range(len(old_dapan)-1, len(old_dapan)+len(new_dapan)-1))
    new_dapan.columns = [‘open‘,‘high‘, ‘close‘, ‘low‘, ‘volumw‘, ‘turover‘, ‘date‘]
    if new_dapan.iloc[0, 4] == old_dapan.iloc[-1, 8]:
        new_dapan.drop(new_dapan.index[0], axis=0, inplace=True)
        dapan = old_dapan.append(new_dapan, ignore_index=True)
        dapan = dapan[old_dapan.columns]
        dapan.fillna(method=‘ffill‘,inplace=True)
        dapan.to_csv("ts大盤數據/大盤數據"+new_date+"/"+i+".csv", index=False)
		
## 000001 --> 999999 上證指數 （新浪財經對指數編號不同）
old_dapan = pd.read_csv("ts大盤數據/大盤數據"+str(last_data_date)+"/"+"999999.csv", engine=‘python‘)
    
start_date = last_data_day[0:4]+"-"+last_data_day[4:6]+"-"+last_data_day[6:8]
end_date = new_date[0:4]+"-"+new_date[4:6]+"-"+new_date[6:8]
new_dapan = ts.get_h_data("000001", start=start_date, end=end_date,  index=True, pause=4)
    
new_dapan.sort_index(inplace=True)
new_dapan[["open", "high", "close", "low"]] = new_dapan[["open", "high", "close", "low"]]*10000
new_dapan["date"] = new_dapan.index
new_dapan["date"] = new_dapan["date"].astype(str).apply(lambda x:x.replace(‘-‘, ‘‘))
new_dapan.columns = [‘open‘,‘high‘, ‘close‘, ‘low‘, ‘volumw‘, ‘turover‘, ‘date‘]

if new_dapan.iloc[0, 4] == old_dapan.iloc[-1, 8]:
  new_dapan.drop(new_dapan.index[0], axis=0, inplace=True)
  dapan = old_dapan.append(new_dapan, ignore_index=True)
  dapan = dapan[old_dapan.columns]
  dapan.fillna(method=‘ffill‘,inplace=True)
  dapan.to_csv("ts大盤數據/大盤數據"+new_date+"/"+"999999.csv", index=False)
  

2、A股股票數據與概括文件

## R
## 自動補全函數
## new.date:補全截至時間
library(WindR)
library(xlsx)
library(data.table)
library(magrittr)
library(tcltk2)
# (.packages())

w.start()
setwd("C:/Users/~~~/Desktop/坴戔/華泰/數據")

## WindR獲取數據規則：退市時期數據為NA，停牌時期數據以上一交易日收盤價補全？
## 文件夾內不能隨便新建文件！！！

## 1.設置截至日期（若補當日數據，五點之後執行）
new.date <- "20171221
 
"

end.date <- paste(substr(new.date, 1, 4), substr(new.date, 5, 6), 
                  substr(new.date, 7, 8), sep = "-")

## 2.摘取當日在市股票代碼
stock.code.df <- w.wset(‘sectorconstituent‘,
                        paste0("date=", end.date, ";sectorid=a001010100000000"))
if(stock.code.df$ErrorCode == 0){
  # if(gsub("-", "", as.Date(stock.code.df$Time)) == new.date){
  stock.code.sh.sz <- stock.code.df$Data$wind_code
  # }else{
  #   print("獲取時間出錯")
  # }
}else{
  print(paste0("獲取數據出錯，錯誤代碼", stock.code.df$ErrorCode))
}
new.stock.code <- substr(stock.code.sh.sz, 1, 6)
# str(stock.code.sh.sz)

## 3.獲取上一次數據日期
old.file.name <- gsub("股票數據", "", dir("股票數據"))
last.data.day <- old.file.name[which.max(old.file.name)] ## 是否需要改為數值型比較
## 4.數據上一次所有股票代碼
old.stock.code <- gsub(".csv", "", dir(paste0("股票數據/股票數據", last.data.day)))

## 5.創建新的文件夾
dir.create(paste0("股票數據/股票數據", new.date))

## 6.股票補全部分

## 概括性文件變量
## 退市數據保留執行以下部分代碼，不保留執行已有那一行註釋代碼
all.stock.code <- unique(c(new.stock.code, old.stock.code))
general.information <- data.frame(array(dim=c(length(all.stock.code), 5)))
# general.information <- data.frame(array(dim=c(length(stock.code.sh.sz), 5)))

colnames(general.information) <- c("stock.code", "stock.name", "type", "starttime", "endtime")

## 大盤日期
dapan.date <- read.csv(paste0("ts大盤數據/大盤數據", new.date, "/399300.csv"))$date

## 設置進度條
pb <- tkProgressBar("進度", "已完成 %", 0, 100) 


# i <- 3
# length(stock.code.sh.sz)
for(i in 1:length(stock.code.sh.sz)){
  if(new.stock.code[i] %in% old.stock.code){
    old.df <- fread(paste0("股票數據/股票數據", last.data.day, "/", new.stock.code[i], ".csv"), 
                    header=TRUE, integer64="numeric", sep = ",")
    ## 尋找原數據中最後有交易數據的行數與日期
    if(old.df$volumw[nrow(old.df)] != 0){
      j <- nrow(old.df)
      last.trading.day <- old.df$date[j]
    }else{
      j <- nrow(old.df)
      while(old.df$volumw[j] == 0 & j > 0){
        j <- j - 1
      }
      last.trading.day <- old.df$date[j]
    }
    start.date <- paste(substr(last.trading.day, 1, 4), substr(last.trading.day, 5, 6), 
                        substr(last.trading.day, 7, 8), sep = "-")

    new.data <- w.wsd(stock.code.sh.sz[i], "trade_code, sec_name, open, high, low, close,
                      volume, amt, free_turn,free_float_shares",
                      start.date, end.date, "unit=1;PriceAdj=F")
    if(new.data$ErrorCode == 0){
      if(any(is.na(new.data$Data))){
        ## 退市又復市
        if(any(is.na(new.data$Data[1, ]))){
          print(paste(stock.code.sh.sz[i], "退市又復市,但上個交易日數據獲取為NA,僅輸出新數據"))
          wind.df <- data.frame(array(dim=c(nrow(new.data$Data), 12)))
          colnames(wind.df) <- c("wind_code", "name", "date", "time", "open", "high", "low", "close", 
                                 "volumw", "turover", "free_turn", "free_float_shares")
          wind.df[, 1]    <- new.data$Code
          wind.df[, 2]    <- new.data$Data$SEC_NAME
          wind.df[, 3]    <- gsub("-", "", new.data$Data$DATETIME)
          wind.df[, 4]    <- 151500000
          wind.df[, 5:8]  <- new.data$Data[4:7] * 10000
          wind.df[, 9:12] <- new.data$Data[8:11]
          wind.df <- wind.df[!(is.na(wind.df$open)), ]
        }else if(all(round(unlist(old.df[j, 5:8])) == round(unlist(new.data$Data[1, 4:7]*10000)))){
          ## 退市復市股權未變動
          print(paste(stock.code.sh.sz[i], "退市又復市,股權未變動"))
          wind.df <- data.frame(array(dim=c(nrow(old.df)+nrow(new.data$Data)-1, 12)))
          colnames(wind.df) <- c("wind_code", "name", "date", "time", "open", "high", "low", "close", "volumw",
                                 "turover", "free_turn", "free_float_shares")
          wind.df[1:nrow(old.df), ] <- old.df[, 1:12] # 14變為12
          wind.df[(nrow(old.df)+1):nrow(wind.df), 1]    <- new.data$Code
          wind.df[(nrow(old.df)+1):nrow(wind.df), 2]    <- new.data$Data$SEC_NAME[-1]
          wind.df[(nrow(old.df)+1):nrow(wind.df), 3]    <- gsub("-", "", new.data$Data$DATETIME)[-1]
          wind.df[(nrow(old.df)+1):nrow(wind.df), 4]    <- 151500000
          wind.df[(nrow(old.df)+1):nrow(wind.df), 5:8]  <- new.data$Data[-1, 4:7] * 10000
          wind.df[(nrow(old.df)+1):nrow(wind.df), 9:12] <- new.data$Data[-1, 8:11]
          wind.df <- wind.df[!(is.na(wind.df$open)), ]
          # 停牌重復行刪去
          wind.df <- wind.df[!duplicated(wind.df$date), ]
        }else{
          ## 退市復市股權變動
          print(paste(stock.code.sh.sz[i], "退市又復市,股權變動"))
          wind.df <- data.frame(array(dim=c(nrow(old.df)+nrow(new.data$Data)-1, 12)))
          colnames(wind.df) <- c("wind_code", "name", "date", "time", "open", "high", "low", "close", "volumw",
                                 "turover", "free_turn", "free_float_shares")
          ## 以開盤價定復權因子
          ratio <- unlist(new.data$Data[1, 4]*10000 / old.df[j, 5])
          wind.df[1:nrow(old.df), 1:4]  <- old.df[, 1:4]
          wind.df[1:nrow(old.df), 5:8]  <- old.df[, 5:8] * ratio
          wind.df[1:nrow(old.df), 9:12] <- old.df[, 9:12]
          
          wind.df[(nrow(old.df)+1):nrow(wind.df), 1]    <- new.data$Code
          wind.df[(nrow(old.df)+1):nrow(wind.df), 2]    <- new.data$Data$SEC_NAME[-1]
          wind.df[(nrow(old.df)+1):nrow(wind.df), 3]    <- gsub("-", "", new.data$Data$DATETIME)[-1]
          wind.df[(nrow(old.df)+1):nrow(wind.df), 4]    <- 151500000
          wind.df[(nrow(old.df)+1):nrow(wind.df), 5:8]  <- new.data$Data[-1, 4:7] * 10000
          wind.df[(nrow(old.df)+1):nrow(wind.df), 9:12] <- new.data$Data[-1, 8:11]
          wind.df <- wind.df[!(is.na(wind.df$open)), ]
          # 停牌重復行刪去
          wind.df <- wind.df[!duplicated(wind.df$date), ]
        }
      }else{
        ## 是否停牌不影響，只需判斷股權是否變動
        if(all(round(unlist(old.df[j, 5:8])) == round(unlist(new.data$Data[1, 4:7]*10000)))){
          ## 股權未變動
          wind.df <- data.frame(array(dim=c(nrow(old.df)+nrow(new.data$Data)-1, 12)))
          colnames(wind.df) <- c("wind_code", "name", "date", "time", "open", "high", "low", "close", "volumw",
                                 "turover", "free_turn", "free_float_shares")
          wind.df[1:nrow(old.df), ] <- old.df[, 1:12]
          wind.df[(nrow(old.df)+1):nrow(wind.df), 1]    <- new.data$Code
          wind.df[(nrow(old.df)+1):nrow(wind.df), 2]    <- new.data$Data$SEC_NAME[-1]
          wind.df[(nrow(old.df)+1):nrow(wind.df), 3]    <- gsub("-", "", new.data$Data$DATETIME)[-1]
          wind.df[(nrow(old.df)+1):nrow(wind.df), 4]    <- 151500000
          wind.df[(nrow(old.df)+1):nrow(wind.df), 5:8]  <- new.data$Data[-1, 4:7] * 10000
          wind.df[(nrow(old.df)+1):nrow(wind.df), 9:12] <- new.data$Data[-1, 8:11]
          wind.df <- wind.df[!(is.na(wind.df$open)), ]
          # 停牌重復行刪去
          wind.df <- wind.df[!duplicated(wind.df$date), ]
        }else{
          ## 股權變動
          print(paste(stock.code.sh.sz[i], "股權變動"))
          wind.df <- data.frame(array(dim=c(nrow(old.df)+nrow(new.data$Data)-1, 12)))
          colnames(wind.df) <- c("wind_code", "name", "date", "time", "open", "high", "low", "close", "volumw",
                                 "turover", "free_turn", "free_float_shares")
          ## 以開盤價定復權因子
          ratio <- unlist(new.data$Data[1, 4]*10000 / old.df[j, 5])
          wind.df[1:nrow(old.df), 1:4]  <- old.df[, 1:4]
          wind.df[1:nrow(old.df), 5:8]  <- old.df[, 5:8] * ratio
          wind.df[1:nrow(old.df), 9:12] <- old.df[, 9:12]
          
          wind.df[(nrow(old.df)+1):nrow(wind.df), 1]    <- new.data$Code
          wind.df[(nrow(old.df)+1):nrow(wind.df), 2]    <- new.data$Data$SEC_NAME[-1]
          wind.df[(nrow(old.df)+1):nrow(wind.df), 3]    <- gsub("-", "", new.data$Data$DATETIME)[-1]
          wind.df[(nrow(old.df)+1):nrow(wind.df), 4]    <- 151500000
          wind.df[(nrow(old.df)+1):nrow(wind.df), 5:8]  <- new.data$Data[-1, 4:7] * 10000
          wind.df[(nrow(old.df)+1):nrow(wind.df), 9:12] <- new.data$Data[-1, 8:11]
          wind.df <- wind.df[!(is.na(wind.df$open)), ]
          wind.df <- wind.df[!duplicated(wind.df$date), ]
        }
      }
    }else{
      print(paste0(stock.code.sh.sz[i], "獲取數據出錯，錯誤代碼：", new.data$ErrorCode))
      break
    }
  }else{
    # 新上市股票
    print(paste(stock.code.sh.sz[i], "新上市股票"))
    start.date <- paste(substr(last.data.day, 1, 4), substr(last.data.day, 5, 6), 
                        substr(last.data.day, 7, 8), sep = "-")
    
    new.data <- w.wsd(stock.code.sh.sz[i], "trade_code, sec_name, open, high, low, close, volume, amt,
                      free_turn, free_float_shares",
                      start.date, end.date, "unit=1;PriceAdj=F")
    if(new.data$ErrorCode == 0){
      wind.df <- data.frame(array(dim=c(nrow(new.data$Data), 12)))
      colnames(wind.df) <- c("wind_code", "name", "date", "time", "open", "high", "low", "close", "volumw",
                             "turover", "free_turn", "free_float_shares")
      wind.df[, 1]    <- new.data$Code
      wind.df[, 2]    <- new.data$Data$SEC_NAME
      wind.df[, 3]    <- gsub("-", "", new.data$Data$DATETIME)
      wind.df[, 4]    <- 151500000
      wind.df[, 5:8]  <- new.data$Data[4:7] * 10000
      wind.df[, 9:12] <- new.data$Data[8:11]
      wind.df         <- wind.df[!(is.na(wind.df$open)), ]
    }else{
      print(paste0(stock.code.sh.sz[i], "新上市股票獲取數據出錯，錯誤代碼：", new.data$ErrorCode))
      break
    }
  }
  
  ## 輸出部分
  if(nrow(wind.df) == 0){
    print(paste(stock.code.sh.sz[i], "數據出錯（在市股票不可能全為NA導致數據框行為0）"))
  }else{
    if(any(is.na(wind.df))){
      print(paste(stock.code.sh.sz[i], "數據出錯（數據中仍有NA）"))
    }else{
      if(any(wind.df[, 5:8] == 0)){
        print(paste(stock.code.sh.sz[i], "數據出錯（數據中開高低收存在0）"))
      }else{
        if(any(table(wind.df$date) > 1)){
          print(paste(stock.code.sh.sz[i], "數據出錯（數據中存在日期相同）"))
        }else{
          if(any(wind.df$date != sort(wind.df$date))){
            print(paste(stock.code.sh.sz[i], "數據出錯（數據中日期順序不對）"))
          }else{
            # 是否與大盤日期匹配,所有日期必須在大盤已有日期內
            if(all(wind.df$date %in% dapan.date)){
              first.date <- paste(substr(wind.df$date[1], 1, 4), substr(wind.df$date[1], 5, 6), 
                                  substr(wind.df$date[1], 7, 8), sep = "-")
              test <- w.wsd(stock.code.sh.sz[i], "trade_code, sec_name, open, high, low, close, volume, amt,
                                                  free_turn, free_float_shares",
                            first.date, first.date, "unit=1;PriceAdj=F")
              if(all(round(test$Data[4:7]*10000) == round(wind.df[1, 5:8])) & test$Data$VOLUME == wind.df[1, 9]){
                general.information[i, 1] <- substr(stock.code.sh.sz[i], 1, 6)
                general.information[i, 2] <- wind.df$name[1]
                general.information[i, 3] <- substr(stock.code.sh.sz[i], 8, 9)
                general.information[i, 4] <- wind.df[1, 3]
                general.information[i, 5] <- wind.df[nrow(wind.df), 3]
                write.csv(wind.df, paste0("股票數據/股票數據", new.date, "/", new.stock.code[i], ".csv"), 
                          row.names = FALSE)
              }else{
                print(paste(stock.code.sh.sz[i], "數據出錯（數據補全與wind不同）"))
              }
            }else{
              print(paste(stock.code.sh.sz[i], "數據出錯（數據中日期與大盤日期不符）"))
            }
          }
        }
      }
    }
  }
  rm(wind.df);rm(old.df);rm(j)
  info <- sprintf("已完成 %d%%", round(i*100/length(stock.code.sh.sz)))
  setTkProgressBar(pb, i*100/length(stock.code.sh.sz), sprintf("進度 (%s)", info), info)
}

## 關閉進度條
close(pb) 


## 退市數據繼續保留，則執行下面代碼
tuishi.row <- 1
for(i in 1:length(old.stock.code)){
  if(!(old.stock.code[i] %in% new.stock.code)){
    print(paste(stock.code.sh.sz, "仍然退市"))
    old.df <- fread(paste0("股票數據/股票數據", last.data.day, "/", new.stock.code[i], ".csv"), 
                    header=TRUE, integer64="numeric", sep = ",")
    df <- old.df[, 1:12]
    write.csv(df, paste("股票數據/股票數據", new.date, "/", new.stock.code[i], ".csv", sep=""), 
              row.names = FALSE)
    tuishi.row <- tuishi.row + 1
    general.information[length(stock.code.sh.sz)+tuishi.row, 1] <- old.stock.code[i]
    general.information[length(stock.code.sh.sz)+tuishi.row, 2] <- df$name[1]
    general.information[length(stock.code.sh.sz)+tuishi.row, 3] <- substr(df$wind_code[1], 8, 9)
    general.information[length(stock.code.sh.sz)+tuishi.row, 4] <- df[1, 3]
    general.information[length(stock.code.sh.sz)+tuishi.row, 5] <- df[nrow(df), 3]
  }
}


## 總概括文件中無NA時輸出
if(all(!(is.na(general.information)))){
  write.xlsx(general.information, paste0("概括文件/概括文件", new.date, ".xlsx"), row.names = FALSE)
}else{
  print("總概況文件中存在NA，需查驗")
}

A股數據日級前復權數據補全