rsync 安裝使用實踐

# 編輯 rsync 配置檔案
vim /etc/rsync.conf

# /etc/rsyncd: configuration file for rsync daemon mode

# See rsyncd.conf man page for more options.

# configuration example:

# uid = nobody
# gid = nobody
# use chroot = yes
# max connections = 4
# pid file = /var/run/rsyncd.pid
# exclude = lost+found/
# transfer logging = yes
# timeout = 900
# ignore nonreadable = yes
# dont compress= *.gz *.tgz *.zip *.z *.Z *.rpm *.deb *.bz2

# [ftp]
#path = /home/ftp
#comment = ftp export area

rsync is a file transfer program capable of efficient remote update via a fast differencing algorithm.

Usage: rsync [OPTION]... SRC [SRC]... DEST
orrsync [OPTION]... SRC [SRC]... [USER@]HOST:DEST
orrsync [OPTION]... SRC [SRC]... [USER@]HOST::DEST
orrsync [OPTION]... SRC [SRC]... rsync://[USER@]HOST[:PORT]/DEST
orrsync [OPTION]... [USER@]HOST:SRC [DEST]
orrsync [OPTION]... [USER@]HOST::SRC [DEST]
orrsync [OPTION]... rsync://[USER@]HOST[:PORT]/SRC [DEST]
The ':' usages connect via remote shell, while '::' & 'rsync://' usages connect
to an rsync daemon, and require SRC or DEST to start with a module name.

Options
 -v, --verboseincrease verbosity
--info=FLAGSfine-grained informational verbosity
--debug=FLAGSfine-grained debug verbosity
--msgs2stderrspecial output handling for debugging
 -q, --quietsuppress non-error messages
--no-motdsuppress daemon-mode MOTD (see manpage caveat)
 -c, --checksumskip based on checksum, not mod-time & size
 -a, --archivearchive mode; equals -rlptgoD (no -H,-A,-X)
--no-OPTIONturn off an implied OPTION (e.g. --no-D)
 -r, --recursiverecurse into directories
 -R, --relativeuse relative path names
--no-implied-dirsdon't send implied dirs with --relative
 -b, --backupmake backups (see --suffix & --backup-dir)
--backup-dir=DIRmake backups into hierarchy based in DIR
--suffix=SUFFIXset backup suffix (default ~ w/o --backup-dir)
 -u, --updateskip files that are newer on the receiver
--inplaceupdate destination files in-place (SEE MAN PAGE)
--appendappend data onto shorter files
--append-verifylike --append, but with old data in file checksum
 -d, --dirstransfer directories without recursing
 -l, --linkscopy symlinks as symlinks
 -L, --copy-linkstransform symlink into referent file/dir
--copy-unsafe-linksonly"unsafe"symlinks are transformed
--safe-linksignore symlinks that point outside the source tree
--munge-linksmunge symlinks to make them safer (but unusable)
 -k, --copy-dirlinkstransform symlink to a dir into referent dir
 -K, --keep-dirlinkstreat symlinked dir on receiver as dir
 -H, --hard-linkspreserve hard links
 -p, --permspreserve permissions
 -E, --executabilitypreserve the file's executability
--chmod=CHMODaffect file and/or directory permissions
 -A, --aclspreserve ACLs (implies --perms)
 -X, --xattrspreserve extended attributes
 -o, --ownerpreserve owner (super-user only)
 -g, --grouppreserve group
--devicespreserve device files (super-user only)
--copy-devicescopy device contents as regular file
--specialspreserve special files
 -Dsame as --devices --specials
 -t, --timespreserve modification times
 -O, --omit-dir-timesomit directories from --times
 -J, --omit-link-timesomit symlinks from --times
--superreceiver attempts super-user activities
--fake-superstore/recover privileged attrs using xattrs
 -S, --sparsehandle sparse files efficiently
--preallocateallocate dest files before writing them
 -n, --dry-runperform a trial run with no changes made
 -W, --whole-filecopy files whole (without delta-xfer algorithm)
 -x, --one-file-systemdon't cross filesystem boundaries
 -B, --block-size=SIZEforce a fixed checksum block-size
 -e, --rsh=COMMANDspecify the remote shell to use
--rsync-path=PROGRAMspecify the rsync to run on the remote machine
--existingskip creating new files on receiver
--ignore-existingskip updating files that already exist on receiver
--remove-source-filessender removes synchronized files (non-dirs)
--delan alias for --delete-during
--deletedelete extraneous files from destination dirs
--delete-beforereceiver deletes before transfer, not during
--delete-duringreceiver deletes during the transfer
--delete-delayfind deletions during, delete after
--delete-afterreceiver deletes after transfer, not during
--delete-excludedalso delete excluded files from destination dirs
--ignore-missing-argsignore missing source args without error
--delete-missing-argsdelete missing source args from destination
--ignore-errorsdelete even if there are I/O errors
--forceforce deletion of directories even if not empty
--max-delete=NUMdon't delete more than NUM files
--max-size=SIZEdon't transfer any file larger than SIZE
--min-size=SIZEdon't transfer any file smaller than SIZE
--partialkeep partially transferred files
--partial-dir=DIRput a partially transferred file into DIR
--delay-updatesput all updated files into place at transfer's end
 -m, --prune-empty-dirsprune empty directory chains from the file-list
--numeric-idsdon't map uid/gid values by user/group name
--usermap=STRINGcustom username mapping
--groupmap=STRINGcustom groupname mapping
--chown=USER:GROUPsimple username/groupname mapping
--timeout=SECONDSset I/O timeout in seconds
--contimeout=SECONDSset daemon connection timeout in seconds
 -I, --ignore-timesdon't skip files that match in size and mod-time
 -M, --remote-option=OPTIONsend OPTION to the remote side only
--size-onlyskip files that match in size
--modify-window=NUMcompare mod-times with reduced accuracy
 -T, --temp-dir=DIRcreate temporary files in directory DIR
 -y, --fuzzyfind similar file for basis if no dest file
--compare-dest=DIRalso compare destination files relative to DIR
--copy-dest=DIR... and include copies of unchanged files
--link-dest=DIRhardlink to files in DIR when unchanged
 -z, --compresscompress file data during the transfer
--compress-level=NUMexplicitly set compression level
--skip-compress=LISTskip compressing files with a suffix in LIST
 -C, --cvs-excludeauto-ignore files the same way CVS does
 -f, --filter=RULEadd a file-filtering RULE
 -Fsame as --filter='dir-merge /.rsync-filter'
repeated: --filter='- .rsync-filter'
--exclude=PATTERNexclude files matching PATTERN
--exclude-from=FILEread exclude patterns from FILE
--include=PATTERNdon't exclude files matching PATTERN
--include-from=FILEread include patterns from FILE
--files-from=FILEread list of source-file names from FILE
 -0, --from0all *-from/filter files are delimited by 0s
 -s, --protect-argsno space-splitting; only wildcard special-chars
--address=ADDRESSbind address for outgoing socket to daemon
--port=PORTspecify double-colon alternate port number
--sockopts=OPTIONSspecify custom TCP options
--blocking-iouse blocking I/O for the remote shell
--statsgive some file-transfer stats
 -8, --8-bit-outputleave high-bit chars unescaped in output
 -h, --human-readableoutput numbers in a human-readable format
--progressshow progress during transfer
 -Psame as --partial --progress
 -i, --itemize-changesoutput a change-summary for all updates
--out-format=FORMAToutput updates using the specified FORMAT
--log-file=FILElog what we're doing to the specified FILE
--log-file-format=FMTlog updates using the specified FMT
--password-file=FILEread daemon-access password from FILE
--list-onlylist the files instead of copying them
--bwlimit=RATElimit socket I/O bandwidth
--outbuf=N|L|Bset output buffering to None, Line, or Block
--write-batch=FILEwrite a batched update to FILE
--only-write-batch=FILE like --write-batch but w/o updating destination
--read-batch=FILEread a batched update from FILE
--protocol=NUMforce an older protocol version to be used
--iconv=CONVERT_SPECrequest charset conversion of filenames
--checksum-seed=NUMset block/file checksum seed (advanced)
 -4, --ipv4prefer IPv4
 -6, --ipv6prefer IPv6
--versionprint version number
(-h) --helpshow this help (-h is --help only if used alone)

Use"rsync --daemon --help"to see the daemon-mode command-line options.
Please see the rsync(1) and rsyncd.conf(5) man pages for full documentation.
See http://rsync.samba.org/ for updates, bug reports, and answers

# rsync 常用引數 
-v : 展示詳細的同步資訊
-a : 歸檔模式，相當於 -rlptgoD
-r : 遞迴目錄
-l : 同步軟連線檔案
-p : 保留許可權
-t : 將原始檔的"modify time"同步到目標機器
-g : 保持檔案屬組
-o : 保持檔案屬主
-D : 和 --devices --specials 一樣，保持裝置檔案和特殊檔案
-z : 傳送資料前，先壓縮再傳輸
-H : 保持硬連結
-n : 進行試執行，不作任何更改
-P same as --partial --progress
--partial : 支援斷點續傳
--progress : 展示傳輸的進度
--delete : 如果原始檔消失，目標檔案也會被刪除
--delete-excluded : 指定要在目的端刪除的檔案
--delete-after : 預設情況下，rsync 是先清理目的端的檔案再開始資料同步；如果使用此選項，則 rsync 會先進行資料同步，都完成後再刪除那些需要清理的檔案。
--exclude=PATTERN : 排除匹配 PATTERN 的檔案
--exclude-from=FILE : 如果要排除的檔案很多，可以統一寫在某一檔案中
-e ssh : 使用 SSH 加密隧道傳輸

# 遠端 Shell 方式
rsync [OPTION]... SRC [SRC]... [USER@]HOST:DEST # 執行“推” 操作 
orrsync [OPTION]... [USER@]HOST:SRC [DEST]# 執行“拉” 操作 

# 遠端 C/S 方式
rsync [OPTION]... SRC [SRC]... [USER@]HOST::DEST# 執行“推” 操作 
orrsync [OPTION]... SRC [SRC]... rsync://[USER@]HOST[:PORT]/DEST # 執行“推” 操作 
orrsync [OPTION]... [USER@]HOST::SRC [DEST]# 執行“拉” 操作 
orrsync [OPTION]... rsync://[USER@]HOST[:PORT]/SRC [DEST]# 執行“拉” 操作

mkdir dir1
mkdir dir2
touch dir1/somefile{1..100}
# dir1 中有 100 檔案，dir2 中為空。使用 rsync 把 dir1 內容同步到 dir2，-r 選項代表遞迴，在同步目錄時使用。
rsync -r dir1/ dir2
# 你也可以使用 -a 選項，代表同步所有，包括修改時間、群組、許可權、特殊檔案、也包括遞迴。
rsync -anv dir1/ dir2
# 注意上面的 dir1 / 中的 “/” 不能少，它代表同步目錄下檔案， 如果沒有 “/” 代表同步這個目錄。

# 和遠端主機進行同步目錄首先，你要確保有遠端主機的 SSH 訪問許可權

# 把本地目錄同步到遠端主機：
rsync -a dir1/ root@linux:~/dir2
# 把遠端主機目錄同步到本地：
rsync -a root@linux:~/dir2/ dir1

# 如果沒有 desc 目錄，會自動建立
rsync -avH /opt/resource/ /tmp/desc/

# 從本地傳到遠端，目標檔案會被寫成 ssh 登入使用者的屬組和屬主（如下 www）
rsync -avH /opt/nginx-1.12.1/ [email protected]:/tmp/nginx/

# 使用 ssh 加密隧道方式傳輸，保障資料的安全性
rsync -avHe ssh /opt/nginx-1.12.1/ [email protected]:/tmp/nginx/

# 從遠端傳到本地，只要對目標檔案有讀的許可權，就可以同步到本地
rsync -avH [email protected]:/tmp/nginx/ /tmp/nginx/

# 如果遠端伺服器 ssh 埠不是預設的 22
rsync -avHe "ssh -p 11222" /opt/nginx-1.12.1/ [email protected]:/tmp/nginx/

# 建立 rsync 服務的目錄和配置檔案 (可選)
mkdir /etc/rsync 
cd /etc/rsync
touch rsyncd.conf
touch rsyncd.secrets
touch rsyncd.motd
chmod 600 rsyncd.secrets

### rsyncd.conf 檔案的配置
vim /etc/rsync/rsyncd.conf
# /etc/rsyncd: configuration file for rsync daemon mode
# See rsyncd.conf man page for more options.
# 傳輸檔案使用的使用者和使用者組，如果是從伺服器 => 客戶端，要保證 www 使用者對檔案有讀取的許可權；如果是從客戶端 => 服務端，要保證 www 對檔案有寫許可權。
uid = www
gid = www
# 允許 chroot，提升安全性，客戶端連線模組，首先 chroot 到模組 path 引數指定的目錄下，chroot 為 yes 時必須使用 root 許可權，且不能備份 path 路徑外的連結檔案
use chroot = yes
# 只讀
read only = no
# 只寫
write only = no
# 設定白名單，可以指定 IP 段（172.18.50.1/255.255.255.0）, 各個 Ip 段用空格分開
hosts allow = 172.18.50.110 172.18.50.111
hosts deny = *
# 允許的客戶端最大連線數
max connections = 4
# 歡迎檔案的路徑，非必須
motd file = /etc/rsync/rsyncd.motd
# pid 檔案路徑
pid file = /var/run/rsyncd.pid
# 記錄傳輸檔案日誌
transfer logging = yes
# 日誌檔案格式
log format = %t %a %m %f %b
# 指定日誌檔案
log file = /var/log/rsync.log
# 剔除某些檔案或目錄，不同步
exclude = lost+found/
# 設定超時時間
timeout = 900
ignore nonreadable = yes
# 設定不需要壓縮的檔案
dont compress= *.gz *.tgz *.zip *.z *.Z *.rpm *.deb *.bz2

# 模組，可以配置多個，使用如: [email protected]::125to110
[125to110]
# 模組的根目錄，同步目錄，要注意許可權
path = /tmp/nginx
# 是否允許列出模組內容
list = no
# 忽略錯誤
ignore errors
# 添加註釋
comment = ftp export area
# 模組驗證的使用者名稱稱，可使用空格或者逗號隔開多個使用者名稱
auth users = sate
# 模組驗證密碼檔案 可放在全域性配置裡
secrets file = /etc/rsync/rsyncd.secrets
# 剔除某些檔案或目錄，不同步
exclude = lost+found/ conf/ man/

### rsyncd.secrets 檔案的配置
cat rsyncd.secrets 
# 使用者名稱: 密碼
sate:111111

### rsync 啟動
rsync --daemon --config=/etc/rsync/rsyncd.conf

# 從 服務端 => 客戶端 同步資料，會提示輸入密碼
rsync -avzP --delete [email protected]::125to110 /tmp/sync/

# 從 客戶端 => 服務端 同步資料，會提示輸入密碼
rsync -avzP --delete /tmp/sync/ [email protected]::125to110

# 注: 如果是 /tmp/sync，則同步 sync 目錄；如果 /tmp/sync/，則同步 sync 目錄下的檔案

# 免密碼同步，將密碼寫到檔案，再通過 --password-file 指定該檔案，注：該檔案的許可權必須是 600
echo "111111" > /tmp/secrets.file
chmod 600 /tmp/secrets.file
rsync -avzP --delete --password-file=/tmp/secrets.file [email protected]::125to110 /tmp/sync/

# --exclude 排除檔案目錄時，如果有多個同名目錄的情況
# 目錄結構
tree
.
├── dir1
│└── test
│├── 3.file
│├── 4.file
│└── 5.file
├── dir2
└── test
├── 1.file
├── 2.file
└── 3.file

# 情況一 ： 排除 /test 目錄，同步其他目錄（同步的是 / tmp/sync/ 下邊的檔案）
rsync -avP --delete --password-file=/tmp/secrets.file --exclude=test/tmp/sync/ [email protected]::125to110 

# 會發現，該目錄下所有 test 目錄都被排除了，如果想只排除第一層目錄的 test，可以如下（/ 代表所同步目錄第一層）：
rsync -avP --delete --password-file=/tmp/secrets.file --exclude=/test//tmp/sync/ [email protected]::125to110 

# 情況二 ： 和情況一不同的是 同步的 /tmp/sync 這個目錄（同步的是 / tmp/sync 目錄本身，導致 exclude 後邊的引數也會變化）
rsync -avP --delete --password-file=/tmp/secrets.file --exclude=/sync/test//tmp/sync [email protected]::125to110

# 配置服務端 rsyncd.conf
vim /etc/rsyncd.conf

read only = no
list = yes
uid = root
gid = root

[backup]
path= /data/
hosts allow = 10.71.12.0/23

# 設定服務
systemctl start rsyncd
systemctl enable rsyncd

# 配置 rsync 客戶端

# 編輯 backup.sh 同步指令碼
vim backup.sh

#!/bin/sh
SOURCE=$1
DEST=$2

CMD="rsync -ravz --bwlimit=2000 $1 rsync://{{log_server_ip}}:873/backup/$2"

PROCS=$(pgrep -f "{{log_server_ip}}:873/backup/$2")

if [ "x" != "x$PROCS" ]; then
echo "not finished"
exit
fi

$CMD

# 修改 crontab
vim /etc/crontab
15 * * * * root cd /opt/sa_scripts/ && ./backup.sh /var/log/ocha/pos_python_server/10.71.12.89/$(date +\%Y-\%m)