Python筆記day32（網路）|解決黏包現象、Struct用法、實現ftp功能
阿新 • • 發佈：2018-12-31
1，黏包的解決方案
解決方案一
問題的根源在於，接收端不知道傳送端將要傳送的位元組流的長度，所以解決粘包的方法就是圍繞，如何讓傳送端在傳送資料前，把自己將要傳送的位元組流總大小讓接收端知曉，然後接收端來一個死迴圈接收完所有資料。
#_*_coding:utf-8_*_
import socket,subprocess
ip_port=('127.0.0.1',8080)
s=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind #_*_coding:utf-8_*_ 存在的問題：
程式的執行速度遠快於網路傳輸速度，所以在傳送一段位元組前，先用send去傳送該位元組流長度，這種方式會放大網路延遲帶來的效能損耗
解決方案進階
剛剛的方法，問題在於我們我們在傳送
我們可以藉助一個模組，這個模組可以把要傳送的資料長度轉換成固定長度的位元組。這樣客戶端每次接收訊息之前只要先接受這個固定長度位元組的內容看一看接下來要接收的資訊大小，那麼最終接受的資料只要達到這個值就停止，就能剛好不多不少的接收完整的資料了。
struct模組
該模組可以把一個型別，如數字，轉成固定長度的bytes
struct.pack(‘i’,1111111111111)
struct.error: ‘i’ format requires -2147483648 <= number <= 2147483647 #這個是範圍
import json,struct
#假設通過客戶端上傳1T:1073741824000的檔案a.txt
#為避免粘包,必須自定製報頭
header={'file_size':1073741824000,'file_name':'/a/b/c/d/e/a.txt','md5':'8f6fbf8347faa4924a76856701edb0f3'} #1T資料,檔案路徑和md5值
#為了該報頭能傳送,需要序列化並且轉為bytes
head_bytes=bytes(json.dumps(header),encoding='utf-8') #序列化並轉成bytes,用於傳輸
#為了讓客戶端知道報頭的長度,用struck將報頭長度這個數字轉成固定長度:4個位元組
head_len_bytes=struct.pack('i',len(head_bytes)) #這4個位元組裡只包含了一個數字,該數字是報頭的長度
#客戶端開始傳送
conn.send(head_len_bytes) #先發報頭的長度,4個bytes
conn.send(head_bytes) #再發報頭的位元組格式
conn.sendall(檔案內容) #然後發真實內容的位元組格式
#服務端開始接收
head_len_bytes=s.recv(4) #先收報頭4個bytes,得到報頭長度的位元組格式
x=struct.unpack('i',head_len_bytes)[0] #提取報頭的長度
head_bytes=s.recv(x) #按照報頭長度x,收取報頭的bytes格式
header=json.loads(json.dumps(header)) #提取報頭
#最後根據報頭的內容提取真實的資料,比如
real_data_len=s.recv(header['file_size'])
s.recv(real_data_len)關於struct的詳細用法
使用struct解決黏包
藉助struct模組，我們知道長度數字可以被轉換成一個標準大小的4位元組數字。因此可以利用這個特點來預先發送資料長度。
傳送時 接收時
先發送struct轉換好的資料長度4位元組 先接受4個位元組使用struct轉換成數字來獲取要接收的資料長度
再發送資料 再按照長度接收資料
import socket,struct,json
import subprocess
phone=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
phone.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) #就是它，在bind前加
phone.bind(('127.0.0.1',8080))
phone.listen(5)
while True:
conn,addr=phone.accept()
while True:
cmd=conn.recv(1024)
if not cmd:break
print('cmd: %s' %cmd)
res=subprocess.Popen(cmd.decode('utf-8'),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
err=res.stderr.read()
print(err)
if err:
back_msg=err
else:
back_msg=res.stdout.read()
conn.send(struct.pack('i',len(back_msg))) #先發back_msg的長度
conn.sendall(back_msg) #在發真實的內容
conn.close()#_*_coding:utf-8_*_
import socket,time,struct
s=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
res=s.connect_ex(('127.0.0.1',8080))
while True:
msg=input('>>: ').strip()
if len(msg) == 0:continue
if msg == 'quit':break
s.send(msg.encode('utf-8'))
l=s.recv(4)
x=struct.unpack('i',l)[0]
print(type(x),x)
# print(struct.unpack('I',l))
r_s=0
data=b''
while r_s < x:
r_d=s.recv(1024)
data+=r_d
r_s+=len(r_d)
# print(data.decode('utf-8'))
print(data.decode('gbk')) #windows預設gbk編碼我們還可以把報頭做成字典，字典裡包含將要傳送的真實資料的詳細資訊，然後json序列化，然後用struck將序列化後的資料長度打包成4個位元組（4個自己足夠用了）
傳送時 接收時
先發報頭長度
先收報頭長度，用struct取出來
再編碼報頭內容然後傳送 根據取出的長度收取報頭內容，然後解碼，反序列化
最後發真實內容 從反序列化的結果中取出待取資料的詳細資訊，然後去取真實的資料內容
import socket,struct,json
import subprocess
phone=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
phone.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) #就是它，在bind前加
phone.bind(('127.0.0.1',8080))
phone.listen(5)
while True:
conn,addr=phone.accept()
while True:
cmd=conn.recv(1024)
if not cmd:break
print('cmd: %s' %cmd)
res=subprocess.Popen(cmd.decode('utf-8'),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
err=res.stderr.read()
print(err)
if err:
back_msg=err
else:
back_msg=res.stdout.read()
headers={'data_size':len(back_msg)}
head_json=json.dumps(headers)
head_json_bytes=bytes(head_json,encoding='utf-8')
conn.send(struct.pack('i',len(head_json_bytes))) #先發報頭的長度
conn.send(head_json_bytes) #再發報頭
conn.sendall(back_msg) #在發真實的內容
conn.close()from socket import *
import struct,json
ip_port=('127.0.0.1',8080)
client=socket(AF_INET,SOCK_STREAM)
client.connect(ip_port)
while True:
cmd=input('>>: ')
if not cmd:continue
client.send(bytes(cmd,encoding='utf-8'))
head=client.recv(4)
head_json_len=struct.unpack('i',head)[0]
head_json=json.loads(client.recv(head_json_len).decode('utf-8'))
data_len=head_json['data_size']
recv_size=0
recv_data=b''
while recv_size < data_len:
recv_data+=client.recv(1024)
recv_size+=len(recv_data)
print(recv_data.decode('utf-8'))
#print(recv_data.decode('gbk')) #windows預設gbk編碼2，上傳下載檔案
server端
import json
import struct
import socket
sk = socket.socket()
sk.bind(('127.0.0.1',8080))
sk.listen()
conn,addr = sk.accept()
dic_len = conn.recv(4) # 4個位元組 數字的大小
dic_len = struct.unpack('i',dic_len)[0]
content = conn.recv(dic_len).decode('utf-8') # 70
content_dic = json.loads(content)
if content_dic['operate'] == 'upload':
with open(content_dic['filename'],'wb') as f:
while content_dic['filesize']:
file = conn.recv(1024)
f.write(file)
content_dic['filesize'] -= len(file)
conn.close()
sk.close()client端
import os
import json
import struct
import socket
sk = socket.socket()
sk.connect(('127.0.0.1',8080))
def get_filename(file_path):
filename = os.path.basename(file_path)
return filename
#選擇 操作
operate = ['upload','download']
for num,opt in enumerate(operate,1):
print(num,opt)
num = int(input('請輸入您要做的操作序號 : '))
if num == 1:
'''上傳操作'''
file_path = input('請輸入要上傳的檔案路徑 : ')
file_size = os.path.getsize(file_path) # 獲取檔案大小
file_name = get_filename(file_path)
dic = {'operate': 'upload', 'filename': file_name,'filesize':file_size}
str_dic = json.dumps(dic).encode('utf-8')
ret = struct.pack('i', len(str_dic)) # 將字典的大小轉換成一個定長(4)的bytes
sk.send(ret + str_dic)
with open(file_path,'rb') as f:
while file_size:
content = f.read(1024)
sk.send(content)
file_size -= len(content)
elif num == 2:
'''下載操作'''
sk.close()3，FTP作業：上傳下載檔案
import socket
import struct
import json
import subprocess
import os
class MYTCPServer:
address_family = socket.AF_INET
socket_type = socket.SOCK_STREAM
allow_reuse_address = False
max_packet_size = 8192
coding='utf-8'
request_queue_size = 5
server_dir='file_upload'
def __init__(self, server_address, bind_and_activate=True):
"""Constructor. May be extended, do not override."""
self.server_address=server_address
self.socket = socket.socket(self.address_family,
self.socket_type)
if bind_and_activate:
try:
self.server_bind()
self.server_activate()
except:
self.server_close()
raise
def server_bind(self):
"""Called by constructor to bind the socket.
"""
if self.allow_reuse_address:
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.bind(self.server_address)
self.server_address = self.socket.getsockname()
def server_activate(self):
"""Called by constructor to activate the server.
"""
self.socket.listen(self.request_queue_size)
def server_close(self):
"""Called to clean-up the server.
"""
self.socket.close()
def get_request(self):
"""Get the request and client address from the socket.
"""
return self.socket.accept()
def close_request(self, request):
"""Called to clean up an individual request."""
request.close()
def run(self):
while True:
self.conn,self.client_addr=self.get_request()
print('from client ',self.client_addr)
while True:
try:
head_struct = self.conn.recv(4)
if not head_struct:break
head_len = struct.unpack('i', head_struct)[0]
head_json = self.conn.recv(head_len).decode(self.coding)
head_dic = json.loads(head_json)
print(head_dic)
#head_dic={'cmd':'put','filename':'a.txt','filesize':123123}
cmd=head_dic['cmd']
if hasattr(self,cmd):
func=getattr(self,cmd)
func(head_dic)
except Exception:
break
def put(self,args):
file_path=os.path.normpath(os.path.join(
self.server_dir,
args['filename']
))
filesize=args['filesize']
recv_size=0
print('----->',file_path)
with open(file_path,'wb') as f:
while recv_size < filesize:
recv_data=self.conn.recv(self.max_packet_size)
f.write(recv_data)
recv_size+=len(recv_data)
print('recvsize:%s filesize:%s' %(recv_size,filesize))
tcpserver1=MYTCPServer(('127.0.0.1',8080))
tcpserver1.run()
#下列程式碼與本題無關
class MYUDPServer:
"""UDP server class."""
address_family = socket.AF_INET
socket_type = socket.SOCK_DGRAM
allow_reuse_address = False
max_packet_size = 8192
coding='utf-8'
def get_request(self):
data, client_addr = self.socket.recvfrom(self.max_packet_size)
return (data, self.socket), client_addr
def server_activate(self):
# No need to call listen() for UDP.
pass
def shutdown_request(self, request):
# No need to shutdown anything.
self.close_request(request)
def close_request(self, request):
# No need to close anything.
passimport socket
import struct
import json
import os
class MYTCPClient:
address_family = socket.AF_INET
socket_type = socket.SOCK_STREAM
allow_reuse_address = False
max_packet_size = 8192
coding='utf-8'
request_queue_size = 5
def __init__(self, server_address, connect=True):
self.server_address=server_address
self.socket = socket.socket(self.address_family,
self.socket_type)
if connect:
try:
self.client_connect()
except:
self.client_close()
raise
def client_connect(self):
self.socket.connect(self.server_address)
def client_close(self):
self.socket.close()
def run(self):
while True:
inp=input(">>: ").strip()
if not inp:continue
l=inp.split()
cmd=l[0]
if hasattr(self,cmd):
func=getattr(self,cmd)
func(l)
def put(self,args):
cmd=args[0]
filename=args[1]
if not os.path.isfile(filename):
print('file:%s is not exists' %filename)
return
else:
filesize=os.path.getsize(filename)
head_dic={'cmd':cmd,'filename':os.path.basename(filename),'filesize':filesize}
print(head_dic)
head_json=json.dumps(head_dic)
head_json_bytes=bytes(head_json,encoding=self.coding)
head_struct=struct.pack('i',len(head_json_bytes))
self.socket.send(head_struct)
self.socket.send(head_json_bytes)
send_size=0
with open(filename,'rb') as f:
for line in f:
self.socket.send(line)
send_size+=len(line)
print(send_size)
else:
print('upload successful')
client=MYTCPClient(('127.0.0.1',8080))
client.run()