1. 程式人生 > >C++ 多執行緒呼叫Python指令碼

C++ 多執行緒呼叫Python指令碼

由於Python直譯器有全域性解釋所GIL的原因,導致在同一時刻只能有一個執行緒擁有直譯器,所以在C++多執行緒呼叫python指令碼時,需要控制GIL,執行緒獲取GIL。

在主執行緒中初始化Python直譯器環境,程式碼如下:

		{
			Py_Initialize(); //初始化Python環境
			if ( !Py_IsInitialized() ) //檢測是否初始化成功
			{
				return NULL;
			}
			else
			{
				PyEval_InitThreads();     //開啟多執行緒支援
				int nInit = PyEval_ThreadsInitialized();  //檢測執行緒支援是否開啟成功
				if ( nInit )
				{
					PyEval_SaveThread();  //因為呼叫PyEval_InitThreads成功後,當前執行緒就擁有了GIL,釋放當前執行緒的GIL,
				}
			}
		}
然後可以建立子執行緒,在子執行緒呼叫中 加入如下程式碼:
int nHold = PyGILState_Check() ;   //檢測當前執行緒是否擁有GIL
	PyGILState_STATE gstate;
	if ( !nHold )
	{
		gstate = PyGILState_Ensure();   //如果沒有GIL,則申請獲取GIL
	}
	Py_BEGIN_ALLOW_THREADS;
	Py_BLOCK_THREADS;
/**************************以下加入需要呼叫的python指令碼程式碼  Begin***********************/
/**************************以下加入需要呼叫的python指令碼程式碼  End***********************/
	Py_UNBLOCK_THREADS;
	Py_END_ALLOW_THREADS;
	if (!nHold)
	{
		PyGILState_Release(gstate);    //釋放當前執行緒的GIL
	}

boost::python對Python原生的C API有較好的封裝,可以很方便通過C++對python指令碼進行呼叫一下是一些學習總結,以便今後需要使用的時候能快速使用
假設有python模組   School.py  包含類 Student   和 函式  Call  和返回類物件的CallClass方法

class Student():
	def __init__(name,age,properties):
		self.name = name
		self.age = age
		self.properties = proterties

	def show():
		print(self.name, slef.age, self.properties)

def Call(p1,p2):
	print(p1,p2)
	return "yes"

def CallClass():
	s = Student("universal",20,"good")
	return s
以下為C++呼叫python指令碼程式碼:
1>  在C++中構造python中定義的類,並且呼叫類方法
boost::python::handle<>* school_module = NULL;
string strModule = "School";
school_module = new boost::python::handle<>(PyImport_ImportModule((char*)strModule.c_str()));    //通過模組名稱獲取模組handle
boost::python::object main_module(*manager_module);     //轉化為boost::python::object型別表示的模組 
boost::python::object main_namespace = main_module.attr("__dict__");   //得到模組內的字典,字典中包含了 所有的類和函式
boost::python::handle<> Student;    //定義一個Student類 的handle
//得到Student類的例項,引數可以任意傳遞  比如以下設定properties為一個dict
Student = boost::python::handle<>((PyRun_String(
"Student('alen',30,{\"errorcode\":0, \"message\":\"execute succeed\"})", Py_eval_input,
main_namespace.ptr(), main_namespace.ptr())));

/*********以下將properties設定為一個tuple
Student = boost::python::handle<>((PyRun_String(
"Student('alen',30,(1,2,3,4,5))", Py_eval_input,
main_namespace.ptr(), main_namespace.ptr())));
**********/

/*********以下將properties設定為一個list
Student = boost::python::handle<>((PyRun_String(
"Student('alen',30,[1,2,3,4,5])", Py_eval_input,
main_namespace.ptr(), main_namespace.ptr())));
**********/
/**********
//通過Student類呼叫類方法show
boost::python::object result = boost::python::call_method<boost::python::object>(Student.get(), "show");

2>呼叫指令碼中的函式 Call   假設傳遞的第一個引數為string  第二個為int

boost::python::handle<>* school_module = NULL;
string strModule = "School";
school_module = new boost::python::handle<>(PyImport_ImportModule((char*)strModule.c_str()));    //通過模組名稱獲取模組handle
boost::python::object result = boost::python::call_method<boost::python::object>(school_module->get(),"Call","schoolname",2);

3>呼叫指令碼時,可以傳遞各種引數,boost 已經封裝了boost::python::dict 對應dict       boost::python::tuple對應 tuple

比如構造dict引數,程式碼如下:

boost::python::dict inParams;
inParams.setdefault<std::string,std::string>("name","clear");
inParams.setdefault<std::string,std::string>("code","56821334");
inParams.setdefault<std::string,int>("number",123456);


4>處理返回的資料比如 python類型別
boost::python::handle<>* school_module = NULL;
string strModule = "School";
school_module = new boost::python::handle<>(PyImport_ImportModule((char*)strModule.c_str()));    //通過模組名稱獲取模組handle
boost::python::object student = boost::python::call_method<boost::python::object>(school_module->get(),"CallClass");
//可以通過返回的類物件 繼續呼叫類Student 的 show方法 如下:
boost::python::object q = boost::python::call_method<boost::python::object>(student.ptr(), "show");
//也可以獲取類的屬性值 比如 name  string 型別  和 age  Int型別
boost::python::object obj_dict  = student.attr("__dict__");   //獲取物件的字典
//獲取name
boost::python::object name = obj_dict["name"];
std::string sname = boost::python::extract<std::string>(name);
//獲取age
boost::python::object age = obj_dict["age"];
int age = boost::python::extract<int>(age);
將全域性直譯器鎖和執行緒的相關操作用類封裝,使用建構函式和解構函式 去做locker 和 unlocker的操作
/*全域性解釋和執行緒鎖*/
class PyGILThreadLock
{
public:
	PyGILThreadLock()
	{
		_save = NULL;
		nStatus = 0;
		nStatus = PyGILState_Check() ;   //檢測當前執行緒是否擁有GIL  
		PyGILState_STATE gstate;  
		if ( !nStatus )  
		{  
			gstate = PyGILState_Ensure();   //如果沒有GIL,則申請獲取GIL 
			nStatus = 1;
		}  
		_save = PyEval_SaveThread();
		PyEval_RestoreThread(_save);
	}
	~PyGILThreadLock()
	{
		_save = PyEval_SaveThread();
		PyEval_RestoreThread(_save);
		if (nStatus)  
		{  
			PyGILState_Release(gstate);    //釋放當前執行緒的GIL  
		}  
	}

private:
	PyGILState_STATE gstate; 
	PyThreadState *_save;
	int nStatus;
};
在程式碼中呼叫時,只需要
{
        PyGILThreadLock  locker;
///////以下新增python呼叫程式碼
}

使用boost::python的時候捕獲異常獲取異常資訊

std::string GetPythonErrorInfo(void)  
{   
	using namespace boost::python;                                  

	PyObject *exc,*val,*tb;                                         
	PyErr_Fetch(&exc,&val,&tb);                                     
	PyErr_NormalizeException(&exc,&val,&tb);                        
	handle<> hexc(exc),hval(allow_null(val)),htb(allow_null(tb));   
	if(!hval)                                                       
	{
		return extract<std::string>(str(hexc));                       
	}                                                               
	else                                                            
	{                                                               
		object traceback(import("traceback"));                        
		object format_exception(traceback.attr("format_exception"));  
		object formatted_list(format_exception(hexc,hval,htb));       
		object formatted(str("").join(formatted_list));               
		return extract<std::string>(formatted);                       
	}                                                        
}   


以下為VC控制檯測試程式
#include "stdafx.h"
#include <Windows.h>
#include <iostream>
#include <process.h>
#include <boost/python.hpp>
using namespace std;

#define Check_Init(init)  \
	if(!init){\
	cout<<"請先通過init指令初始化python環境"<<endl;\
	goto __next;\
	}


bool bInit = false;
std::string strModule = "manager";

unsigned int __stdcall ThreadCall(void* lparam)
{
	char szMessage[1024] = { 0 };
	int id  = GetCurrentThreadId();
	sprintf_s(szMessage,sizeof(szMessage),"thread id: %d ",id);
	std::string sResult = "";
	boost::python::handle<>* manager_module = NULL;
	for ( int i = 0 ; i < 2000 ; i ++)
	{
		PyGILThreadLock locker;
		manager_module = new boost::python::handle<>(PyImport_ImportModule((char*)strModule.c_str()));
		sResult = boost::python::call_method<std::string>(manager_module->get(),"threadcall",szMessage);
		cout<<sResult.c_str()<<endl;

		delete manager_module;
		manager_module = NULL;
	}
	return 0;
}

void callsay(const char* szMessage)
{
	boost::python::handle<>* manager_module = NULL;
	std::string sResult = "";
	PyGILThreadLock locker;
	try
	{	
		manager_module = new boost::python::handle<>(PyImport_ImportModule((char*)strModule.c_str()));
		sResult = boost::python::call_method<std::string>(manager_module->get(),"say",szMessage);
	}
	catch(...)
	{
		delete manager_module;
		manager_module = NULL;
		cout<<"捕獲一個異常在callsay方法中"<<endl;
	}
	delete manager_module;
	manager_module = NULL;
}

int _tmain(int argc, _TCHAR* argv[])
{
	char szCommand[20]={0}  ;
	cout<<"請輸入指令"<<endl;
	cin>>szCommand;
	while ( strcmp("quit",szCommand) != 0 )
	{
		if ( strcmp("threadcall",szCommand) == 0 )
		{
			Check_Init(bInit);
			for ( int i = 0 ; i < 5; i++ )
			{
				_beginthreadex(NULL,
					0,
					ThreadCall,
					NULL,
					0,
					NULL);
			}
		}
		else if ( strcmp("init",szCommand) == 0 )
		{
			if ( bInit )
			{
				cout<<"Python 環境已經初始化"<<endl;
				continue;
			}
			Py_Initialize(); //初始化Python環境
			if ( !Py_IsInitialized() ) //檢測是否初始化成功
			{
				cout<<"Python環境初始化失敗"<<endl;
				return NULL;
			}
			else
			{
				PyEval_InitThreads();     //開啟多執行緒支援
				int nInit = PyEval_ThreadsInitialized();  //檢測執行緒支援是否開啟成功
				if ( nInit )
				{
					PyEval_SaveThread();  //因為呼叫PyEval_InitThreads成功後,當前執行緒就擁有了GIL,釋放當前執行緒的GIL,
				}
				bInit = true;
				cout<<"Python環境初始化成功,退出前請使用free指令釋放Python環境"<<endl;
			}	
		}
		else if ( strcmp("say",szCommand) == 0 )
		{
			Check_Init(bInit);
			std::string msg = "Python Manager";
			callsay(msg.c_str());
		}
		else if ( strcmp("free",szCommand) == 0 )
		{
			Py_Finalize();
			bInit = false;
			cout<<"Python直譯器環境被釋放"<<endl;
		}
		else{
			cout<<"未識別的指令資訊"<<endl;
			
		}
__next:
		memset(szCommand,0,20);
		cin>>szCommand;
	}
	system("pause");
	return 0;
}

新增一個測試過程中的程式碼, 因為還封裝了類所以單獨編譯不過,用來標記一下吧 ,方便下次在使用的時候可以快速的重新學習

// PythonManager.cpp : Defines the entry point for the console application.
//

#include "stdafx.h"
#include <Windows.h>
#include <iostream>
#include <process.h>
// #include <boost/python.hpp>
// #include "Python.h"
#include "PythonAssister.h"
using namespace std;

#define Check_Init(init)  \
	if(!init){\
	cout<<"請先通過init指令初始化python環境"<<endl;\
	goto __next;\
	}


bool bInit = false;
std::string strModule = "manager";

unsigned int __stdcall ThreadCall(void* lparam)
{
	char szMessage[1024] = { 0 };
	int id  = GetCurrentThreadId();
	sprintf_s(szMessage,sizeof(szMessage),"thread id: %d ",id);
	std::string sResult = "";
	boost::python::handle<>* manager_module = NULL;
	for ( int i = 0 ; i < 2000 ; i ++)
	{
		PyGILThreadLock locker;
// 		int nHold = PyGILState_Check() ;   //檢測當前執行緒是否擁有GIL  
// 		PyGILState_STATE gstate;  
// 		if ( !nHold )  
// 		{  
// 			gstate = PyGILState_Ensure();   //如果沒有GIL,則申請獲取GIL  
// 		}  
// 		Py_BEGIN_ALLOW_THREADS;  
// 		Py_BLOCK_THREADS; 
		manager_module = new boost::python::handle<>(PyImport_ImportModule((char*)strModule.c_str()));
		sResult = boost::python::call_method<std::string>(manager_module->get(),"threadcall",szMessage);
		cout<<sResult.c_str()<<endl;
// 		Py_UNBLOCK_THREADS;  
// 		Py_END_ALLOW_THREADS;  
// 		if (!nHold)  
// 		{  
// 			PyGILState_Release(gstate);    //釋放當前執行緒的GIL  
// 		} 
	}
	return 0;
}


void calltest()
{
	PyGILThreadLock locker;
	boost::python::handle<>* manager_module = NULL;
	manager_module = new boost::python::handle<>(PyImport_ImportModule((char*)strModule.c_str()));
	boost::python::object result = boost::python::call_method<boost::python::object>(manager_module->get(),"calltest");
	cout<< result.ptr()->ob_type->tp_name<<endl;
	boost::python::object q = boost::python::call_method<boost::python::object>(result.ptr(), "show");
	boost::python::object r1  = result.attr("__dict__");
	boost::python::object r2 = r1["name"];
//	boost::python::call_method<boost::python::object>(r1,"__getdict__","name");
	std::string sname = boost::python::extract<std::string>(r2);
	cout<<sname.c_str()<<endl;
	delete manager_module;
	manager_module = NULL;

}

void classtest()
{
	PyGILThreadLock locker;
	boost::python::handle<>* manager_module = NULL;
	try
	{
		manager_module = new boost::python::handle<>(PyImport_ImportModule((char*)strModule.c_str()));
		boost::python::object main_module(*manager_module);
		boost::python::object main_namespace = main_module.attr("__dict__");
		boost::python::handle<> Student;
		Student = boost::python::handle<>((PyRun_String(
			"Student('alen',30,{\"errorcode\":0, \"message\":\"execute succeed\"})", Py_eval_input,
			main_namespace.ptr(), main_namespace.ptr())));
		boost::python::object result = boost::python::call_method<boost::python::object>(Student.get(), "show");

	}
	catch(...)
	{
		PyErr_Print();
		PyErr_Clear();
	}

}

 void calldict()
 {
 	boost::python::handle<>* manager_module = NULL;
 	boost::python::dict mydict;
 	int nHold = PyGILState_Check() ;   //檢測當前執行緒是否擁有GIL  
 	PyGILState_STATE gstate;  
 	if ( !nHold )  
 	{  
 		gstate = PyGILState_Ensure();   //如果沒有GIL,則申請獲取GIL  
 	}  
 	Py_BEGIN_ALLOW_THREADS;  
 	Py_BLOCK_THREADS; 
 	try
 	{	
 		boost::python::dict inParams;
 		inParams.setdefault<std::string,std::string>("name","clear");
 		inParams.setdefault<std::string,std::string>("code","56821334");
 		inParams.setdefault<std::string,int>("number",123456);
 		int nLenDict = len(inParams);
 		const char* szType = inParams.ptr()->ob_type->tp_name;
 		boost::python::list dictlist;
 		dictlist = inParams.items();
 		int nLenList = len(dictlist);
 		for ( int i = 0 ; i< nLenList; i++ )
 		{
 			boost::python::object obj;
 			
 			obj = dictlist.pop();
 			const char* szType = obj.ptr()->ob_type->tp_name;
 			int n = PyType_Check(obj.ptr());
 			int nLenTuple = len(obj);
 			for ( int j = 0 ; j <nLenTuple; j++ )
 			{
 				int n1 = PyType_Check(((boost::python::object)obj[j]).ptr());
 				std::string value = boost::python::extract<std::string>(obj[j]);
 				
 				boost::python::extract<const std::string&>ex((obj[j]));
 				if ( ex.check() )
 				{
 					cout<<"yes it is string"<<endl;
 				}
 				cout<<value.c_str()<<endl;
 			}
 		}
 		//boost::python::tuple
  		boost::python::object itemTuple;
  		itemTuple = dictlist.pop();
  		while ( itemTuple != NULL )
  		{
  			int nLen = len(itemTuple);
  			string strKey = boost::python::extract<std::string>(itemTuple[0]);
  			string strValue = boost::python::extract<std::string>(itemTuple[1]);
  			itemTuple = dictlist.pop();
  			int a = 10;
  		}
 		
 
 
 		manager_module = new boost::python::handle<>(PyImport_ImportModule((char*)strModule.c_str()));
 		//mydict = boost::python::call_method<boost::python::dict>(manager_module->get(),"calldict",inParams);
 		boost::python::object result = boost::python::call_method<boost::python::object>(manager_module->get(),"calldict",inParams);
 		const char* szType1 = result.ptr()->ob_type->tp_name;
 		cout<<"Type : "<<szType1<<endl;
 		
 	}
 	catch(...)
 	{
 // 		delete manager_module;
 // 		manager_module = NULL;
 		cout<<"捕獲一個異常在calldict方法中"<<endl;
 	}
 	if ( NULL != manager_module )
 	{
 		delete manager_module;
 		manager_module = NULL;
 	}
 
 	Py_UNBLOCK_THREADS;  
 	Py_END_ALLOW_THREADS;  
 	if (!nHold)  
 	{  
 		PyGILState_Release(gstate);    //釋放當前執行緒的GIL  
 	}  
}



void callsay(const char* szMessage)
{
	boost::python::handle<>* manager_module = NULL;
	std::string sResult = "";
	int nHold = PyGILState_Check() ;   //檢測當前執行緒是否擁有GIL  
	PyGILState_STATE gstate;  
	if ( !nHold )  
	{  
		gstate = PyGILState_Ensure();   //如果沒有GIL,則申請獲取GIL  
	}  
	Py_BEGIN_ALLOW_THREADS;  
	Py_BLOCK_THREADS; 
	try
	{	
		manager_module = new boost::python::handle<>(PyImport_ImportModule((char*)strModule.c_str()));
		sResult = boost::python::call_method<std::string>(manager_module->get(),"say",szMessage);
	}
	catch(...)
	{
		delete manager_module;
		manager_module = NULL;
		cout<<"捕獲一個異常在callsay方法中"<<endl;
	}
	delete manager_module;
	manager_module = NULL;
	Py_UNBLOCK_THREADS;  
	Py_END_ALLOW_THREADS;  
	if (!nHold)  
	{  
		PyGILState_Release(gstate);    //釋放當前執行緒的GIL  
	}  
}

int _tmain(int argc, _TCHAR* argv[])
{
	char szCommand[20]={0}  ;
	cout<<"請輸入指令"<<endl;
	cin>>szCommand;
	while ( strcmp("quit",szCommand) != 0 )
	{
		if ( strcmp("threadcall",szCommand) == 0 )
		{
			Check_Init(bInit);
			for ( int i = 0 ; i < 5; i++ )
			{
				_beginthreadex(NULL,
					0,
					ThreadCall,
					NULL,
					0,
					NULL);
			}
		}
		else if ( strcmp("classtest",szCommand) == 0 )
		{
			classtest();
		}
		else if ( strcmp("init",szCommand) == 0 )
		{
			if ( bInit )
			{
				cout<<"Python 環境已經初始化"<<endl;
				continue;
			}
			Py_Initialize(); //初始化Python環境
			if ( !Py_IsInitialized() ) //檢測是否初始化成功
			{
				cout<<"Python環境初始化失敗"<<endl;
				return NULL;
			}
			else
			{
				PyEval_InitThreads();     //開啟多執行緒支援
				int nInit = PyEval_ThreadsInitialized();  //檢測執行緒支援是否開啟成功
				if ( nInit )
				{
					PyEval_SaveThread();  //因為呼叫PyEval_InitThreads成功後,當前執行緒就擁有了GIL,釋放當前執行緒的GIL,
				}
				bInit = true;
				cout<<"Python環境初始化成功,退出前請使用free指令釋放Python環境"<<endl;
			}	
		}
		else if ( strcmp("say",szCommand) == 0 )
		{
			Check_Init(bInit);
			std::string msg = "Python Manager";
			callsay(msg.c_str());
		}
		else if ( strcmp("calldict",szCommand) == 0 )
		{
			Check_Init(bInit);
			std::string msg = "Python Manager";
			calldict();
		}
		else if ( strcmp("free",szCommand) == 0 )
		{
			int nHold = PyGILState_Check() ;   //檢測當前執行緒是否擁有GIL  
			PyGILState_STATE gstate;  
			if ( !nHold )  
			{  
				gstate = PyGILState_Ensure();   //如果沒有GIL,則申請獲取GIL  
			}  
			Py_Finalize();
			bInit = false;
			cout<<"Python直譯器環境被釋放"<<endl;
		}
		else{
			cout<<"未識別的指令資訊"<<endl;
			
		}
__next:
		memset(szCommand,0,20);
		cin>>szCommand;
	}
	system("pause");
	return 0;
}