Develop Reference

Segmentation fault on gc , using ctypes

I am calling a native c library using ctypes, passing in a specialiezed struct and getting back another struct. I get a seg fault when gc is called, can't find much details in gdb trace:
Thread 1 "python" received signal SIGSEGV, Segmentation fault.
0x00007ffff7a5ca44 in visit_decref (op=0xf5ee10, data=0x0) at Modules/gcmodule.c:374
374 Modules/gcmodule.c: No such file or directory.
gdb) where
#0 0x00007ffff7a5ca44 in visit_decref (op=0xf5ee10, data=0x0) at Modules/gcmodule.c:374
#1 0x00007ffff798bb0c in subtype_traverse (self=0x7fff7744fd90, visit=0x7ffff7a5ca40 <visit_decref>, arg=0x0) at Objects/typeobject.c:1011
#2 0x00007ffff7a5bd77 in subtract_refs (containers=<optimized out>) at Modules/gcmodule.c:399
#3 collect (generation=generation#entry=0, n_collected=n_collected#entry=0x7fffffffb708, n_uncollectable=n_uncollectable#entry=0x7fffffffb710, nofail=nofail#entry=0) at Modules/gcmodule.c:956
#4 0x00007ffff7a5c95d in collect_with_callback (generation=0) at Modules/gcmodule.c:1128
#5 0x00007ffff7a5d1eb in collect_generations () at Modules/gcmodule.c:1151
#6 _PyObject_GC_Alloc (basicsize=<optimized out>, use_calloc=0) at Modules/gcmodule.c:1726
#7 _PyObject_GC_Malloc (basicsize=<optimized out>) at Modules/gcmodule.c:1736
The code sample is this:
_C_DOUBLE_P = POINTER(c_double)
_C_INT_P = POINTER(c_int)
class _IN_DATA(Structure):
_fields_ = [('in_kps', _C_DOUBLE_P),
('in_desc', _C_DOUBLE_P)
]
def __init__(self):
t = c_int
self.test = ctypes.cast(t, POINTER(c_int))
class _OUT_DATA(Structure):
_fields_ = [
('num_out_kps', ctypes.c_int),
('out_kps', _C_DOUBLE_P),
('out_desc', _C_DOUBLE_P)
]
class _IN_DATA_LIST(ctypes.Structure):
_fields_ = [
('num_crops', c_int),
('crops', POINTER(_IN_DATA))
]
def __init__(self, crops_raw_kps: List[np.ndarray], crops_raw_descriptors: List[np.ndarray]):
num_crops = len(crops_raw_kps)
self.num_crops = num_crops
crops = (POINTER(_IN_DATA) * num_crops)()
self.crops = ctypes.cast(crops, POINTER(_IN_DATA))
for i in range(num_crops):
self.crops[i].in_kps = crops_raw_kps[i].ctypes.data_as(_C_DOUBLE_P)
self.crops[i].in_desc = crops_raw_descriptors[i].ctypes.data_as(_C_DOUBLE_P)
class _OUT_DATA_LIST(ctypes.Structure):
_fields_ = [
('crops_data', ctypes.POINTER(_OUT_DATA)),
('num_results', c_int)
]
class SPPostWrapper:
def __init__(self):
self._post_processor_lib = ctypes.cdll.LoadLibrary("multitracker/custom_features/build/ffme/libffme.so")
self._post_processor_lib.py_postprocess.restype = _OUT_DATA_LIST
self._post_processor_lib.py_postprocess.argtypes = [POINTER(_IN_DATA_LIST)]
def post_process_multi(self, crops_raw_kps: List[np.ndarray], crops_raw_descriptors: List[np.ndarray]):
num_crops = len(crops_raw_kps)
adjusted_kps = [np.asarray(np.squeeze(kp), np.double) for kp in crops_raw_kps]
adjusted_desc = [np.asarray(np.squeeze(desc), np.double) for desc in crops_raw_descriptors]
crops_struct = _IN_DATA_LIST(adjusted_kps, adjusted_desc)
out_result= self._post_processor_lib.py_postprocess(ctypes.byref(crops_struct))
I allocate the IN_DATA in python, and the OUT_DATA is allocated in the c code , I tried using cache in c (assuming python will not clean the memory) or allocating new memory for each call (assuming python does free the out_data memory) - both methods fail when gc of python is called.
Update:
To better isolate the problem I removed the usage of out_data, setting the method to void, and still the problem happens. I also tried keeping the in data as a member and i think this prevents the data from happening, until the process shutdown. So it must be something to do with the memory I allocated to input/input list.
Update 2:
I was able to verify that the problem only happens if I pass more than 1 item in the IN_DATA_LIST (more than 1 crop). This is very strange indeed...

After validating that the problem happens even if the c code does nothing and returns nothing, and not being able to solve it, I ended up using a more shallow input (and output) , this seems to run ok, without seg faults:
class _IN_DATA(Structure):
_fields_ = [
('num_crops', c_int),
('in_kps', _C_DOUBLE_P),
('in_desc', _C_DOUBLE_P)
]
def __init__(self, kps_flat, desc_falt, num_crops):
self.num_crops = num_crops
self.in_kps = kps_flat.ctypes.data_as(_C_DOUBLE_P)
self.in_desc = desc_falt.ctypes.data_as(_C_DOUBLE_P)
class _OUT_DATA(Structure):
_fields_ = [
('num_results', ctypes.c_int),
('results_per_crop', _C_INT_P),
('out_kps', _C_DOUBLE_P), # x,y, score
('out_desc', _C_DOUBLE_P)
]

Convert memory address from POINTER(BYTE) type to hex

I'm trying to find the base address of render.dll used in the process process.exe. I have modified the code in this question, and I'm able to get a result for the base address of render.dll.
from ctypes import *
from ctypes.wintypes import *
import psutil
class MODULEENTRY32(Structure):
_fields_ = [( 'dwSize', DWORD),
( 'th32ModuleID', DWORD),
( 'th32ProcessID', DWORD),
( 'GlblcntUsage', DWORD),
( 'ProccntUsage', DWORD),
( 'modBaseAddr', POINTER(BYTE)),
( 'modBaseSize', DWORD),
( 'hModule', HMODULE),
( 'szModule', c_char * 256),
( 'szExePath', c_char * 260)]
CreateToolhelp32Snapshot = windll.kernel32.CreateToolhelp32Snapshot
Module32First = windll.kernel32.Module32First
Module32Next = windll.kernel32.Module32Next
CloseHandle = windll.kernel32.CloseHandle
TH32CS_SNAPMODULE = 0x00000008
TH32CS_SNAPMODULE32 = 0x00000010
def getpid(processname):
for proc in psutil.process_iter():
if str(processname) in str(proc.name):
return proc.pid
def GetModuleByName(name):
snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPMODULE32 | TH32CS_SNAPMODULE, getpid('process.exe'))
entry = MODULEENTRY32()
entry.dwSize = sizeof(MODULEENTRY32)
if Module32First(snapshot, pointer(entry)):
while Module32Next(snapshot, entry):
if entry.szModule == name:
CloseHandle(snapshot)
return entry.modBaseAddr
CloseHandle(snapshot)
return None
baseAddr = GetModuleByName('render.dll')
print baseAddr
Which results in <__main__.LP_c_byte object at 0x00000000023C9348>. I know this is to do with the result being a POINTER(BYTE) type, but I am not sure how to get from this type to a normal hex memory address that I can use to read the process memory at this location.

I found an answer here. Turns out I had the same misunderstanding as the other poster and that using ctypes.addressof(baseAddr.contents) returns the correct memory address for the .dll.

Error using GetExtendedTcpTable in python

I've got some troubles with using "GetExtendedTcpTable". When I tried to run my script, i've got message like this:
AssertionError: [Error 0] The operation completed successfully
Rarely script working normally, I dont understand this message, Operation completed, what`s wrong?
This is code, i tried to execute:
from ctypes import *
from ctypes.wintypes import *
from socket import inet_aton, inet_ntoa, htons
AF_INET = 2
TCP_TABLE_BASIC_LISTENER = 0
TCP_TABLE_BASIC_CONNECTIONS = 1
TCP_TABLE_BASIC_ALL = 2
TCP_TABLE_OWNER_PID_LISTENER = 3
TCP_TABLE_OWNER_PID_CONNECTIONS = 4
TCP_TABLE_OWNER_PID_ALL = 5
TCP_TABLE_OWNER_MODULE_LISTENER = 6
TCP_TABLE_OWNER_MODULE_CONNECTIONS = 7
TCP_TABLE_OWNER_MODULE_ALL = 8
# for storing socket info python style.
class socket_info:
State = None
LocalAddr = None
LocalPort = None
RemoteAddr = None
RemotePort = None
def __init__ (self, **kwargs):
for key, word in kwargs.items():
setattr(self, key, word)
def formatip (ip):
ip = inet_aton (str(ip))
return inet_ntoa (ip[::-1])
states = {
1 : "TCP_STATE_CLOSED",
2 : "TCP_STATE_LISTEN",
3 : "TCP_STATE_SYN_SENT",
4 : "TCP_STATE_SYN_RCVD",
5 : "TCP_STATE_ESTAB",
6 : "TCP_STATE_FIN_WAIT",
7 : "TCP_STATE_FIN_WAIT2",
8 : "TCP_STATE_CLOSE_WAIT",
9 : "TCP_STATE_CLOSING",
10 : "TCP_STATE_LAST_ACK",
11 : "TCP_STATE_TIME_WAIT",
12 : "TCP_STATE_DELETE_TCB",
"TCP_STATE_CLOSED" : 1,
"TCP_STATE_LISTEN" : 2,
"TCP_STATE_SYN_SENT" : 3,
"TCP_STATE_SYN_RCVD" : 4,
"TCP_STATE_ESTAB" : 5,
"TCP_STATE_FIN_WAIT" : 6,
"TCP_STATE_FIN_WAIT2" : 7,
"TCP_STATE_CLOSE_WAIT" : 8,
"TCP_STATE_CLOSING" : 9,
"TCP_STATE_LAST_ACK" :10,
"TCP_STATE_TIME_WAIT" : 11,
"TCP_STATE_DELETE_TCB" : 12 }
class MIB_TCPROW_OWNER_PID(Structure):
_fields_ = [
("dwState", DWORD),
("dwLocalAddr", DWORD),
("dwLocalPort", DWORD),
("dwRemoteAddr", DWORD),
("dwRemotePort", DWORD),
("dwOwningPid", DWORD)
]
class MIB_TCPTABLE_OWNER_PID(Structure):
_fields_ = [
("dwNumEntries", DWORD),
("MIB_TCPROW_OWNER_PID", MIB_TCPROW_OWNER_PID * 100)
]
def GetExtendedTcpTable (vip=AF_INET):
table = MIB_TCPTABLE_OWNER_PID ()
so = sizeof (table)
size = DWORD (so)
order = c_int(1)
failure= windll.iphlpapi.GetExtendedTcpTable (
byref (table),
addressof (size),
order,
vip,
TCP_TABLE_OWNER_PID_ALL,
0 )
assert not failure, WinError (GetLastError ())
pytables = []
tables = table.MIB_TCPROW_OWNER_PID
for index in range(table.dwNumEntries):
table = tables [index]
pytables.append (
socket_info (
State=states.get (table.dwState, "UNKNOWN_STATE_%s" %(str(table.dwState))),
LocalAddr=formatip (table.dwLocalAddr),
LocalPort=htons(table.dwLocalPort),
RemoteAddr=formatip (table.dwRemoteAddr),
RemotePort=htons(table.dwRemotePort),
OwningPid = int (table.dwOwningPid)
)
)
return pytables
def GetTcpTableForPid (pid):
tables = GetExtendedTcpTable ()
for table in tables:
if table.OwningPid == pid: return table
raise "Cannot find tcp table for pid %s" %pid
dict_process = {}
pid_set =set()
pid_list = []
tcp_info_list = []
tcp_info = GetExtendedTcpTable()
for item in tcp_info:
LocalAddr = item.LocalAddr
LocalPort = item.LocalPort
RemoteAddr = item.RemoteAddr
RemotePort = item.RemotePort
OwningPid = item.OwningPid
print('local Addr: '+ LocalAddr,'local port: '+ str(LocalPort),'remote Addr: ' + RemoteAddr, 'Remote Port: ' + str(RemotePort), OwningPid)
The script is run from time to time. It can run for 5 minutes and then don't work about an hour with this stupid mistake. How to get around it?
I really dont know, what's with it. Please, help me, what i do wrong?
I use python 3.2 on Win7 SP1 x64
Thank you a lot!

You shouldn't use addressof(size). That returns a Python integer which will be cast as a 32-bit C int. Use byref(size) to create a pointer, which will be a 64-bit value if you're using 64-bit Python.
GetExtendedTcpTable doesn't call SetLastError. It returns a DWORD with one of the following codes:
NO_ERROR = 0
ERROR_INVALID_PARAMETER = 87
ERROR_INSUFFICIENT_BUFFER = 122
The pdwSize argument has the required size if the buffer was too small. One option here is to start with a length 0 array; then resize the struct; and finally cast the array to the correct size:
class MIB_TCPTABLE_OWNER_PID(Structure):
_fields_ = [
("dwNumEntries", DWORD),
("MIB_TCPROW_OWNER_PID", MIB_TCPROW_OWNER_PID * 0),
]
_GetExtendedTcpTable = windll.iphlpapi.GetExtendedTcpTable
def GetExtendedTcpTable(vip=AF_INET):
table = MIB_TCPTABLE_OWNER_PID()
size = DWORD()
order = 1
failure = _GetExtendedTcpTable(
byref(table),
byref(size),
order,
vip,
TCP_TABLE_OWNER_PID_ALL,
0)
if failure == ERROR_INSUFFICIENT_BUFFER:
resize(table, size.value)
memset(byref(table), 0, sizeof(table))
failure = _GetExtendedTcpTable(
byref(table),
byref(size),
order,
vip,
TCP_TABLE_OWNER_PID_ALL,
0)
if failure:
raise WinError(failure)
ptr_type = POINTER(MIB_TCPROW_OWNER_PID * table.dwNumEntries)
tables = cast(table.MIB_TCPROW_OWNER_PID, ptr_type)[0]
pytables = []
for table in tables:
# rest unchanged
Regarding the Win32 LastError value, in general you shouldn't rely on GetLastError in Python. You don't know if you're seeing an old error code from a previous call or if an intervening call modified the LastError value. If you're checking a single API call that uses LastError, then it should be OK to check GetLastError immediately afterward if the call failed. But more generally you may need to load the DLL with use_last_error=True:
iphlpapi = WinDLL('iphlpapi', use_last_error=True)
Function pointers created from this WinDLL instance will save LastError to thread local storage immediately after the call returns. Calling get_last_error returns the saved error code. Beforehand you can call set_last_error(0) to have 0 swapped in to LastError before the function is called.

How to use ctypes.windll.Wininet.InternetQueryOptionW in python

several month ago,I wrote https://github.com/325862401/goagent/blob/master/local/ieproxysetting.cpp. that program can set internet proxy automatically.
Now I want to use ctypes.windll.Wininet to do the same thing.
But I donot know how to pass the argument to InternetQueryOptionW.
and I also don't find the class refer to the INTERNET_PER_CONN_OPTION_LIST structure.
is there any demo for the usage of InternetQueryOptionW or InternetSetOptionW in python.
this https://bitbucket.org/canassa/switch-proxy/src/685ac447783a7cb3ead6ce8369b66483790c26be/proxy.py?at=default maybe be helpful.
but I donot want to modify Registry.
just use the api.
Thanks a lot.
sorry for poor English.

Here is Python equivalent to your cpp code, pretty much straight forward conversation (I didn't test it that much):
#-*- coding: utf-8 -*-
#!python
from ctypes import *
from ctypes.wintypes import *
# stick to unicode version
LPWSTR = POINTER(WCHAR)
HINTERNET = LPVOID
INTERNET_PER_CONN_FLAGS = 1
INTERNET_PER_CONN_AUTOCONFIG_URL = 4
INTERNET_PER_CONN_AUTODISCOVERY_FLAGS = 5
INTERNET_OPTION_REFRESH = 37
INTERNET_OPTION_SETTINGS_CHANGED = 39
INTERNET_OPTION_PER_CONNECTION_OPTION = 75
PROXY_TYPE_AUTO_PROXY_URL = 4
class INTERNET_PER_CONN_OPTION(Structure):
class Value(Union):
_fields_ = [
('dwValue', DWORD),
('pszValue', LPWSTR),
('ftValue', FILETIME),
]
_fields_ = [
('dwOption', DWORD),
('Value', Value),
]
class INTERNET_PER_CONN_OPTION_LIST(Structure):
_fields_ = [
('dwSize', DWORD),
('pszConnection', LPWSTR),
('dwOptionCount', DWORD),
('dwOptionError', DWORD),
('pOptions', POINTER(INTERNET_PER_CONN_OPTION)),
]
InternetSetOption = windll.wininet.InternetSetOptionW
InternetSetOption.argtypes = [HINTERNET, DWORD, LPVOID, DWORD]
InternetSetOption.restype = BOOL
if __name__ == '__main__':
proxy = create_unicode_buffer('http://127.0.0.1:8086/proxy.pac')
List = INTERNET_PER_CONN_OPTION_LIST()
Option = (INTERNET_PER_CONN_OPTION * 2)()
nSize = c_ulong(sizeof(INTERNET_PER_CONN_OPTION_LIST))
Option[0].dwOption = INTERNET_PER_CONN_AUTOCONFIG_URL
Option[0].Value.pszValue = proxy
Option[1].dwOption = INTERNET_PER_CONN_FLAGS
Option[1].Value.dwValue = PROXY_TYPE_AUTO_PROXY_URL
List.dwSize = sizeof(INTERNET_PER_CONN_OPTION_LIST)
List.pszConnection = None
List.dwOptionCount = 2
List.dwOptionError = 0
List.pOptions = Option
assert InternetSetOption(None, INTERNET_OPTION_PER_CONNECTION_OPTION, byref(List), nSize) == True
assert InternetSetOption(None, INTERNET_OPTION_SETTINGS_CHANGED, None, 0) == True
assert InternetSetOption(None, INTERNET_OPTION_REFRESH, None, 0) == True
You don't need MultiByteToWideChar, create_unicode_buffer and the unicode version of InternetSetOption should do the job.

Windows WlanApi and Python Ctypes

I'm working on a captive portal projet on Windows. I've written this piece of code (similar to this) :
from ctypes import wintypes
import ctypes
WlanApi = ctypes.windll.wlanapi
hClientHandle = wintypes.HANDLE()
phClientHandle = ctypes.pointer(hClientHandle)
dwNegotiatedVersion = wintypes.DWORD()
pdwNegotiatedVersion = ctypes.pointer(dwNegotiatedVersion)
dwClientVersion = wintypes.DWORD()
dwClientVersion.value = 2L
rc = WlanApi.WlanOpenHandle(dwClientVersion, None, pdwNegotiatedVersion, phClientHandle)
print rc
class GUID(ctypes.Structure):
_fields_ = [("Data1", wintypes.DWORD),
("Data2", wintypes.WORD),
("Data3", wintypes.WORD),
("Data4", wintypes.BYTE * 8)]
class WLAN_INTERFACE_INFO (ctypes.Structure):
_fields_ = [('InterfaceGuid', GUID),
('strInterfaceDescription', wintypes.WCHAR * 256),
('isState', wintypes.????)]
class WLAN_INTERFACE_INFO_LIST(ctypes.Structure):
_fields_ = [('dwNumberOfItems', wintypes.DWORD),
('dwIndex', wintypes.DWORD),
('InterfaceInfo', WLAN_INTERFACE_INFO * 10)]
IfList = WLAN_INTERFACE_INFO_LIST()
pIfList = ctypes.pointer(IfList)
rc = WlanApi.WlanEnumInterfaces(hClientHandle, None, pIfList)
print rc
print "Num Entries: %s" % IfList.dwNumberOfItems
I can't find how to structure "WLAN_INTERFACE_STATE enumeration" and when I try with a WCHAR array or anything else, this script return my 6000000 wireless interfaces !!!
Can somebody help me?

It's just an integer, there is no structure 0 = Not ready, 1 = connected etc.

Hmm it starts to make sense,as most of these structs have a corresponding pointer.
According to the boys at PInvoke
[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)]
public struct WLAN_INTERFACE_INFO
{
/// GUID->_GUID
public Guid InterfaceGuid;
/// WCHAR[256]
[MarshalAs(UnmanagedType.ByValTStr, SizeConst = 256)]
public string strInterfaceDescription;
/// WLAN_INTERFACE_STATE->_WLAN_INTERFACE_STATE
public WLAN_INTERFACE_STATE isState;
}
Where WLAN_INTERFACE_STATE is
public enum WLAN_INTERFACE_STATE
{
wlan_interface_state_not_ready = 0,
...
// 1 to 6
...
wlan_interface_state_authenticating = 7,
}
PInvoke on WLAN...