I have copied all the codes to the work directory on all my engine machines. And my code are:
my_test.py
my_startegy.py
main.py
So, main.py will be ran on Client Machine, the codes in main.py are:
from ipyparallel import Client
import my_test
import my_strategy as strategy
class Beck_Test_Parallel(object):
"""
"""
def __init__(self):
self.rc = None
self.dview = None
def start_client(self, path):
self.rc = Client(path)
self.dview = self.rc[:]
#self.dview.push(dict(
# Account=my_test.Account,
# dataImport=my_test.dataImport
# ))
def parallel_map(self, deal_function, accounts):
import my_test
return self.dview.map_sync(deal_function, accounts)
def create_accounts(time_list, account):
accounts = []
for index, time in enumerate(time_list):
acc = my_test.Account(
strategy.start,
strategy.end,
strategy.freq,
strategy.universe_code,
strategy.capital_base,
strategy.short_capital,
strategy.benchmark,
strategy.self_defined
)
account.share_data(acc)
acc.iniData2()
acc.iniData3()
acc.current_time = time
acc.days_counts = index+1
acc.dynamic_record['capital'] = acc.capital_base
del acc.connect
accounts.append(acc)
return accounts
def let_us_deal(account):
account = strategy.handle_data(account)
print ' >>>', account.current_time
return account
if __name__ == '__main__':
account = my_test.Account(
strategy.start,
strategy.end,
strategy.freq,
strategy.universe_code,
strategy.capital_base,
strategy.short_capital,
strategy.benchmark,
strategy.self_defined
)
account.iniData()
account.iniData2()
account.iniData3()
time_list = my_test.get_deal_time_list(account)
accounts = parallel.create_accounts(time_list, account)
back_test_parallel = parallel.Beck_Test_Parallel()
back_test_parallel.start_client(
'/home/fit/.ipython/profile_default/security/ipcontroller-client.json')
back_test_parallel.dview.execute('import my_test')
back_test_parallel.dview.execute('import my_strategy as strategy')
# get the result
result = back_test_parallel.parallel_map(let_us_deal, accounts)
for acc in result.get():
print acc.reselected_stocks, acc.current_time
And I have imported my_test module in parallel_map() function in Class Back_Test_Parallel and I have also imported my_test module in back_test_parallel.dview.execute('import my_test').
And the corresponding modules are on the engine machine's work directory. I have copied the ipcontroller-client.json and ipcontroller-engine.json to the work directory on engine machine.
But when it runs, the error is ImportError: No module named my_test, since the module my_test.py is already on the work directory. It really made me feel frustrated!
---------------------------------------------------------------------------
CompositeError Traceback (most recent call last)
/home/fit/log/1027/back_test/main.py in <module>()
119 import ipdb
120 ipdb.set_trace()
--> 121 for acc in result.get():
122 print acc.reselected_stocks, acc.current_time
123
/usr/local/lib/python2.7/dist-packages/ipyparallel/client/asyncresult.pyc in get(self, timeout)
102 return self._result
103 else:
--> 104 raise self._exception
105 else:
106 raise error.TimeoutError("Result not ready.")
CompositeError: one or more exceptions from call to method: let_us_deal
[0:apply]: ImportError: No module named my_test
[1:apply]: ImportError: No module named my_test
something about result:
In [2]: result
Out[2]: <AsyncMapResult: finished>
In [3]: type(result)
Out[3]: ipyparallel.client.asyncresult.AsyncMapResult
Note that, when it runs on single machine by using ipcluster start -n 8, it works fine, without any error.
Thanks in advance
I think my CWD is not in the right directory. So you can check your CWD
>>> import os
>>> print(dview.apply_sync(os.getcwd).get())
If it is in wrong directory, before parallel computing, you can set the right CWD to make sure you ipyparallel env is in the right work directory:
>>> import os
>>> dview.map(os.chdir, ['/path/to/my/project/on/engine']*number_of_engines)
>>> print(dview.apply_sync(os.getcwd).get())
You can also check your engines' name by
>>> import socket
>>> print(dview.apply_sync(socket.gethostname))
And it works fine!
pip install ipyparallel --upgrade
Related
I just started on Python, and still trying to learn it. I recently started learning Sikulix and its applications. Would highly appreciate if you could share your ideas...
Problem is that I have 3 py files. Say:
main.py
checkInventory.py
startMining.py
I only defined 1 function on checkInventory.py, startMining.py each, functions are also named after the file names (checkInventory(), startMining() respectively)
When I run the main.py it is calling checkInventory() using:
beginning of main.py...
from checkInventoryStatus import *
No problem on this. I can call checkInventoryStatus() function with no problem on main.py.
However, checkInventory() also runs startMining() at some point and it keeps giving this error:
NameError ( global name 'startMining' is not defined )
I am also importing all the content of startMining.py to checkInventory.py using:
beginning of checkInventory.py...
from startMining import *
I tried to import all elements of startMining.py to checkInventory.py but it is failing to call the function startMining() from checkInventory.py.
Main
from startMining import *
from watchMining import *
from checkInventoryStatus import *
def startApp():
KO = switchApp("KL")
checkInventoryStatus()
with open("C:\Users\PC\Desktop\skx\logs.txt", "w") as f:
f.write("App started!\n")
startApp()
checkInventoryStatus
from sikuli import *
from startMining import *
from watchMining import *
def checkInventoryStatus():
if(exists("1673723539914.png")):
if(exists(Pattern("1673723411692.png").exact())):
startMining()
startMining.py
from sikuli import *
from watchMining import *
from checkInventoryStatus import *
def startMining():
wait(0.6)
type(Key.ENTER + "/town" + Key.ENTER)
arrived = False
wait(0.3)
try:
click(Pattern("1673722724479.png").similar(0.58))
wait(1.5)
while(True):
if(exists(Pattern("1673729480660.png").exact())):
click(Pattern("1673729480660.png").exact())
mouseMove((Location(700, 500)))
X_Coordinate_Region = Region(105,75,30,14)
Y_Coordinate_Region = Region(139,77,25,11)
X_Coordinate = X_Coordinate_Region.text()
Y_Coordinate = Y_Coordinate_Region.text()
X_temp = X_Coordinate_Region.text()
Y_temp = Y_Coordinate_Region.text()
wait(0.45)
X_Coordinate_Region = Region(105,75,30,14)
Y_Coordinate_Region = Region(139,77,25,11)
X_Coordinate = X_Coordinate_Region.text()
Y_Coordinate = Y_Coordinate_Region.text()
if X_temp==X_Coordinate_Region.text() and Y_temp==Y_Coordinate_Region.text() and arrived==False:
startMining()
try:
if abs(int(X_Coordinate_Region.text())-1490)<30 and abs(int(Y_Coordinate_Region.text())-540)<30:
arrrived=True
type("s")
mouseMove(mouseMove(Location(0, 500)))
wait(0.95)
mouseMove((Location(700, 500)))
click(Pattern("1673398228807.png").similar(0.50))
while(True):
if(exists(Pattern("1673729480660.png").exact())):
click(Pattern("1673729480660.png").exact())
mouseMove((Location(700, 500)))
arrived=False
try:
if abs(int(X_Coordinate_Region.text())-1453)<30 and abs(int(Y_Coordinate_Region.text())-380)<30:
arrrived=True
type("s")
type(" ")
break
except:
continue
except:
continue
#f.write("\nX:" +X_Coordinate+" Y: "+Y_Coordinate+"\n")
with open("C:\Users\PC\Desktop\skx\logs.txt", "a") as f:
f.write("\nX:" +X_Coordinate+" Y: "+Y_Coordinate+"\n")
wait(0.5)
except:
mouseMove(mouseMove(Location(0, 500)))
wait(0.4)
mouseMove((Location(700, 500)))
startMining()
Here is the error message:
[error] script [ main ] stopped with error in line 13
[error] NameError ( global name 'startMining' is not defined )
[error] --- Traceback --- error source first
line: module ( function ) statement
9: checkInventoryStatus ( checkInventoryStatus ) startMining()
8: main ( startApp ) checkInventoryStatus()
13: main ( <module> ) startApp()
[error] --- Traceback --- end --------------
I believe it is due to "circular import" behavior between checkInventoryStatus.py and startMining.py.
You can structure your code in a way that eliminates calling modules in a circular motion. For example, you can move checkInventoryStatus() into startMining.py which will kill the dependency to checkInventoryStatus.py (assuming that all you code consists of only these three scripts.)
I downloaded the code from here https://github.com/SpaceNetChallenge/SpaceNet_SAR_Buildings_Solutions, specifically using model 1. I downloaded the weights corresponding and created the following file to load the model and test. First, I copy the Unet part in main.py into a separate file umodel.py and the test file as follows
import torch
exec(open("./umodel.py").read())
network_data = torch.load('snapshot_fold_8_best')
print(network_data.keys())
import sys
sys.path.append("geffnet")
class Namespace:
def __init__(self, **kwargs):
self.__dict__.update(kwargs)
args = Namespace(extra_num = 1,
dec_ch = [32, 64, 128, 256, 1024],
stride=32,
net='b5',
bot1x1=True,
glob=True,
bn=True,
aspp=True,
ocr=True,
aux_scale=True)
def load_state_dict(model, state_dict):
missing_keys = []
# from UnetOS.umodel import Unet
exec(open("./umodel.py").read())
try:
from torch.hub import load_state_dict_from_url
except ImportError:
from torch.utils.model_zoo import load_url as load_state_dict_from_url
# from UnetOS.umodel import *
model = Unet(extra_num = args.extra_num, dec_ch = args.dec_ch, stride=args.stride, net=args.net, bot1x1 = args.bot1x1, glob=args.glob, bn=args.bn, aspp=args.aspp,
ocr=args.ocr, aux = args.aux_scale > 0).cuda()
load_state_dict(model, network_data)
My question is, why exec(open("./umodel.py").read()) works nicely but whenever I tried to import from umodel import Unet it has errors
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
~\AppData\Local\Temp/ipykernel_10492/1282530406.py in <module>
9 # ah
10 model = Unet(extra_num = args.extra_num, dec_ch = args.dec_ch, stride=args.stride, net=args.net, bot1x1 = args.bot1x1, glob=args.glob, bn=args.bn, aspp=args.aspp,
---> 11 ocr=args.ocr, aux = args.aux_scale > 0).cuda()
12 #model = Unet()
13 #print(network_data.key())
D:\hines\Pretrained\1-zbigniewwojna\UnetOS\umodel.py in __init__(self, extra_num, dec_ch, stride, net, bot1x1, glob, bn, aspp, ocr, aux)
238 ['ir_r4_k5_s2_e6_c192_se0.25'],
239 ['ir_r1_k3_s1_e6_c320_se0.25']]
--> 240 enc = GenEfficientNet(in_chans=3, block_args=decode_arch_def(arch_def, depth_multiplier),
241 num_features=round_channels(1280, channel_multiplier, 8, None), stem_size=32,
242 channel_multiplier=channel_multiplier, act_layer=resolve_act_layer({}, 'swish'),
NameError: name 'decode_arch_def' is not defined
The main file is as follow https://github.com/SpaceNetChallenge/SpaceNet_SAR_Buildings_Solutions/blob/master/1-zbigniewwojna/main.py
From the error message, it appears that decode_arch_def is not available and looking at your imports, that has to come from from geffnet.efficientnet_builder import * (it does https://github.com/rwightman/gen-efficientnet-pytorch/blob/master/geffnet/efficientnet_builder.py)
Your exec must have worked because it followed a similar import, that brought decode_arch_def in scope - exec() executes code in the current scope, so it will work because in that scope decode_arch_def is already defined.
However, when you import, the imported module itself doesn't have the function you need in scope. You should add the required import statements to the file you're importing to bring them into scope and it should work.
For example, with a mod.py containing this:
def my_func():
print(datetime.now())
This works:
from datetime import datetime
exec(open("./mod.py").read())
my_func()
But this does not:
from datetime import datetime
import mod
mod.my_func()
To make that work, mod.py would have to be:
from datetime import datetime
def my_func():
print(datetime.now())
And the import of datetime wouldn't be needed in the main program, since it's not referenced there. Your code has a similar issue - you need to determine all the dependencies of your Unet class and import them.
when I try to run the code. I am getting an error.
The error is
Traceback (most recent call last):
File "A2.py", line 2, in <module>
from easysnmp import Session
ImportError: No module named 'easysnmp'
Note: I am getting the above error.Even though, I have installed easysnmp module.
The code is
#!/usr/bin/python
from easysnmp import Session
import argparse
import time
parser = argparse.ArgumentParser(description='probe')
parser.add_argument('cred',help='credentials')
parser.add_argument('freq',type=float,help='enter frequency')
parser.add_argument('samples',type=int,help='enter samples')
parser.add_argument('oid',nargs='+',help='enter oid')
args=parser.parse_args()
t=1/args.freq
s=args.samples
cred1=args.cred
ip,port,comm=cred1.split(":")
count=0
session=Session(hostname=ip,remote_port=port,community=comm, version=2,timeout=2,retries=1)
args.oid.insert(0, '1.3.6.1.2.1.1.3.0')
old=[]
out1=[]
t4=0
while (count!=s):
t1=time.time()
new = session.get(args.oid)
t2=time.time()
if len(new)==len(old):
newtime=float(new[0].value)/100
oldtime=float(old[0].value)/100
if args.freq > 1:
tdiff = newtime-oldtime
if args.freq <= 1:
tdiff1 = t1-t4
if tdiff!=0:
tdiff = int(tdiff1)
else:
tdiff = int(t)
for i in range(1,len(args.oid)):
if new[i].value!="NOSUCHINSTANCE" and old[i].value!="NOSUCHINSTANCE":
a=int(new[i].value)
b=int(old[i].value)
if a>=b:
out=(a-b)/tdiff
out1.append(out)
if a<b and new[i].snmp_type=="COUNTER64":
out=((2**64+a)-b)/tdiff
out1.append(out)
if a<b and new[i].snmp_type=="COUNTER32":
out=((2**32+a)-b)/tdiff
out1.append(out)
else:
print t1, "|"
count=count+1
if len(out1)!=0:
sar = [str(get) for get in out1]
print int(t1) ,'|', ("|" . join(sar))
old = new[:]
t4=t1
del out1[:]
t3=time.time()
if t-t3+t1>0:
time.sleep(t-t3+t1)
else:
time.sleep(0.0)
Try to put import easysnmp at the top of your code, it solved the problem for me at a similar situation!
My wild guess is that you installed the module for python 3 and used python 2 or the other way around.
Try
pip install easysnmp
or
pip3 install easysnmp
I am using rpyc within an application and have come across a small hiccup.
I would just like to know if it's possible to update remote properties?
I have a test server:
import rpyc
from rpyc.utils.server import ThreadedServer
class Test:
def __init__(self):
self._v = 0
#property
def V(self):
return self._v
#V.setter
def V(self, value):
self._v = value
class TestService(rpyc.Service):
def exposed_Test(self):
return Test()
if __name__ == '__main__':
t = ThreadedServer(TestService, port = 2942,
protocol_config={"allow_all_attrs":True})
t.start()
and within an ipython console:
In [1]: import rpyc
In [2]: conn = rpyc.connect('localhost', 2942, config={'allow_all_attrs':True})
In [3]: test = conn.root.Test()
In [4]: test.V
Out[4]: 0
In [5]: test.V = 2
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-5-d3ae0dcd1075> in <module>()
----> 1 test.V = 2
<blah blah traceback>
AttributeError: cannot access 'V'
Is it possible to update remote properties at all?
Yes, setting remote attributes is blocked by default, but is allowed if you set allow_setattr=True.
(Setting allow_delattr=True also makes sense if you're setting allow_setattr=True anyway)
t = ThreadedServer(TestService, port = 2942,
protocol_config={"allow_all_attrs":True,
"allow_setattr": True,
"allow_delattr": True,})
See the docs.
You can also hackily bypass the setattr-protection, by accessing the __dict__ of the remote object directly (but of course the first soluiton is much better):
test.__dict__['V'] = 2
A number of people in my organization have different email names from perforce names, so I need to create an IEmailLookup derivation that overrides getAddress to do my evil bidding:
(From my master.cfg)
class MyIEmailLookup:
from buildbot import interfaces
__implements__ = interfaces.IEmailLookup
def getAddresses(user):
address_dict = {"user1", "user_one#our_domain.com"}
try:
address = address_dict[user]
except KeyError:
address = user + "#our_domain.com"
return address
maillookup = MyIEmailLookup()
from buildbot.status import mail
c['status'].append(mail.MailNotifier(....
....
lookup=maillookup
))
I've tried any number of permutations, but I either get:
Traceback (most recent call last):
File "/Library/Python/2.6/site-packages/buildbot-0.8.3p1-py2.6.egg/buildbot/scripts/runner.py", line 1071, in doCheckConfig
ConfigLoader(configFileName=configFileName)
File "/Library/Python/2.6/site-packages/buildbot-0.8.3p1-py2.6.egg/buildbot/scripts/checkconfig.py", line 46, in __init__
self.loadConfig(configFile, check_synchronously_only=True)
File "/Library/Python/2.6/site-packages/buildbot-0.8.3p1-py2.6.egg/buildbot/master.py", line 727, in loadConfig
exec f in localDict
File "/Users/playbuilder/buildbot/master.cfg", line 207, in <module>
lookup=maillookup
File "/Library/Python/2.6/site-packages/buildbot-0.8.3p1-py2.6.egg/buildbot/status/mail.py", line 293, in __init__
assert interfaces.IEmailLookup.providedBy(lookup)
AssertionError
...or any other number of issues, dependant upon how I try to implement the IEmailLookup interface.
I'm using buildbot 0.8.3p1 and python 2.6.1.
I see precious few examples of how to do this, and every one of them fails in my context. What am I missing here?
I just solved this problem myself.
First you need to add (somewhere at the top of the file)
from zope.interface import implements
and then change
__implements__ = interfaces.IEmailLookup
to
if implements:
implements( interfaces.IEmailLookup )
else:
__implements__ = interfaces.IEmailLookup
If you want to fetch email from perforce user, you can use this class:
# .-----------------------.
# | Perforce Email Lookup |
# `-----------------------'
from twisted.internet import defer, utils
from buildbot.util import ComparableMixin
from buildbot.interfaces import IEmailLookup
from zope.interface import implements
import os
import re
class PerforceEmailLookup(ComparableMixin):
implements(IEmailLookup)
compare_attrs = ["p4port", "p4user", "p4passwd", "p4bin"]
env_vars = ["P4CLIENT", "P4PORT", "P4PASSWD", "P4USER",
"P4CHARSET"]
def __init__(self,
p4port = None,
p4user = None,
p4passwd = None,
p4bin = 'p4'):
self.p4port = p4port
self.p4user = p4user
self.p4passwd = p4passwd
self.p4bin = p4bin
self.email_re = re.compile(r"Email:\s+(?P<email>\S+#\S+)\s*$")
def _get_process_output(self, args):
env = dict([(e, os.environ.get(e)) for e in self.env_vars if os.environ.get(e)])
d = utils.getProcessOutput(self.p4bin, args, env)
return d
#defer.deferredGenerator
def getAddress(self, name):
if '#' in name:
yield name
return
args = []
if self.p4port:
args.extend(['-p', self.p4port])
if self.p4user:
args.extend(['-u', self.p4user])
if self.p4passwd:
args.extend(['-P', self.p4passwd])
args.extend(['user', '-o', name])
wfd = defer.waitForDeferred(self._get_process_output(args))
yield wfd
result = wfd.getResult()
for line in result.split('\n'):
line = line.strip()
if not line: continue
m = self.email_re.match(line)
if m:
yield m.group('email')
return
yield name
usage would look like:
c['status'].append(
MailNotifier(
sendToInterestedUsers = True,
mode = 'failing',
lookup = PerforceEmailLookup(
p4port = "perforce:1666",
p4user = "buildbot",
p4passwd = "buildbot")))
Try this:
from buildbot.interfaces import IEmailLookup
from buildbot.util import ComparableMixin
from zope.interface import implements
class lookup_example_email(ComparableMixin):
implements(IEmailLookup)
def getAddress(self,user):
return "%s#example.com"%(user)
...
mn = MailNotifier(..., lookup=lookup_example_email(), extraRecipients=m)
Here's the piece of code I use which works with buildbot 2.3.1 in python3.6.
from buildbot.interfaces import IEmailLookup
from buildbot.util import ComparableMixin
from zope.interface import implementer
#implementer(IEmailLookup)
class EmailMap(ComparableMixin):
def getAddress(self, name):
return f'{name}#xxxxx'