Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

support executing global setup function early to provide environment for subsequent dependencies #220

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

support executing global setup function early to provide environment for subsequent dependencies #220

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
bcfcd71
Update __init__.py
bsync Jun 3, 2021
511ac12
Update obj_instances.py
bsync Jun 3, 2021
09aff83
Update obj_instances.py
bsync Jun 3, 2021
13fa89f
Merge pull request #1 from bsync/bsync-patch-1
bsync Jun 3, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions py3/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2660,7 +2660,7 @@ def __init__(self, computation, nodes=None, depends=[], job_status=None, cluster
if inspect.isfunction(setup):
if setup.__defaults__:
print('\n dispy does not support calling "setup" with keyword arguments\n')
depends.append(setup)
depends.insert(0, setup)
compute.setup = setup.__name__
compute.setup_args_count = setup.__code__.co_argcount
elif isinstance(setup, functools.partial):
Expand Down Expand Up @@ -2761,8 +2761,9 @@ def __init__(self, computation, nodes=None, depends=[], job_status=None, cluster

elif (inspect.isfunction(dep) or inspect.isclass(dep) or
(hasattr(dep, '__class__') and hasattr(dep, '__module__'))):
immediate=None
if inspect.isfunction(dep) or inspect.isclass(dep):
pass
immediate = dep.__name__ if dep.__name__.startswith('nodeinit_') else None
elif hasattr(dep, '__class__') and inspect.isclass(dep.__class__):
dep = dep.__class__
try:
Expand All @@ -2772,6 +2773,8 @@ def __init__(self, computation, nodes=None, depends=[], job_status=None, cluster
raise
lines[0] = lines[0].lstrip()
compute.code += '\n' + ''.join(lines)
if immediate:
compute.code += '\n' + immediate + '()\n'
elif isinstance(dep, functools.partial):
try:
lines = inspect.getsourcelines(dep.func)[0]
Expand Down
11 changes: 9 additions & 2 deletions py3/examples/obj_instances.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
# example program that sends object instances in local program
# as arguments to distributed computation
class C:
# Also shows use of nodeinit_ feature to ensure that cmd module is immediately imported so that class C can extend cmd.Cmd on the node
import cmd

class C(cmd.Cmd):
def __init__(self, i, n):
self.i = i
self.n = n
Expand All @@ -17,7 +20,11 @@ def compute(obj):

if __name__ == '__main__':
import random, dispy
cluster = dispy.JobCluster(compute, depends=[C])
def nodeinit_setup():
global cmd
import cmd
return 0
cluster = dispy.JobCluster(compute, depends=[C], setup=nodeinit_setup)
jobs = []
for i in range(10):
c = C(i, random.uniform(1, 3)) # create object of C
Expand Down