I've updated ipython (and ipython cluster).
When I submit some job to cluster, and any of arguments has large size, this results in failure:
def test_client(x):
return x
from ipyparallel import Client
import numpy
x = Client(profile='ssh-ipy').load_balanced_view()
x.map_sync(test_client, [numpy.ones(100)])
>> [array([ 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,
1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,
1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,
1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,
1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,
1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,
1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,
1., 1., 1., 1., 1., 1., 1., 1., 1.])]
x.map_sync(test_client, [numpy.ones(10000)])
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-7-1dc792795519> in <module>()
3 x = Client(profile='ssh-ipy').load_balanced_view()
4 print x.map_sync(test_client, [numpy.ones(100)])
----> 5 print x.map_sync(test_client, [numpy.ones(10000)])
/moosefs/ipython_env/local/lib/python2.7/site-packages/ipyparallel/client/view.pyc in map_sync(self, f, *sequences, **kwargs)
351 raise TypeError("map_sync doesn't take a `block` keyword argument.")
352 kwargs['block'] = True
--> 353 return self.map(f,*sequences,**kwargs)
354
355 def imap(self, f, *sequences, **kwargs):
/moosefs/ipython_env/local/lib/python2.7/site-packages/ipyparallel/client/view.pyc in map(self, f, *sequences, **kwargs)
/moosefs/ipython_env/local/lib/python2.7/site-packages/ipyparallel/client/view.pyc in sync_results(f, self, *args, **kwargs)
52 self._in_sync_results = True
53 try:
---> 54 ret = f(self, *args, **kwargs)
55 finally:
56 self._in_sync_results = False
/moosefs/ipython_env/local/lib/python2.7/site-packages/ipyparallel/client/view.pyc in map(self, f, *sequences, **kwargs)
/moosefs/ipython_env/local/lib/python2.7/site-packages/ipyparallel/client/view.pyc in save_ids(f, self, *args, **kwargs)
37 n_previous = len(self.client.history)
38 try:
---> 39 ret = f(self, *args, **kwargs)
40 finally:
41 nmsgs = len(self.client.history) - n_previous
/moosefs/ipython_env/local/lib/python2.7/site-packages/ipyparallel/client/view.pyc in map(self, f, *sequences, **kwargs)
1117
1118 pf = ParallelFunction(self, f, block=block, chunksize=chunksize, ordered=ordered)
-> 1119 return pf.map(*sequences)
1120
1121 __all__ = ['LoadBalancedView', 'DirectView']
/moosefs/ipython_env/local/lib/python2.7/site-packages/ipyparallel/client/remotefunction.pyc in map(self, *sequences)
266 self._mapping = True
267 try:
--> 268 ret = self(*sequences)
269 finally:
270 self._mapping = False
/moosefs/ipython_env/local/lib/python2.7/site-packages/ipyparallel/client/remotefunction.pyc in __call__(self, *sequences)
/moosefs/ipython_env/local/lib/python2.7/site-packages/ipyparallel/client/remotefunction.pyc in sync_view_results(f, self, *args, **kwargs)
73 view = self.view
74 if view._in_sync_results:
---> 75 return f(self, *args, **kwargs)
76 view._in_sync_results = True
77 try:
/moosefs/ipython_env/local/lib/python2.7/site-packages/ipyparallel/client/remotefunction.pyc in __call__(self, *sequences)
238 view = self.view if balanced else client[t]
239 with view.temp_flags(block=False, **self.flags):
--> 240 ar = view.apply(f, *args)
241
242 msg_ids.extend(ar.msg_ids)
/moosefs/ipython_env/local/lib/python2.7/site-packages/ipyparallel/client/view.pyc in apply(self, f, *args, **kwargs)
218 ``f(*args, **kwargs)``.
219 """
--> 220 return self._really_apply(f, args, kwargs)
221
222 def apply_async(self, f, *args, **kwargs):
/moosefs/ipython_env/local/lib/python2.7/site-packages/ipyparallel/client/view.pyc in _really_apply(self, f, args, kwargs, block, track, after, follow, timeout, targets, retries)
/moosefs/ipython_env/local/lib/python2.7/site-packages/ipyparallel/client/view.pyc in sync_results(f, self, *args, **kwargs)
49 """sync relevant results from self.client to our results attribute."""
50 if self._in_sync_results:
---> 51 return f(self, *args, **kwargs)
52 self._in_sync_results = True
53 try:
/moosefs/ipython_env/local/lib/python2.7/site-packages/ipyparallel/client/view.pyc in _really_apply(self, f, args, kwargs, block, track, after, follow, timeout, targets, retries)
/moosefs/ipython_env/local/lib/python2.7/site-packages/ipyparallel/client/view.pyc in save_ids(f, self, *args, **kwargs)
37 n_previous = len(self.client.history)
38 try:
---> 39 ret = f(self, *args, **kwargs)
40 finally:
41 nmsgs = len(self.client.history) - n_previous
/moosefs/ipython_env/local/lib/python2.7/site-packages/ipyparallel/client/view.pyc in _really_apply(self, f, args, kwargs, block, track, after, follow, timeout, targets, retries)
1044
1045 msg = self.client.send_apply_request(self._socket, f, args, kwargs, track=track,
-> 1046 metadata=metadata)
1047 tracker = None if track is False else msg['tracker']
1048
/moosefs/ipython_env/local/lib/python2.7/site-packages/ipyparallel/client/client.pyc in send_apply_request(self, socket, f, args, kwargs, metadata, track, ident)
1243
1244 msg = self.session.send(socket, "apply_request", buffers=bufs, ident=ident,
-> 1245 metadata=metadata, track=track)
1246
1247 msg_id = msg['header']['msg_id']
/moosefs/ipython_env/local/lib/python2.7/site-packages/jupyter_client/session.pyc in send(self, stream, msg_or_type, content, parent, ident, buffers, track, header, metadata)
670 if buffers and track and not copy:
671 # only really track when we are doing zero-copy buffers
--> 672 tracker = stream.send_multipart(to_send, copy=False, track=True)
673 else:
674 # use dummy tracker, which will be done immediately
/moosefs/ipython_env/local/lib/python2.7/site-packages/zmq/sugar/socket.pyc in send_multipart(self, msg_parts, flags, copy, track)
324 raise TypeError(
325 "Frame %i (%s) does not support the buffer interface." % (
--> 326 i, rmsg,
327 ))
328 for msg in msg_parts[:-1]:
TypeError: Frame 10 (<read-only buffer for 0x1ea31c0,...) does not support the buffer interface.