python examples/jobs/localjob.py

Job State   : Done
Exitcode    : 0
Exec. hosts : ['localhost']
An exception occured: (NotImplemented) NotImplemented: Job._get_created is not implemented for saga.adaptor.shell_job.ShellJob 

*** Backtrace:
   File "examples/jobs/localjob.py", line 81, in <module>
    main()
  File "examples/jobs/localjob.py", line 69, in main
    print "Create time : %s" % (sleebjob_clone.created)
  File "/tmp/test/lib/python2.7/site-packages/saga_python-0.9.2-py2.7.egg/saga/attributes.py", line 2240, in __getattr__
    return self._attributes_i_get      (key, flow=self._DOWN)
  File "/tmp/test/lib/python2.7/site-packages/saga_python-0.9.2-py2.7.egg/saga/attributes.py", line 1017, in _attributes_i_get
    self._attributes_t_call_getter (key)
  File "/tmp/test/lib/python2.7/site-packages/saga_python-0.9.2-py2.7.egg/saga/attributes.py", line 589, in _attributes_t_call_getter
    val=key_getter ()
  File "/tmp/test/lib/python2.7/site-packages/saga_python-0.9.2-py2.7.egg/saga/job/job.py", line 434, in _get_created
    return self._adaptor.get_created (ttype=ttype)
  File "/tmp/test/lib/python2.7/site-packages/saga_python-0.9.2-py2.7.egg/saga/adaptors/cpi/decorators.py", line 68, in wrap_function
    self.__class__.__name__))

NOSE_VERBOSE=10 python tests/run_tests.py --config=tests/configs/ssh_localhost.cfg

======================================================================
FAIL: submit a job via run_job, and retrieve id
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/Users/oweidner/Projects/saga-python/venv/lib/python2.7/site-packages/nose/case.py", line 197, in runTest
    self.test(*self.arg)
  File "/Users/oweidner/Projects/saga-python/tests/unittests/api/job/test_service.py", line 76, in test_run_job
    assert False, "Unexpected exception: %s" % se
AssertionError: Unexpected exception: NoSuccess: find failed (process I/O failed): Shared connection to localhost closed.

======================================================================
FAIL: Testing if a submitted job shows up in Service.list()
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/vishal/sagadevel/local/lib/python2.7/site-packages/nose/case.py", line 197, in runTest
    self.test(*self.arg)
  File "/home/vishal/saga-python-devel/tests/unittests/api/job/test_service.py", line 62, in test_list_jobs
    assert False, "Unexpected exception: %s" % se
AssertionError: Unexpected exception: NoSuccess: failed to list jobs via 'qstat': 
/opt/torque/bin/qstat: invalid option -- l
usage: 
                          qstat [-f [-1]] [-W site_specific] [-x] [ job_identifier... | destination... ]
                          qstat [-a|-i|-r|-e] [-u user] [-n [-1]] [-s] [-G|-M] [-R] [job_id... | destination...]
                          qstat -Q [-f [-1]] [-W site_specific] [ destination... ]
                          qstat -q [-G|-M] [ destination... ]
                          qstat -B [-f [-1]] [-W site_specific] [ server_name... ]

find from pty process [MainThread] failed (NoSuccess: cannot read from dead pty process ()) (ssh: connect to host localhost port 22: Connection refused

Traceback (most recent call last):
  File "/usr/lib/python2.7/threading.py", line 552, in __bootstrap_inner
    self.run()
  File "/usr/lib/python2.7/threading.py", line 505, in run
    self.__target(*self.__args, **self.__kwargs)
  File "tests/_andre/test_perf.py", line 30, in workload
    service = saga.job.Service (url)
  File "/home/merzky/.virtualenv/saga-python/local/lib/python2.7/site-packages/saga-1.0-py2.7.egg/saga/job/service.py", line 70, in __init__
    session = Session (default=True)
  File "/home/merzky/.virtualenv/saga-python/local/lib/python2.7/site-packages/saga-1.0-py2.7.egg/saga/session.py", line 89, in __init__
    saga.base.SimpleBase.__init__ (self)
  File "/home/merzky/.virtualenv/saga-python/local/lib/python2.7/site-packages/saga-1.0-py2.7.egg/saga/base.py", line 17, in __init__
    self._engine    = saga.engine.engine.Engine ()
  File "/home/merzky/.virtualenv/saga-python/local/lib/python2.7/site-packages/saga-1.0-py2.7.egg/saga/utils/singleton.py", line 21, in __call__
    cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
  File "/home/merzky/.virtualenv/saga-python/local/lib/python2.7/site-packages/saga-1.0-py2.7.egg/saga/engine/engine.py", line 166, in __init__
    signal.signal (signal.SIGINT, signal_handler)
ValueError: signal only works in main thread

======================================================================
FAIL: Testing if a submitted job shows up in Service.list()
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/Users/oweidner/Projects/saga-python/venv/lib/python2.7/site-packages/nose/case.py", line 197, in runTest
    self.test(*self.arg)
  File "/Users/oweidner/Projects/saga-python/tests/unittests/api/job/test_service.py", line 57, in test_list_jobs
    assert False, "Unexpected exception: %s" % se
AssertionError: Unexpected exception: NoSuccess: find failed (unexpected EOF ()): mux_client_request_session: session request failed: Session open refused by peer
PTY allocation request failed on channel 0

======================================================================
FAIL: Test pty_shell with prompt change
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/Users/oweidner/Projects/saga-python/venv/lib/python2.7/site-packages/nose-1.2.1-py2.7.egg/nose/case.py", line 197, in runTest
    self.test(*self.arg)
  File "/Users/oweidner/Projects/saga-python/tests/unittests/utils/test_pty_shell.py", line 81, in test_ptyshell_prompt
    assert (out == txt), "OUT: %s* TXT: %s*" % (repr(out), repr(txt))
AssertionError: OUT: '______1______2_____3_____\x1b]7;file://teahupoo.local/Users/oweidner/Projects/saga-python/tests/unittests/utils\x07'* TXT: '______1______2_____3_____'*

SAGA_VERBOSE=2 NOSE_VERBOSE=10 python tests/run_tests.py --config=tests/configs/basetests.cfg

saga.exceptions.BadParameter: BadParameter: 'JobDescription.WorkingDirectory' is not supported by this adaptor

FWIW, a related approach would be to go full-monty and to add full
signature type checking and overloading via decorators, a la [1].  I
would kinda like it...

Note that in both approaches, we would need to keep different method
names on CPI / adaptor level -- which is only fair, as the call
semantic is indeed different...

Cheers, Andre.


[1] http://code.activestate.com/recipes/577065-type-checking-function-overloading-decorator/

AssertionError: Unexpected exception: NoSuccess: find from pty process [MainThread] failed (NoSuccess: cannot read from dead pty process ()) (command-line: line 0: Bad configuration option: ControlPersist

easy_install . && SAGA_VERBOSE=5 python examples/files/sftp_remotedir_list.py

An error occured during file operation: NotImplemented: Directory.is_dir is not implemented for saga.adaptor.shell_file.ShellDirectory (is_dir)

easy_install . && SAGA_VERBOSE=5 python examples/files/sftp_file_copy.py

An error occured during file operation: NoSuccess: failed to list(): (1)(ls: h*: No such file or directory
)

#!/usr/bin/env python
# encoding: utf-8

""" This examples shows how to run a job on the local machine
    using the 'local' job adaptor.
"""

__author__    = "Ole Weidner"
__copyright__ = "Copyright 2012-2013, The SAGA Project"
__license__   = "MIT"

import sys
import saga


def main():

    try:
        # create a job service for the local machine. both, 'fork' and
        # 'local' schemes trigger the local job adaptor.
        js = saga.job.Service("fork://localhost")

        # describe our job
        jd = saga.job.Description()

        # environment, executable & arguments. We use '/bin/sleep' to simulate
        # a job that runs for $RUNTIME seconds.
        jd.environment = {'RUNTIME': '10'}
        jd.executable  = '/usr/bin/env'
        jd.arguments   = ["""python -c "import sys
import os
import urllib
import sys
import time
start_time = time.time()
home = os.environ.get(\"HOME\")
#print \"Home: \" + home
if home==None: home = os.getcwd()
BIGJOB_AGENT_DIR= os.path.join(home, \".bigjob\")
if not os.path.exists(BIGJOB_AGENT_DIR): os.mkdir (BIGJOB_AGENT_DIR)
BIGJOB_PYTHON_DIR=BIGJOB_AGENT_DIR+\"/python/\"
if not os.path.exists(BIGJOB_PYTHON_DIR): os.mkdir(BIGJOB_PYTHON_DIR)
BOOTSTRAP_URL=\"https://raw.github.com/saga-project/BigJob/master/bootstrap/bigjob-bootstrap.py\"
BOOTSTRAP_FILE=BIGJOB_AGENT_DIR+\"/bigjob-bootstrap.py\"
#ensure that BJ in .bigjob is upfront in sys.path
sys.path.insert(0, os.getcwd() + \"/../\")
#sys.path.insert(0, /User/luckow/.bigjob/python/lib\")
#sys.path.insert(0, os.getcwd() + \"/../../\")
p = list()
for i in sys.path:
    if i.find(\".bigjob/python\")>1:
          p.insert(0, i)
for i in p: sys.path.insert(0, i)
print \"Python path: \" + str(sys.path)
print \"Python version: \" + str(sys.version_info)
try: import saga
except: print \"SAGA and SAGA Python Bindings not found.\";
try: import bigjob.bigjob_agent
except: 
    print \"BigJob not installed. Attempt to install it.\"; 
    opener = urllib.FancyURLopener({}); 
    opener.retrieve(BOOTSTRAP_URL, BOOTSTRAP_FILE); 
    print \"Execute: \" + \"python \" + BOOTSTRAP_FILE + \" \" + BIGJOB_PYTHON_DIR
    os.system(\"/usr/bin/env\")
    try:
        os.system(\"python \" + BOOTSTRAP_FILE + \" \" + BIGJOB_PYTHON_DIR); 
        activate_this = os.path.join(BIGJOB_PYTHON_DIR, \"bin/activate_this.py\"); 
        execfile(activate_this, dict(__file__=activate_this))
    except:
        print \"BJ installation failed. Trying system-level python (/usr/bin/python)\";
        os.system(\"/usr/bin/python \" + BOOTSTRAP_FILE + \" \" + BIGJOB_PYTHON_DIR); 
        activate_this = os.path.join(BIGJOB_PYTHON_DIR, \"bin/activate_this.py\"); 
        execfile(activate_this, dict(__file__=activate_this))
#try to import BJ once again
import bigjob.bigjob_agent
# execute bj agent
args = list()
args.append(\"bigjob_agent.py\")
args.append(\"redis://[email protected]:6379\")
args.append(\"bigjob:bj-5e195b66-85ee-11e2-99a7-14109fd519a1:localhost\")
args.append(\"\")
print \"Bootstrap time: \" + str(time.time()-start_time)
print \"Starting BigJob Agents with following args: \" + str(args)
bigjob_agent = bigjob.bigjob_agent.bigjob_agent(args)
"
"""]


        # output options (will just be empty files for /bin/sleep)
        jd.output = "saga_localjob.stdout"
        jd.error  = "saga_localjob.stderr"

        # create the job (state: New)
        catjob = js.create_job(jd)

        # check our job's id and state
        print "Job ID    : %s" % (catjob.id)
        print "Job State : %s" % (catjob.state)

        print "\n...starting job...\n"
        catjob.run()

        print "Job ID    : %s" % (catjob.id)
        print "Job State : %s" % (catjob.state)

        print "\nListing active jobs: "
        for job in js.list():
            print " * %s" % job

        # wait for our job to complete
        print "\n...waiting for job...\n"
        catjob.wait()

        print "Job State : %s" % (catjob.state)
        print "Exitcode  : %s" % (catjob.exit_code)

    except saga.SagaException, ex:
        print "An exception occured: %s " % ((str(ex)))
        # get the whole traceback in case of an exception -
        # this can be helpful for debugging the problem
        print " *** %s" % ex.traceback
        sys.exit(-1)

if __name__ == "__main__":
    main()

First, these are the methods we are talking about, sorted by signature types:

   def copy      (self,      tgt, flags=None, ttype=None) : pass
   def copy      (self, src, tgt, flags=None, ttype=None) : pass
   def link      (self,      tgt, flags=None, ttype=None) : pass
   def link      (self, src, tgt, flags=None, ttype=None) : pass
   def move      (self,      tgt, flags=None, ttype=None) : pass
   def move      (self, src, tgt, flags=None, ttype=None) : pass

   def get_size  (self,           flags=None, ttype=None) : pass
   def get_size  (self,      tgt, flags=None, ttype=None) : pass
   def remove    (self,           flags=None, ttype=None) : pass
   def remove    (self,      tgt, flags=None, ttype=None) : pass

   def is_dir    (self,                       ttype=None) : pass
   def is_dir    (self,      tgt,             ttype=None) : pass
   def is_entry  (self,                       ttype=None) : pass
   def is_entry  (self,      tgt,             ttype=None) : pass
   def is_file   (self,                       ttype=None) : pass
   def is_file   (self,      tgt,             ttype=None) : pass
   def is_link   (self,                       ttype=None) : pass
   def is_link   (self,      tgt,             ttype=None) : pass
   def read_link (self,                       ttype=None) : pass
   def read_link (self,      tgt,             ttype=None) : pass

So, the differences are never in the kwargs, always in the args w/o
keyword.  The kwargs are always flags, i.e. integers, or enums, which
we can also define as integers.

Python does not allow method overloading, only method replacement.
But it *does* allow for variable parameter lists (*args, **kwargs).
We can discard the int types from the args, and count the remaining
arguments:

 d = {'src' : 'src', 'tgt' : 'tgt', 'flags' : 3, 'ttype' : 2}
 l = list (v for v in d.values () if not isinstance (v, int))

 print len(l) # 2

So, one could code the following:


API level:

 class ns.entry :

 # def copy  (self,        tgt, flags=None, ttype=None) : pass
 # def copy  (self, src,   tgt, flags=None, ttype=None) : pass
   def copy  (self,      *args,               **kwargs) :

     non_int_args = args + list (v  for v in kwargs.values () \
                                 if not v == None and not isinstance (v, int))
     if  len (non_int_args) == 2 :
         return self._adaptor.copy_self (args, kwargs)
     if  len (non_int_args) == 3 :
         return self._adaptor.copy      (args, kwargs)

     raise BadParameter ("signature mismatch: copy (url, [url], flags, ttype)")


   All further signature checks are automatically done on CPI /
adaptor level -- although one could easily add type checking for flags
and ttypes at this point.  As the number of cases we need to support
is quite limited, the above should be easy to put in decorators.

Hi Ole,

AndreL: can you please give me an example how the mpi/omp and the 'wayness'
are passed to SGE? I will try to implement this asap!

I don't have access to Anton's machine, but as far as I can see, the

#$ -pe %sway %s

command needs to be more flexible, e.g. it needs to be

#$ -pe mpi %s

on Anton's machine.

This is the line of question:

https://github.com/saga-project/bliss/blob/master/bliss/plugins/sge/cmdlinewrapper.py#L588

I would prefer to do this in a lean way and not to make this a Berlin
Airport style project.

        # Next, we describe the job we want to run. A complete set of job
        # description attributes can be found in the API documentation.
        jd.environment     = {'RUNTIME': '10'}
        jd.wall_time_limit = 1 # minutes
        jd.executable      = '/bin/sleep'
        jd.arguments       = ['$RUNTIME']

        jd.output  = "OUT"
        jd.error = "ERR"

__author__    = "Andre Merzky"
__copyright__ = "Copyright 2012-2013, The SAGA Project"
__license__   = "MIT"

_ADAPTOR_OPTIONS       = [
     {
     'category':      'saga.adaptor.pbsjob',
     'name':          'ptydebug',
     'type':          bool,
     'default':       False,
     'valid_options': [True, False],
     'documentation': """Turns PTYWrapper debugging on or off.""",
     'env_variable':  "SAGA_PTYDEBUG"
     }

2013:03:13 10:19:42 140735128273280 saga.engine           : [ERROR   ] Skipping adaptor saga.adaptors.condor.condorjob: loading failed: 
2013:03:13 10:19:42 140735128273280 saga.engine           : [DEBUG   ]   File "tests/run_tests.py", line 133, in <module>
  File "tests/run_tests.py", line 111, in launch_tests
  File "/Users/oweidner/Projects/saga-python/venv/lib/python2.7/site-packages/nose/core.py", line 284, in run
    return TestProgram(*arg, **kw).success
  File "/Users/oweidner/Projects/saga-python/venv/lib/python2.7/site-packages/nose/core.py", line 118, in __init__
    **extra_args)
  File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/unittest/main.py", line 95, in __init__
    self.runTests()
  File "/Users/oweidner/Projects/saga-python/venv/lib/python2.7/site-packages/nose/core.py", line 197, in runTests
    result = self.testRunner.run(self.test)
  File "/Users/oweidner/Projects/saga-python/venv/lib/python2.7/site-packages/nose/core.py", line 61, in run
    test(result)
  File "/Users/oweidner/Projects/saga-python/venv/lib/python2.7/site-packages/nose/suite.py", line 176, in __call__
    return self.run(*arg, **kw)
  File "/Users/oweidner/Projects/saga-python/venv/lib/python2.7/site-packages/nose/suite.py", line 223, in run
    test(orig)
  File "/Users/oweidner/Projects/saga-python/venv/lib/python2.7/site-packages/nose/suite.py", line 176, in __call__
    return self.run(*arg, **kw)
  File "/Users/oweidner/Projects/saga-python/venv/lib/python2.7/site-packages/nose/suite.py", line 223, in run
    test(orig)
  File "/Users/oweidner/Projects/saga-python/venv/lib/python2.7/site-packages/nose/suite.py", line 176, in __call__
    return self.run(*arg, **kw)
  File "/Users/oweidner/Projects/saga-python/venv/lib/python2.7/site-packages/nose/suite.py", line 223, in run
    test(orig)
  File "/Users/oweidner/Projects/saga-python/venv/lib/python2.7/site-packages/nose/case.py", line 45, in __call__
    return self.run(*arg, **kwarg)
  File "/Users/oweidner/Projects/saga-python/venv/lib/python2.7/site-packages/nose/case.py", line 133, in run
    self.runTest(result)
  File "/Users/oweidner/Projects/saga-python/venv/lib/python2.7/site-packages/nose/case.py", line 151, in runTest
    test(result)
  File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/unittest/case.py", line 391, in __call__
    return self.run(*args, **kwds)
  File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/unittest/case.py", line 327, in run
    testMethod()
  File "/Users/oweidner/Projects/saga-python/venv/lib/python2.7/site-packages/nose/case.py", line 197, in runTest
    self.test(*self.arg)
  File "/Users/oweidner/Projects/saga-python/tests/unittests/api/job/test_job.py", line 52, in test_job_service_invalid_url
    js = saga.job.Service(invalid_url, tc.session)
  File "/Users/oweidner/Projects/saga-python/venv/lib/python2.7/site-packages/saga_python-0.9.2-py2.7.egg/saga/utils/test_config.py", line 224, in session
    s = saga.Session (default=False)
  File "/Users/oweidner/Projects/saga-python/venv/lib/python2.7/site-packages/saga_python-0.9.2-py2.7.egg/saga/session.py", line 93, in __init__
    saga.base.SimpleBase.__init__ (self)
  File "/Users/oweidner/Projects/saga-python/venv/lib/python2.7/site-packages/saga_python-0.9.2-py2.7.egg/saga/base.py", line 20, in __init__
    self._engine    = saga.engine.engine.Engine ()
  File "/Users/oweidner/Projects/saga-python/venv/lib/python2.7/site-packages/saga_python-0.9.2-py2.7.egg/saga/utils/singleton.py", line 21, in __call__
    cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
  File "/Users/oweidner/Projects/saga-python/venv/lib/python2.7/site-packages/saga_python-0.9.2-py2.7.egg/saga/engine/engine.py", line 148, in __init__
    self._load_adaptors ()
  File "/Users/oweidner/Projects/saga-python/venv/lib/python2.7/site-packages/saga_python-0.9.2-py2.7.egg/saga/engine/engine.py", line 213, in _load_adaptors
    self._logger.trace ()