Changeset 87

Timestamp:

06/30/06 20:24:59 (3 years ago)

Author:

robin

Message:

significant rationalization of process management

Files:

• slavetools/trunk/lib/slavetools/entrypoints.py (added)
• slavetools/trunk/lib/slavetools/noseplug.py (modified) (12 diffs)
• slavetools/trunk/lib/slavetools/optionsapi.py (moved) (moved from slavetools/trunk/lib/slavetools/cliapi.py) (7 diffs)
• slavetools/trunk/lib/slavetools/slaveapi.py (added)
• slavetools/trunk/lib/slavetools/slaveapi_posix.py (moved) (moved from slavetools/trunk/lib/slavetools/clitool.py) (15 diffs)
• slavetools/trunk/setup.py (modified) (2 diffs)
• slavetools/trunk/slavetools.rst (moved) (moved from slavetools/trunk/optrun.rst) (3 diffs)

slavetools/trunk/lib/slavetools/noseplug.py

@@ -6 +6 @@
-protocol in unsupported ways.
-
-
-overrides
-
+
+start/restart and
+
-This plugin does not attempt to introspect or otherwise monkey with
-sys.argv. It is driven entirely by: the options passed in by nose; the
@@ -14 +14 @@
-there corresponding default values;
-
-By default all options defined by this plugin are prefixed with `slave`.
-This can be changed by derving a from SlaveService and modfying the class
-attribute OPT_PREFIX. command line help automaticaly substitutes the default
-for the value of OPT_PREFIX when the help is generated.
-
-In `begin` (ie., before any tests run), this plugin attempt to launch the
-slave. Excepting certain plugin specific options (listed below); `begin` will
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-python's standard `subprocess.Popen` is used to launch the process.  The
@@ -38 +57 @@
-plugin.
-
-
-
-
-
-
-
+
+
+
+
+
-NOTE:setuptools test - If you are running tests through setuptools,
-`finalize` does not get called, however the next test run will attempt to
@@ -54 +72 @@
-from subprocess import Popen
-from nose.plugins import Plugin
-
-
+
+
+
+
+
+
-
-PRUNE_SLAVE_LONGOPTS=['--dont-forward-opts']
@@ -63 +85 @@
-        help='By default no slave program is invoked. Set this option '
-        'to reflect the program name part of the command line desired '
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-   ('--slave-program-split', dict(
-None
+' '
-        'simplistic (and only supported) splitting technique')),
-   ('--slave-args', dict(
@@ -81 +111 @@
-        'command line is `\' \'.join([SLAVE_PROGRAM, SLAVE_ARGS])`.')),
-   ('--slave-args-split', dict(
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-   ('--slave-more-help', dict(default=False, action='store_true'))
-    ]
@@ -102 +119 @@
-USAGE="""\
-This plugin attempts to manage the slave process by inspecting the options:
-
-
-
+
-
-In all cases The following options are *NEVER* forwarded: 
-%(NEVER_FORWARDS)s
-
-
-
-
-
+
+
-
-nosetest --slave-more-help will print a much more detailed description of
@@ -123 +136 @@
-    OPT_PREFIX='slave-'
-    LOGCHAN='slaveservice'
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-    def help(self):
-        # used for help generation only, changing this does *not* effect 
@@ -130 +161 @@
-            self.usage.replace('--slave-','--%s' % self.OPT_PREFIX))
-
-    SLAVE_OPTIONS=list(prune_shortopts(reduce(update_options, [
-        COMMON_OPTS, COMMON_UNIX_OPTS, START_UNIX_OPTS])))
-
-    # END user customizable class parameters
@@ -181 +210 @@
-        self.slave_args=self._getprefixedattr(options, 'args')
-        self.slave_args_split=self._getprefixedattr(options, 'args_split')
-
-
+
+
+
+
+
+
+
-        if not self.slave_program:
-            self.enabled=False
-
-    def _getprefixedattr(self, options, attr):
-
-
-
+ 
+ 
+ 
- 
-    def prepare_slave_program(self):
@@ -259 +293 @@
-        (typical) idiom of read pid, kill if read, write pid then that child
-        becomes a suicide. (this abnormal termination will be noticed by this
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
-        """
-
+
+
-        abspidfile = None
-        log = logging.getLogger(self.LOGCHAN)
-=
-=
-=
+ = 
+ = 
+ = 
-        try:
-            os.chdir(options.directory)
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-            p = Popen(args)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-                log.warning(
-
+
+
+
-            else:
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-        finally:
-            os.chdir(cwd)
@@ -348 +360 @@
-    def finalize(self, result):
-        log = logging.getLogger(self.LOGCHAN)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+

slavetools/trunk/lib/slavetools/optionsapi.py

@@ -3 +3 @@
-"""
-__all__=(
-fmt_msg get_log get_program_name 
-add_options_common update_options option_defaults
+
+update_options option_defaults build_parser
-        'prune_shortopts prefix_longopts '
-        'build_parser run_wrapper run_common '
-        ).split()
-
-sys, os, logging, optparse, types
+optparse
-
-COMMON_OPTS=[
@@ -27 +26 @@
-            default='',
-            help='Read a python syntax configuration file by importing it.')),
-
+ 
-            action='store_true',
-            default=False, help=
@@ -47 +46 @@
-
-def add_options(parser, options):
+    """add a list of options to an optparse.OptionParser compatible thing."""
-    for opt in options:
-        if not isinstance(opt[-1], dict):
@@ -53 +53 @@
-
-def prune_shortopts(options):
+    """return a copy of options with all short options removed.
+    
+    transformations are::
+
+        (shopt, lopt, kwargs, ...)  => (lopt, kwargs, ...)
+        (lopt, shopt, kwargs, ...)  => (lopt, kwargs, ...)
+        (shopt, kwargs)             => ** discarded completely **
+        (lopt, kwargs, ...)         => (lopt, kwargs, ...)
+
+    """
-    for opt in options:
-        if opt[0].startswith('--'):
@@ -98 +108 @@
-    both lists any items after the first dict are ignored (and will not be
-    reflected in the result)
-
+
+
+
+
+
-    """
-    optmaps=[]
@@ -120 +134 @@
- 
-def add_options_common(parser, 
- # intentionaly *not* copying the list
+[:]
-    ):
+    """trivial helper,
+    
+    calls `add_options` with the options defined as COMMON
+    by this module"""
-    add_options(parser, options)
-
-def get_program_name(argv=None):
-    if argv is None:
-        argv=sys.argv
-    return os.path.splitext(os.path.basename(argv[0]))[0]
-
-def get_log(options, defaultlogger=''):
-    """get the logger spefified by COMMON_OPTS"""
-    if options.logger is None:
-        log = logging.getLogger(defaultlogger)
-    # this call to basicConfig does nothing if the caller has allready
-    # configured logging or called any of the logging module top level loging
-    # functions
-    logging.basicConfig()
-    return log
-
-def run_wrapper(programname, options, args, run, *runargs, **runkw):
-    """handles the common options
-    
-    including set and restore of working directory.
-
-    exists mainly to make it easy to do `os.chdri(cwd)` after run completes
-    """
-
-    rcode = -1
-    log = get_log(options, programname)
-    if options.loglevel is not None:
-        try:
-            lvl=int(options.loglevel)
-        except ValueError:
-            try:
-                lvl=getattr(logging,options.loglevel)
-            except AttributeError:
-                # the default configuration of the loggin package will show
-                # this message
-                log.warn("Unable to set loglevel %s" % str(options.loglevel))
-                lvl = None
-        if lvl is not None:
-            log.setLevel(lvl)
-    if options.wingdbstub:
-        try:
-            import wingdbstub
-        except ImportError:
-            log.warning(
-                    'Unable to import wingdbstub, '
-                    'have you copied it into place ?')
-    cwd = os.getcwd()
-    try:
-        os.chdir(options.directory)
-        # do the path dance *after* changing directory and *before* reading
-        # the config.
-        if options.insert_path:
-            sys.path[:1]=options.insert_path.split(os.path.sep)
-        if options.append_path:
-            sys.path.extend(options.append_path.split(os.path.sep))
-        configfile={}
-        try:
-            if options.configfile:
-                execfile(options.configfile, configfile)
-        except IOError,e:
-            log.warning(
-                'Failed to read configuration file %s' % 
-                options.configfile)
-        conf = type('confile',(),configfile)
-        rcode = run(programname, options, args, conf, *runargs, **runkw)
-    finally:
-        os.chdir(cwd)
-    return rcode
-
-def fmt_msg(msgkey, msgfmtmap, extrafmtargs, **fmtargs):
-    fmtargs.update(extrafmtargs)
-    msgfmt = msgfmtmap[msgkey]
-    if callable(msgfmt):
-        return msgfmt(**fmtargs)
-    else:
-        return msgfmt % fmtargs
-
-def build_parser(optparsekwopts=None, *optlists):
+    """build an optparse.OptionParser from the argument option *lists*."""
-    if optparsekwopts is None:
-        optparsekwopts = {}
@@ -280 +223 @@
-    return type('options',(),defaults)
-
-
-def run_common(argv, commandmap, msgfmtmap=dict(
-        LIST_AVAILABLE_COMMANDS='Available commands are: %(commandnamelist)s',
-        ERROR_UNKNOWN_COMMAND='ERROR: Unknown command: %(commandname)s',
-        SHOW_USAGE='%(commandname)s:%(commandusage)s'
-        ), 
-    extrafmtargs={},
-    stderr=sys.stderr, stdout=sys.stdout):
-    if not argv:
-        argv=sys.argv
-    def show_usage():
-        usage=[(command, build_parser(
-                *bpargs, **bpkwargs).get_usage()) 
-                for command, (_, build_parser, bpargs, bpkwargs,
-                        _1, _2, _3) in commandmap.items()]
-        print >> stdout, fmt_msg(
-            'LIST_AVAILABLE_COMMANDS', msgfmtmap, extrafmtargs,
-            commandnamelist =','.join(commandmap.keys()))
-        for c,u in usage:
-            print >> stdout
-            print >> stdout, fmt_msg('SHOW_USAGE', msgfmtmap, extrafmtargs,
-                commandname=c, commandusage=u)
-    if (len(argv) >= 2 and argv[1].find('--help') == 0):
-        show_usage()
-        return
-    if len(argv)==1 or argv[1].startswith('-'):
-        command = sorted(
-            [(cvals[0], c) for c, cvals in commandmap.items()])[0][1]
-    else:
-        command=argv[1]
-        argv=len(argv) > 1 and argv[1:] or []
-    if command not in commandmap:
-        print >> stderr, fmt_msg('ERROR_UNKNOWN_COMMAND', msgfmtmap,
-                extrafmtargs, commandname=command)
-        show_usage()
-        return
-    build_parser, bpargs, bpkwargs = commandmap[command][1:4]
-    parser = build_parser(*bpargs, **bpkwargs)
-    options, args = parser.parse_args(argv[1:])
-    runner, runxargs, runxkwargs = commandmap[command][4:]
-    return runner(parser.get_prog_name(), options, args, *runxargs, **runxkwargs)
- 
-

slavetools/trunk/lib/slavetools/slaveapi_posix.py

@@ -1 +1 @@
-#!/usr/bin/env python
-
-
-
-
+
+
+
+
+
-
-__all__=(
-unix kill_un
+posix kill_pos
-        'daemonize_posix '
-unix start_unix_not_optrun stop_unix main_un
+posix start_posix_noforward stop_posix main_pos
-        'minimal_daemon_example '
-UNIX_OPTS START_UNIX_OPTS STOP_UNIX_OPTS MAIN_UN
+POSIX_OPTS START_POSIX_OPTS STOP_POSIX_OPTS MAIN_POS
-        #COMMANDMAP is typicaly defined by many modules so its not included in
-        #all
@@ -16 +17 @@
-
-def pidfile_name(options, program=None):
+    """Workout the pidfile name according to slavetool conventions."""
-    pidfile = getattr(options, 'pidfile', None)
-    if not pidfile:
+        # if pidfile is not set derive based on program name. we only
+        # inspect sys.argv as a last resort. typicaly the caller will
+        # supply a non None value for program
-        if program is not None:
-            argv=[program]
-        else:
-            argv=sys.argv
-
-
+
+
+
+
+
+
+
+
+
+
+
-    return pidfile
-
-def write_pidfile(pidfile, pid, log=None):
+    """Write a pidfile and return the absoloute path of the file written
+
+    If the write succedes return:
+     (abspidfile, True)
+    Otherwise:
+     (abspidfile, False)
+
+    NOTE: this function does *not* take options into account. The caller
+    is expected to do this. (see pidfile_name)"""
-    try:
-        pidf = open(pidfile, 'w+b')
@@ -38 +61 @@
-    
-def read_pidfile(pidfile, defaultpid=None):
+    """read a pid from a file but return defaultpid if it can't be read.
+
+    If the pid is read *AND* successfully converted to an int return:
+     (pid, True)
+    Otherwise:
+     (defaultpid, False)
+    """
-    if pidfile and os.path.exists(pidfile):
-        try:
-            return (int(open(pidfile).read()), True)
-, e
+
-            pass
-    return (defaultpid,False)
-
-un
+pos
-    """Determine if a process has terminated.
-
@@ -66 +96 @@
-    return rcode
-   
-un
+pos
-    options, 
-    program=None, 
-
+# If set prevents *all* attempts to read or delete pidfiles
-    killsignalname='SIGINT',
-    abspidfile=None, # usualy you pass pid or derive pidfile from options
@@ -79 +109 @@
-    allwaysremovestalepidfile=True
-    ):
+    """kill the posix process implied by the arguments.
+
+    `pid`: defaults to None, setting this disables *all* attempts to read
+    or delete the pidfile, irrespective of other options. 
+
+    `abdpidfile`: defaults to None, setting this disables *all* attempts to
+    gues the pidfile name. Provided `pid` is None, this is *exactly* the file
+    that is read and possibly deleted.
+
+    `removepidfile`: provided that `pid` is None setting this option will cause
+    the derived pidfile to be deleted if it was successfully read. defaults to
+    False.
+
+    `allwaysremovestalepidfile`: by default, and provided `pid` is None, this
+    function will allways delete the pidfile it derives if: a pid was read from
+    that file AND the process does not exist (as determined by os.kill).
+    Setting this argument True will leave stale pidfiles alone.
+
+    `killsignalname`: If a pid can be derived from the arguments AND provided 
+    that the derived pid is *not* equivelent to os.getpid() (the current 
+    process!) an attempt will be made to kill it using the python signal module 
+    attribute corresponding to `killsignalname`
+    """
+
-    haveremovedpidfile=False
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
-        # prefer file contents but fall back on options.pid
-        pid,readpidfile=read_pidfile(pidfile, getattr(options, 'pid', None))
@@ -93 +150 @@
-        readpidfile=False
-    if pid and pid != os.getpid(): # Incase the parent wrote the pidfile
-
-
+
+
+
-        else:
-            log.info("killing pid [%s]" % pid)
@@ -100 +158 @@
-            os.kill(pid, getattr(signal, killsignalname))
-        except OSError, e:
-
-
+
+
+
+
+
-                        pidfile, pid))
-
-
+    
+    
-            elif e[0] == errno.ESRCH:
-                log.info("process [%s] is gone" % pid)
-
+
+
-            os.remove(pidfile)
-    return pidfile
-
-def daemonize_posix():
+    """shamelessy looted from Twisted."""
-    # See http://www.erlenstar.demon.co.uk/unix/faq_toc.html#TOC16
-    if os.fork():   # launch child and...
@@ -129 +192 @@
-    return os.getpid() # only get here if this is the grandchild.
- 
-un
-from subprocess import Popen
+pos
+"""helper for trivial init style start/stop scripts."""
-    if not args:
-        print "no command specified"
@@ -136 +199 @@
-    log=get_log(options, progname)
-    pid=None
-un
+pos
-    if not options.dont_forward_opts:
-        ifirst = 0
@@ -155 +218 @@
-    p = Popen(args)
-    pid=p.pid
+    log.info("started: pid=[%s] args: %s" % (str(pid), ' '.join(args)))
-    if pidfile and not options.daemonize:
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
-    if not args:
-        print "no command specified"
@@ -170 +237 @@
-    log=get_log(options, progname)
-    pid=None
-un
+pos
-    log.info("launching [%s] args: %s" % (args[0], ' '.join(args[1:])))
-    p = Popen(args)
@@ -178 +245 @@
-    log.info("started: %s\npid=%s,pidfile=%s" % (' '.join(args), str(pid),
-            pidfile))
-0
+p
- 
-
+
+
-    log=get_log(options, progname)
-    print progname
-    if not args and progname is None and options.pid is None:
-        print ('1st argument must be the program name, '
@@ -191 +258 @@
-    # prefer pidfile
-    program = args and args[0] or progname
-
-
-
-
-
-
-
-
-
+
-    return 0
-
-un
+pos
-    """start/stop compatible main.
-    
-    expected use, write your own main entry point and call this as the
-
+.
-    """
+    pidfilewritten=False
-    log = get_log(options, progname)
-    pidfile = pidfile_name(options, progname)
@@ -213 +273 @@
-        pid=daemonize_posix()
-        if pidfile:
-
-
+
-    log.info("process started")
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-def minimal_daemon_example():
-    """provided for exposition only."""
-    def main(progname, options, args, conf):
-
-
-
+
-        from time import time, sleep
-        try:
@@ -232 +358 @@
-            return 0
-    COMMANDMAP['main'][-2][0]=main
-common(None, commandmap
+withconfig(None, COMMANDMAP
- 
-