annotate python/multiproc.py @ 608:9a5cdd49af53

STUB: python/multiproc.py
author Jeff Hammel <k0scist@gmail.com>
date Sun, 02 Feb 2014 11:37:34 -0800
parents 567492ef7b57
children 3ea759399b8f
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
596
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
1 #!/usr/bin/env python
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
2
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
3 """
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
4 multiprocessing/subprocess experiments
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
5 """
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
6
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
7 import argparse
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
8 import os
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
9 import subprocess
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
10 import sys
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
11 import time
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
12 import tempfile
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
13
606
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
14 string = (str, unicode)
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
15
601
7f189613a289 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 600
diff changeset
16 class Process(subprocess.Popen):
7f189613a289 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 600
diff changeset
17 """why would you name a subprocess object Popen?"""
7f189613a289 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 600
diff changeset
18
606
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
19 # http://docs.python.org/2/library/subprocess.html#popen-constructor
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
20 defaults = {'buffsize': 1, # line buffered
603
bd30deecdf4a STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 602
diff changeset
21 }
bd30deecdf4a STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 602
diff changeset
22
606
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
23 def __init__(self, command, **kwargs):
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
24
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
25 # setup arguments
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
26 self.command = command
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
27 _kwargs = self.defaults.copy()
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
28 _kwargs.update(kwargs)
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
29
607
567492ef7b57 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 606
diff changeset
30 # on unix, ``shell={True|False}`` should always come from the
567492ef7b57 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 606
diff changeset
31 # type of command (string or list)
567492ef7b57 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 606
diff changeset
32 if not mswindows:
567492ef7b57 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 606
diff changeset
33 _kwargs['shell'] = isinstance(command, string)
567492ef7b57 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 606
diff changeset
34
606
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
35 # output buffer
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
36 self.output_buffer = tempfile.SpooledTemporaryFile()
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
37 self.location = 0
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
38 self.output = []
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
39 _kwargs['stdout'] = self.output_buffer
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
40
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
41 # launch subprocess
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
42 self.start = time.time()
607
567492ef7b57 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 606
diff changeset
43 subprocess.Popen.__init__(self, command, **_kwargs)
606
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
44
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
45 def wait(self, maxtime=None, sleep=1.):
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
46 """
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
47 maxtime -- timeout in seconds
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
48 sleep -- number of seconds to sleep between polling
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
49 """
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
50 while self.poll() is None:
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
51
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
52 # check for timeout
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
53 curr_time = time.time()
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
54 run_time = curr_time - self.start
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
55 if run_time > maxtime:
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
56 # TODO: read from output
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
57 return process.kill()
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
58
607
567492ef7b57 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 606
diff changeset
59 # read from output buffer
567492ef7b57 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 606
diff changeset
60 self.output_buffer.seek(self.location)
567492ef7b57 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 606
diff changeset
61 read = self.output_buffer.read()
567492ef7b57 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 606
diff changeset
62 self.location += len(read)
567492ef7b57 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 606
diff changeset
63
606
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
64 # naptime
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
65 if sleep:
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
66 time.sleep(sleep)
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
67
608
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
68 # reset tempfile
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
69 output.seek(0)
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
70
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
71 return self.returncode # set by ``.poll()``
606
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
72
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
73 def commandline(self):
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
74 """returns string of command line"""
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
75
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
76 if isinstance(self.command, string):
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
77 return self.command
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
78 return subprocess.list2cmdline(self.command)
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
79
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
80 __str__ = commandline
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
81
596
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
82
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
83 def main(args=sys.argv[1:]):
599
782dc37492c4 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 596
diff changeset
84 """CLI"""
596
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
85
600
a77f7022cc06 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 599
diff changeset
86 # available programs
a77f7022cc06 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 599
diff changeset
87 progs = {'yes': ["yes"],
a77f7022cc06 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 599
diff changeset
88 'ping': ['ping', 'google.com']}
a77f7022cc06 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 599
diff changeset
89
a77f7022cc06 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 599
diff changeset
90
599
782dc37492c4 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 596
diff changeset
91 # parse command line
596
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
92 usage = '%prog [options]'
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
93 parser = argparse.ArgumentParser(usage=usage, description=__doc__)
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
94 parser.add_argument("-t", "--time", dest="time",
599
782dc37492c4 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 596
diff changeset
95 type=float, default=4.,
782dc37492c4 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 596
diff changeset
96 help="seconds to run for")
782dc37492c4 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 596
diff changeset
97 parser.add_argument("-s", "--sleep", dest="sleep",
596
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
98 type=float, default=1.,
600
a77f7022cc06 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 599
diff changeset
99 help="sleep this number of seconds between polling")
a77f7022cc06 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 599
diff changeset
100 parser.add_argument("-p", "--prog", dest='program',
606
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
101 choices=progs.keys(), default='ping',
600
a77f7022cc06 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 599
diff changeset
102 help="subprocess to run")
606
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
103 parser.add_argument("--list-programs", dest='list_programs',
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
104 action='store_true', default=False,
5b561c7b2005 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 603
diff changeset
105 help="list available programs")
596
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
106 options = parser.parse_args(args)
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
107
608
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
108 # list programs
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
109 if options.list_programs:
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
110 for key in sorted(progs.keys()):
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
111 print ('{}: {}'.format(key, subprocess.list2cmdline(progs[key])))
599
782dc37492c4 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 596
diff changeset
112
782dc37492c4 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 596
diff changeset
113 # select program
600
a77f7022cc06 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 599
diff changeset
114 prog = progs[options.program]
599
782dc37492c4 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 596
diff changeset
115
608
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
116 proc = Process(prog)
596
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
117
608
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
118 # # start the main subprocess loop
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
119 # # TODO -> OO
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
120 # output = tempfile.SpooledTemporaryFile()
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
121 # start = time.time()
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
122 # proc = subprocess.Popen(prog, stdout=output)
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
123 # location = 0
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
124 # while proc.poll() is None:
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
125 # curr_time = time.time()
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
126 # run_time = curr_time - start
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
127 # if run_time > options.time:
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
128 # proc.kill()
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
129 # output.seek(location)
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
130 # read = output.read()
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
131 # location += len(read)
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
132 # print ('[{}] {}\n{}'.format(run_time, read, '-==-'*10))
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
133 # if options.sleep:
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
134 # time.sleep(options.sleep)
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
135
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
136 # # reset tempfile
9a5cdd49af53 STUB: python/multiproc.py
Jeff Hammel <k0scist@gmail.com>
parents: 607
diff changeset
137 # output.seek(0)
596
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
138
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
139 n_lines = len(output.read().splitlines())
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
140 print ("{}: {} lines".format(subprocess.list2cmdline(prog), n_lines))
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
141
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
142 if __name__ == '__main__':
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
143 main()