Python Multiprocess Pool

Posted on | Edited on | In

Abstract:

This program is to test python process pool.

Content:

if no pool.join and result, you may not see all the output from subprocess. Be careful that the input functions for multiprocess pool can’t be class method, no matter it’s static or not. Lambda expression that represent a function is also invalid.

The expected input funtions should be normal funtion(outside of class). It’s ok to call other process in the normal funtion.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
import multiprocessing
import time
import subprocess

class JobSubmit(object):
    def func(msg):
        for i in xrange(3):
            print msg
            time.sleep(0.5)
        return "done " + msg

def func(msg):
    for i in xrange(3):
        print msg
        time.sleep(0.5)
    return "done " + msg

def func1(msg):
    for i in xrange(3):
        cmd='echo '+msg
        subprocess.call(cmd, shell=True)
        time.sleep(3)
    return "done " + msg

if __name__ == "__main__":
    pool = multiprocessing.Pool(processes=6)
    result = []

    if 0: #this run well
        for i in xrange(10):
            msg = "hello %d" %(i)
            result.append(pool.apply_async(func, (msg, )))
        print "in the middle"
        time.sleep(3)
        for i in xrange(10,15):
            msg = "hello %d" %(i)
            result.append(pool.apply_async(func, (msg, )))
        pool.close()
        pool.join()
        for res in result:
            print res.get()
        print "Sub-process(es) done."

    if 0: #this fail as pool can handle method in class
        js=JobSubmit()
        for i in xrange(10):
            msg = "hello %d" %(i)
            result.append(pool.apply_async(js.func, (msg, )))
        print "in the middle"
        time.sleep(3)
        for i in xrange(10,15):
            msg = "hello %d" %(i)
            result.append(pool.apply_async(js.func, (msg, )))
        pool.close()
        pool.join()
        for res in result:
            print res.get()
        print "Sub-process(es) done."

    #if 0: # run well to call other process
    for i in xrange(5):
        msg = "hello %d" %(i)
        result.append(pool.apply_async(func1, (msg, )))
    print "in the middle"
    time.sleep(0.1)
    for i in xrange(10,15):
        msg = "hello %d" %(i)
        result.append(pool.apply_async(func1, (msg, )))
    pool.close()
    pool.join()
    for res in result:
        print res.get()
    print "Sub-process(es) done."

    if 0:  # call lambda, faild, can't call lambda
        for i in xrange(10):
            msg = "echo %d" %(i)
            result.append(pool.apply_async(lambda x: subprocess.Popen(x, shell=True), (msg, )))
        pool.close()
        pool.join()
        for res in result:
            print res.get()
        print "Sub-process(es) done."

History:

  • 2016-11-09:create the page