Redirecting Outputs from Parallel Subprocess Runs

Say you are call some executable from python in parallel. You don't want to clog the stdout, so you want to save each stdout from executable as a file. Below is what you can do.

	import subprocess
	import multiprocessing as mp
	def run(cmd):
	with open(cmd[-1], 'w') as f:
	return subprocess.run(cmd[:-1], stdout=f)
	numbers = []
	i = 100000000
	for _ in range(10):
	numbers.append(i)
	i //= 2
	commands = [['./a.out', str(x), str(x) + '.txt'] for x in numbers]
	nproc = 2
	pool = mp.Pool(nproc)
	pool.map(run, commands)

view raw multiproc.py hosted with ❤ by GitHub

	#include <stdio.h>
	#include <math.h>
	double log_factorial(int i) {
	double ans = 0;
	if (i <= 1) return ans;
	while (i > 1) ans += log(i--);
	return ans;
	}
	int main(int argc, char** argv) {
	int input = atoi(argv[1]);
	printf("log(%d!) = %f\n", input, log_factorial(input));
	return 0;
	}

view raw run.c hosted with ❤ by GitHub