Update another example

playground/maxjobs2/compute/wscript

@@ -1,8 +1,10 @@
 #! /usr/bin/env python
-
-# The following example shows how several build processes can collaborate
-# to stop spawning new processes when a special task of type K is executed
-# this may be useful for link tasks for example
+#
+# several processes can wait on a resource limit
+# in this case the maximum amount of "very busy" tasks is one across several processes
+# also:
+# - this example is posix-only (fcntl)
+# - it assumes that the lock file "busy_lock.txt" is already created by another process
 
 def options(opt):
 	opt.add_option('--loops', action='store', type=int, default=5, help='amount of cpu-intensive loops to perform')
@@ -33,60 +35,47 @@ def lock_maxjob(self):
 	# lock the file, telling other build processes to avoid spawining tasks during that time
 	while True:
 		try:
-			self.lockfd = os.open(self.generator.bld.lockfile, os.O_TRUNC | os.O_CREAT | os.O_RDWR)
-			fcntl.flock(self.lockfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+			# each task/thread will have its own "lockfd"
+			# it is important to not create too many of them
+			if not getattr(self, 'lockfd', None):
+				with lock:
+					self.lockfd = os.open(self.generator.bld.lockfile, os.O_TRUNC | os.O_CREAT | os.O_RDWR)
+			with lock:
+				fcntl.flock(self.lockfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
 		except EnvironmentError as e:
 			if e.errno in (errno.EACCES, errno.EAGAIN):
 				time.sleep(0.3)
 				continue
 			raise
-		os.write(self.lockfd, "%d" % os.getpid())
+
+		os.write(self.lockfd, b"%d %s\n" % (os.getpid(), repr(threading.current_thread()).encode()))
 		self.start_time = time.time()
 		break
 
 def release_maxjob(self):
 	# release the lock file
-	print("> long task %d" % (time.time() - self.start_time))
-	try:
-		os.remove(self.generator.bld.lockfile)
-		os.close(self.lockfd)
-	except OSError as e:
-		# of someone else has removed the lock... bad luck! but do not fail here
-		print("unexpected failure", e)
-		pass
-
-def wait_maxjob(self):
-	# wait on the lock file.. up to a certain limit
-	while True:
+	with lock:
 		try:
-			ini = os.stat(self.generator.bld.lockfile).st_mtime
+			fcntl.flock(self.lockfd, fcntl.LOCK_UN)
+			os.close(self.lockfd)
 		except OSError as e:
-			return
-		diff = time.time() - ini
-		if diff > 300: # stale lock file? wait 5 minutes
-			return
-		time.sleep(0.5)
+			# of someone else has removed the lock... bad luck! but do not fail here
+			pass
+		except Exception as e:
+			print(e)
+		#print("lock released!", threading.current_thread())
 
 # the method process is called by threads...
 def process2(self):
-
 	if getattr(self.generator, 'exclusive', False):
-		lock.acquire()
-		try:
-			self.lock_maxjob()
-		finally:
-			lock.release()
-	else:
-		self.wait_maxjob()
+		self.lock_maxjob()
 
+	# regular work
 	ret = self.process_bound_maxjobs()
 
 	if getattr(self.generator, 'exclusive', False):
-		lock.acquire()
-		try:
-			self.release_maxjob()
-		finally:
-			lock.release()
+		self.release_maxjob()
+
 	return ret
 
 def process(self):
@@ -99,5 +88,3 @@ Task.Task.process_bound_maxjobs = Task.Task.process
 Task.Task.process = process
 Task.Task.lock_maxjob = lock_maxjob
 Task.Task.release_maxjob = release_maxjob
-Task.Task.wait_maxjob = wait_maxjob
-

playground/maxjobs2/wscript

@@ -8,6 +8,9 @@ def configure(conf):
 def build(bld):
 	p = bld.srcnode.find_node('compute').abspath()
 
+	lockf = bld.srcnode.make_node('busy_lock.txt')
+	lockf.write('')
+
 	bld.jobs = JOBS = 3
 	for i in range(JOBS):
 		bld(rule='waf configure build -o test%d'%i, cwd=p, always=True)