9393
9494import argparse
9595import datetime
96+ import asyncio
9697import glob
9798import json
9899import hashlib
100+ import multiprocessing
99101import os
100102import tarfile
101103import tempfile
@@ -152,6 +154,99 @@ def build_argument_parser():
152154
153155 return parser
154156
157+ ################################################################################
158+ # Helper class
159+ ################################################################################
160+
161+ class AsyncSubprocessHelper :
162+ def __init__ (self , items , subproc_count = multiprocessing .cpu_count (), verbose = False ):
163+ item_queue = asyncio .Queue ()
164+ for item in items :
165+ item_queue .put_nowait (item )
166+
167+ self .items = items
168+ self .subproc_count = subproc_count
169+ self .verbose = verbose
170+
171+ if 'win32' in sys .platform :
172+ # Windows specific event-loop policy & cmd
173+ asyncio .set_event_loop_policy (asyncio .WindowsProactorEventLoopPolicy ())
174+
175+ async def __get_item__ (self , item , index , size , async_callback , * extra_args ):
176+ """ Wrapper to the async callback which will schedule based on the queue
177+ """
178+
179+ # Wait for the queue to become free. Then start
180+ # running the sub process.
181+ subproc_id = await self .subproc_count_queue .get ()
182+
183+ print_prefix = ""
184+
185+ if self .verbose :
186+ print_prefix = "[{}:{}]: " .format (index , size )
187+
188+ await async_callback (print_prefix , item , * extra_args )
189+
190+ # Add back to the queue, incase another process wants to run.
191+ self .subproc_count_queue .put_nowait (subproc_id )
192+
193+ async def __run_to_completion__ (self , async_callback , * extra_args ):
194+ """ async wrapper for run_to_completion
195+ """
196+
197+ chunk_size = self .subproc_count
198+
199+ # Create a queue with a chunk size of the cpu count
200+ #
201+ # Each run_crossgen invocation will remove an item from the
202+ # queue before running a potentially long running pmi run.
203+ #
204+ # When the queue is drained, we will wait queue.get which
205+ # will wait for when a run_crossgen instance has added back to the
206+ subproc_count_queue = asyncio .Queue (chunk_size )
207+ diff_queue = asyncio .Queue ()
208+
209+ for item in self .items :
210+ diff_queue .put_nowait (item )
211+
212+ for item in range (chunk_size ):
213+ subproc_count_queue .put_nowait (item )
214+
215+ self .subproc_count_queue = subproc_count_queue
216+ tasks = []
217+ size = diff_queue .qsize ()
218+
219+ count = 1
220+ item = diff_queue .get_nowait () if not diff_queue .empty () else None
221+ while item is not None :
222+ tasks .append (self .__get_item__ (item , count , size , async_callback , * extra_args ))
223+ count += 1
224+
225+ item = diff_queue .get_nowait () if not diff_queue .empty () else None
226+
227+ await asyncio .gather (* tasks )
228+
229+ def run_to_completion (self , async_callback , * extra_args ):
230+ """ Run until the item queue has been depleted
231+
232+ Notes:
233+ Acts as a wrapper to abstract the async calls to
234+ async_callback. Note that this will allow cpu_count
235+ amount of running subprocesses. Each time the queue
236+ is emptied, another process will start. Note that
237+ the python code is single threaded, it will just
238+ rely on async/await to start subprocesses at
239+ subprocess_count
240+ """
241+
242+ reset_env = os .environ .copy ()
243+
244+ loop = asyncio .get_event_loop ()
245+ loop .run_until_complete (self .__run_to_completion__ (async_callback , * extra_args ))
246+ loop .close ()
247+
248+ os .environ .update (reset_env )
249+
155250################################################################################
156251# Globals
157252################################################################################
@@ -348,22 +443,22 @@ def __init__(self, crossgen_executable_filename):
348443 self .crossgen_executable_filename = crossgen_executable_filename
349444 self .platform_assemblies_paths_sep = ';' if sys .platform == 'win32' else ':'
350445
351- def crossgen_il_file (self , il_filename , ni_filename , platform_assemblies_paths ):
446+ async def crossgen_il_file (self , il_filename , ni_filename , platform_assemblies_paths ):
352447 """
353448 Runs a subprocess "{crossgen_executable_filename} /nologo /Platform_Assemblies_Paths <path[:path]> /out {ni_filename} /in {il_filename}"
354449 and returns returncode, stdour, stderr.
355450 """
356451 args = self ._build_args_crossgen_il_file (il_filename , ni_filename , platform_assemblies_paths )
357- return self ._run_with_args (args )
452+ return await self ._run_with_args (args )
358453
359- def create_debugging_file (self , ni_filename , debugging_files_dirname , platform_assemblies_paths ):
454+ async def create_debugging_file (self , ni_filename , debugging_files_dirname , platform_assemblies_paths ):
360455 """
361456 Runs a subprocess "{crossgen_executable_filename} /nologo /Platform_Assemblies_Paths <path[:path]> /CreatePerfMap {debugging_files_dirname} /in {il_filename}" on Unix
362457 or "{crossgen_executable_filename} /nologo /Platform_Assemblies_Paths <path[:path]> /CreatePdb {debugging_files_dirname} /in {il_filename}" on Windows
363458 and returns returncode, stdout, stderr.
364459 """
365460 args = self ._build_args_create_debugging_file (ni_filename , debugging_files_dirname , platform_assemblies_paths )
366- return self ._run_with_args (args )
461+ return await self ._run_with_args (args )
367462
368463 def _build_args_crossgen_il_file (self , il_filename , ni_filename , platform_assemblies_paths ):
369464 args = []
@@ -389,15 +484,22 @@ def _build_args_create_debugging_file(self, ni_filename, debugging_files_dirname
389484 args .append (ni_filename )
390485 return args
391486
392- def _run_with_args (self , args ):
487+ async def _run_with_args (self , args ):
393488 """
394489 Creates a subprocess running crossgen with specified set of arguments,
395490 communicates with the owner process - waits for its termination and pulls
396491 returncode, stdour, stderr.
397492 """
398- p = subprocess .Popen (args , stdout = subprocess .PIPE , stderr = subprocess .PIPE )
399- stdout , stderr = p .communicate ()
400- return (p .returncode , stdout .decode (), stderr .decode ())
493+ stdout = None
494+ stderr = None
495+
496+ proc = await asyncio .create_subprocess_shell (" " .join (args ),
497+ stdin = asyncio .subprocess .PIPE ,
498+ stdout = asyncio .subprocess .PIPE ,
499+ stderr = asyncio .subprocess .PIPE )
500+ stdout , stderr = await proc .communicate ()
501+
502+ return (proc .returncode , stdout .decode (), stderr .decode ())
401503
402504
403505def compute_file_hashsum (filename ):
@@ -478,9 +580,9 @@ class FileTypes:
478580 NativeOrReadyToRunImage = 'NativeOrReadyToRunImage'
479581 DebuggingFile = 'DebuggingFile'
480582
481- def run_crossgen (crossgen_executable_filename , il_filename , ni_filename , platform_assemblies_paths , debugging_files_dirname ):
583+ async def run_crossgen (crossgen_executable_filename , il_filename , ni_filename , platform_assemblies_paths , debugging_files_dirname ):
482584 runner = CrossGenRunner (crossgen_executable_filename )
483- returncode , stdout , stderr = runner .crossgen_il_file (il_filename , ni_filename , platform_assemblies_paths )
585+ returncode , stdout , stderr = await runner .crossgen_il_file (il_filename , ni_filename , platform_assemblies_paths )
484586 ni_file_hashsum = compute_file_hashsum (ni_filename ) if returncode == 0 else None
485587 ni_file_size_in_bytes = os .path .getsize (ni_filename ) if returncode == 0 else None
486588 assembly_name = get_assembly_name (il_filename )
@@ -490,7 +592,7 @@ def run_crossgen(crossgen_executable_filename, il_filename, ni_filename, platfor
490592 return [crossgen_assembly_result ]
491593
492594 platform_assemblies_paths = platform_assemblies_paths + [os .path .dirname (ni_filename )]
493- returncode , stdout , stderr = runner .create_debugging_file (ni_filename , debugging_files_dirname , platform_assemblies_paths )
595+ returncode , stdout , stderr = await runner .create_debugging_file (ni_filename , debugging_files_dirname , platform_assemblies_paths )
494596
495597 if returncode == 0 :
496598 filenames = list (filter (lambda filename : not re .match ("^{0}\.ni\." .format (assembly_name ), filename , re .IGNORECASE ) is None , os .listdir (debugging_files_dirname )))
@@ -521,7 +623,7 @@ def create_output_folders():
521623 os .mkdir (debugging_files_dirname )
522624 return ni_files_dirname , debugging_files_dirname
523625
524- def crossgen_corelib (args ):
626+ async def crossgen_corelib (args ):
525627 il_corelib_filename = args .il_corelib_filename
526628 assembly_name = os .path .basename (il_corelib_filename )
527629 ni_corelib_dirname , debugging_files_dirname = create_output_folders ()
@@ -533,7 +635,7 @@ def crossgen_corelib(args):
533635 print ("IL Corelib path does not exist." )
534636 sys .exit (1 )
535637
536- crossgen_results = run_crossgen (args .crossgen_executable_filename , il_corelib_filename , ni_corelib_filename , platform_assemblies_paths , debugging_files_dirname )
638+ crossgen_results = await run_crossgen (args .crossgen_executable_filename , il_corelib_filename , ni_corelib_filename , platform_assemblies_paths , debugging_files_dirname )
537639 shutil .rmtree (ni_corelib_dirname , ignore_errors = True )
538640 save_crossgen_results_to_json_files (crossgen_results , args .result_dirname )
539641
@@ -546,16 +648,25 @@ def crossgen_framework(args):
546648
547649 il_corelib_filename = args .il_corelib_filename
548650 ni_files_dirname , debugging_files_dirname = create_output_folders ()
549- ni_corelib_filename = os .path .join (ni_files_dirname , os .path .basename (il_corelib_filename ))
550- platform_assemblies_paths = [args .core_root ]
551- crossgen_results = run_crossgen (args .crossgen_executable_filename , il_corelib_filename , ni_corelib_filename , platform_assemblies_paths , debugging_files_dirname )
552- save_crossgen_results_to_json_files (crossgen_results , args .result_dirname )
651+ g_Framework_Assemblies = [il_corelib_filename ] + g_Framework_Assemblies
652+
653+ async def run_crossgen_helper (print_prefix , assembly_name ):
654+ platform_assemblies_paths = [args .core_root ]
655+ print ("{}{} {}" .format (print_prefix , args .crossgen_executable_filename , assembly_name ))
656+
657+ if assembly_name == il_corelib_filename :
658+ ni_corelib_filename = os .path .join (ni_files_dirname , os .path .basename (il_corelib_filename ))
659+ crossgen_results = await run_crossgen (args .crossgen_executable_filename , il_corelib_filename , ni_corelib_filename , platform_assemblies_paths , debugging_files_dirname )
660+ save_crossgen_results_to_json_files (crossgen_results , args .result_dirname )
661+ else :
662+ il_filename = os .path .join (args .core_root , assembly_name )
663+ ni_filename = os .path .join (ni_files_dirname , add_ni_extension (assembly_name ))
664+ crossgen_results = await run_crossgen (args .crossgen_executable_filename , il_filename , ni_filename , platform_assemblies_paths , debugging_files_dirname )
665+ save_crossgen_results_to_json_files (crossgen_results , args .result_dirname )
666+
667+ helper = AsyncSubprocessHelper (g_Framework_Assemblies , verbose = True )
668+ helper .run_to_completion (run_crossgen_helper )
553669
554- for assembly_name in g_Framework_Assemblies :
555- il_filename = os .path .join (args .core_root , assembly_name )
556- ni_filename = os .path .join (ni_files_dirname , add_ni_extension (assembly_name ))
557- crossgen_results = run_crossgen (args .crossgen_executable_filename , il_filename , ni_filename , platform_assemblies_paths , debugging_files_dirname )
558- save_crossgen_results_to_json_files (crossgen_results , args .result_dirname )
559670 shutil .rmtree (ni_files_dirname , ignore_errors = True )
560671
561672def load_crossgen_result_from_json_file (json_filename ):
@@ -578,7 +689,7 @@ def dotnet_sdk_enumerate_assemblies(dotnet_sdk_dirname):
578689 filenames = filter (lambda filename : filename != 'System.Private.CoreLib.dll' , filenames )
579690 yield (dirpath , filenames )
580691
581- def crossgen_dotnet_sdk (args ):
692+ async def crossgen_dotnet_sdk (args ):
582693 dotnet_sdk_dirname = tempfile .mkdtemp ()
583694 with tarfile .open (args .dotnet_sdk_filename ) as dotnet_sdk_tarfile :
584695 dotnet_sdk_tarfile .extractall (dotnet_sdk_dirname )
@@ -587,7 +698,7 @@ def crossgen_dotnet_sdk(args):
587698 ni_files_dirname , debugging_files_dirname = create_output_folders ()
588699 ni_corelib_filename = os .path .join (ni_files_dirname , os .path .basename (il_corelib_filename ))
589700 platform_assemblies_paths = [os .path .dirname (il_corelib_filename )]
590- crossgen_results = run_crossgen (args .crossgen_executable_filename , il_corelib_filename , ni_corelib_filename , platform_assemblies_paths , debugging_files_dirname )
701+ crossgen_results = await run_crossgen (args .crossgen_executable_filename , il_corelib_filename , ni_corelib_filename , platform_assemblies_paths , debugging_files_dirname )
591702 save_crossgen_results_to_json_files (crossgen_results , args .result_dirname )
592703
593704 platform_assemblies_paths = [ni_files_dirname ]
@@ -599,7 +710,7 @@ def crossgen_dotnet_sdk(args):
599710 for assembly_name in assembly_names :
600711 il_filename = os .path .join (il_files_dirname , assembly_name )
601712 ni_filename = os .path .join (ni_files_dirname , add_ni_extension (assembly_name ))
602- crossgen_results = run_crossgen (args .crossgen_executable_filename , il_filename , ni_filename , platform_assemblies_paths , debugging_files_dirname )
713+ crossgen_results = await run_crossgen (args .crossgen_executable_filename , il_filename , ni_filename , platform_assemblies_paths , debugging_files_dirname )
603714 save_crossgen_results_to_json_files (crossgen_results , args .result_dirname )
604715 shutil .rmtree (ni_files_dirname , ignore_errors = True )
605716
0 commit comments