Module src.jupyter_process_manager
Expand source code
# Standard library imports
# Third party imports
# Local imports
from jupyter_process_manager.class_processes_manager import \
JupyterProcessesManager
__all__ = ["JupyterProcessesManager"]
Sub-modules
src.jupyter_process_manager.class_one_process
-
Module with class for all operations with one process
src.jupyter_process_manager.class_processes_manager
-
Main module with class which helps in handing many processes
src.jupyter_process_manager.function_wrapper
-
Decorators and wrappers to redirect streams for processes
src.jupyter_process_manager.other
-
Module with other small functions
src.jupyter_process_manager.test_functions
src.jupyter_process_manager.widget
-
Module with functions to handle all ipywidget stuff
Classes
class JupyterProcessesManager (str_dir_for_output, is_to_delete_previous_outputs=True)
-
Class to handle creation and showing output for many processes
Initialize object
Args
str_dir_for_output
:str
- Path to dir where to store output files
is_to_delete_previous_outputs
:bool
, optional-
Flag if to delete previous output files
Expand source code
class JupyterProcessesManager(object): """Class to handle creation and showing output for many processes""" @char def __init__( self, str_dir_for_output, is_to_delete_previous_outputs=True ): """Initialize object Args: str_dir_for_output (str): Path to dir where to store output files is_to_delete_previous_outputs (bool, optional): \ Flag if to delete previous output files """ self.int_processes = 0 # Create directory where to store processes output str_processes_output_dir = os.path.join( str_dir_for_output, "processes_output") if not os.path.exists(str_processes_output_dir): os.makedirs(str_processes_output_dir) self.str_dir_for_output = str_processes_output_dir if is_to_delete_previous_outputs: self._delete_all_previous_outputs() self.dict_all_processes_by_id = OrderedDict() self.dict_alive_processes_by_id = OrderedDict() self.dt_processes_started_at = None @char def add_function_to_processing(self, func_to_process, *args, **kwargs): """Start running function as process Args: func_to_process (function): Function to add for processing """ new_process = OneProcess(self.str_dir_for_output) new_process.start_process(func_to_process, *args, **kwargs) if not self.dict_alive_processes_by_id: self.dt_processes_started_at = datetime.datetime.now() self.dict_all_processes_by_id[new_process.int_process_id] = new_process self.dict_alive_processes_by_id[new_process.int_process_id] = new_process @char def debug_run_of_1_function(self, func_to_process, *args, **kwargs): """""" new_process = OneProcess(self.str_dir_for_output) new_process.debug_run_of_the_func(func_to_process, *args, **kwargs) self.dict_all_processes_by_id[new_process.int_process_id] = new_process self.dict_alive_processes_by_id[new_process.int_process_id] = new_process @char def wait_till_all_processes_are_over( self, int_seconds_step=10, int_max_processes_to_show=20 ): """""" clear_output(wait=True) self.print_info_about_running_processes( int_max_processes_to_show=int_max_processes_to_show) try: while True: if not self.dict_alive_processes_by_id: break with yaspin() as sp: for i in range(int_seconds_step): sp.text = "Updating in {} seconds".format( int_seconds_step - i) sleep(1) clear_output(wait=True) self.print_info_about_running_processes( int_max_processes_to_show=int_max_processes_to_show) except KeyboardInterrupt: clear_output(wait=True) print("Interrupting running processes") self.terminate_all_alive_processes() print("---> Done") print("All processes were finished") @char def terminate_all_alive_processes(self): """""" print("Terminating all alive processes") for process_num in list(self.dict_alive_processes_by_id): print("---> Terminate process: ", process_num + 1) process_obj = self.dict_alive_processes_by_id[process_num] process_obj.terminate() if not process_obj.is_alive(): self.dict_alive_processes_by_id.pop(process_num, None) print("------> Done") @char def show_jupyter_widget( self, int_seconds_step=10, int_max_processes_to_show=20 ): """""" clear_output(wait=True) APP_GUI = widget.create_jupyter_widget(self) display(APP_GUI) with widget.OUTPUT_PROCESSES_CONDITIONS: self.print_info_about_running_processes( int_max_processes_to_show=int_max_processes_to_show) try: while True: if not self.dict_alive_processes_by_id: break for i in range(int_seconds_step): sleep(1) get_ipython().kernel.do_one_iteration() widget.OUTPUT_PROCESSES_CONDITIONS.clear_output(wait=True) with widget.OUTPUT_PROCESSES_CONDITIONS: self.print_info_about_running_processes( int_max_processes_to_show=int_max_processes_to_show) except KeyboardInterrupt: clear_output(wait=True) print("Interrupting running processes") self.terminate_all_alive_processes() print("---> Done") print("All processes were finished") @char def print_info_about_running_processes(self, int_max_processes_to_show=20): """""" display(HTML("<h2>Processes conditions:</h2>")) # print("Conditions of the processes:") if self.dt_processes_started_at: timedelta = datetime.datetime.now() - self.dt_processes_started_at print("Working for:", timedelta_nice_format(timedelta)) self.print_table_with_conditions(int_max_processes_to_show=20) print( "ALIVE PROCESSES: ", len(self.dict_alive_processes_by_id), "/", len(self.dict_all_processes_by_id)) @char def print_table_with_conditions(self, int_max_processes_to_show=20): """""" list_list_processes_info = [] list_headers = ["Process Id", "Status", "Runtime"] for process_num in list(self.dict_all_processes_by_id): list_process_info = [] process_obj = self.dict_all_processes_by_id[process_num] # Delete if process is not alive if process_num in list(self.dict_alive_processes_by_id): if not process_obj.is_alive(): self.dict_alive_processes_by_id.pop(process_num, None) # "Process Id" list_process_info.append(process_num) # "Status" list_process_info.append(process_obj.str_status) # "Runtime" str_runtime = process_obj.get_how_long_this_process_is_running() list_process_info.append(str_runtime) list_list_processes_info.append(list_process_info) if len(list_list_processes_info) > int_max_processes_to_show: list_list_additional_columns = [] list_list_additional_columns.append(["---", "---", "---"]) str_num = str( len(list_list_processes_info) - int_max_processes_to_show) list_list_additional_columns.append( ["Processes", "Hidden", str_num]) list_list_additional_columns.append(["---", "---", "---"]) list_list_processes_info = \ list_list_additional_columns + list_list_processes_info[:20] # github psql orgtbl pretty print(tabulate( list_list_processes_info, headers=list_headers, tablefmt="pretty")) def _delete_all_previous_outputs(self): """Delete outputs of previous processes""" for str_filename in os.listdir(self.str_dir_for_output): str_file_path = os.path.join(self.str_dir_for_output, str_filename) if not os.path.isfile(str_file_path): continue if "stdout_" in str_filename or "stderr_" in str_filename: try: os.remove(str_file_path) except Exception as ex: print("Cant DELETE previous thread file: ", str_filename) print(ex)
Methods
def add_function_to_processing(self, func_to_process, *args, **kwargs)
-
Start running function as process
Args
func_to_process
:function
- Function to add for processing
Expand source code
@char def add_function_to_processing(self, func_to_process, *args, **kwargs): """Start running function as process Args: func_to_process (function): Function to add for processing """ new_process = OneProcess(self.str_dir_for_output) new_process.start_process(func_to_process, *args, **kwargs) if not self.dict_alive_processes_by_id: self.dt_processes_started_at = datetime.datetime.now() self.dict_all_processes_by_id[new_process.int_process_id] = new_process self.dict_alive_processes_by_id[new_process.int_process_id] = new_process
def debug_run_of_1_function(self, func_to_process, *args, **kwargs)
-
Expand source code
@char def debug_run_of_1_function(self, func_to_process, *args, **kwargs): """""" new_process = OneProcess(self.str_dir_for_output) new_process.debug_run_of_the_func(func_to_process, *args, **kwargs) self.dict_all_processes_by_id[new_process.int_process_id] = new_process self.dict_alive_processes_by_id[new_process.int_process_id] = new_process
def print_info_about_running_processes(self, int_max_processes_to_show=20)
-
Expand source code
@char def print_info_about_running_processes(self, int_max_processes_to_show=20): """""" display(HTML("<h2>Processes conditions:</h2>")) # print("Conditions of the processes:") if self.dt_processes_started_at: timedelta = datetime.datetime.now() - self.dt_processes_started_at print("Working for:", timedelta_nice_format(timedelta)) self.print_table_with_conditions(int_max_processes_to_show=20) print( "ALIVE PROCESSES: ", len(self.dict_alive_processes_by_id), "/", len(self.dict_all_processes_by_id))
def print_table_with_conditions(self, int_max_processes_to_show=20)
-
Expand source code
@char def print_table_with_conditions(self, int_max_processes_to_show=20): """""" list_list_processes_info = [] list_headers = ["Process Id", "Status", "Runtime"] for process_num in list(self.dict_all_processes_by_id): list_process_info = [] process_obj = self.dict_all_processes_by_id[process_num] # Delete if process is not alive if process_num in list(self.dict_alive_processes_by_id): if not process_obj.is_alive(): self.dict_alive_processes_by_id.pop(process_num, None) # "Process Id" list_process_info.append(process_num) # "Status" list_process_info.append(process_obj.str_status) # "Runtime" str_runtime = process_obj.get_how_long_this_process_is_running() list_process_info.append(str_runtime) list_list_processes_info.append(list_process_info) if len(list_list_processes_info) > int_max_processes_to_show: list_list_additional_columns = [] list_list_additional_columns.append(["---", "---", "---"]) str_num = str( len(list_list_processes_info) - int_max_processes_to_show) list_list_additional_columns.append( ["Processes", "Hidden", str_num]) list_list_additional_columns.append(["---", "---", "---"]) list_list_processes_info = \ list_list_additional_columns + list_list_processes_info[:20] # github psql orgtbl pretty print(tabulate( list_list_processes_info, headers=list_headers, tablefmt="pretty"))
def show_jupyter_widget(self, int_seconds_step=10, int_max_processes_to_show=20)
-
Expand source code
@char def show_jupyter_widget( self, int_seconds_step=10, int_max_processes_to_show=20 ): """""" clear_output(wait=True) APP_GUI = widget.create_jupyter_widget(self) display(APP_GUI) with widget.OUTPUT_PROCESSES_CONDITIONS: self.print_info_about_running_processes( int_max_processes_to_show=int_max_processes_to_show) try: while True: if not self.dict_alive_processes_by_id: break for i in range(int_seconds_step): sleep(1) get_ipython().kernel.do_one_iteration() widget.OUTPUT_PROCESSES_CONDITIONS.clear_output(wait=True) with widget.OUTPUT_PROCESSES_CONDITIONS: self.print_info_about_running_processes( int_max_processes_to_show=int_max_processes_to_show) except KeyboardInterrupt: clear_output(wait=True) print("Interrupting running processes") self.terminate_all_alive_processes() print("---> Done") print("All processes were finished")
def terminate_all_alive_processes(self)
-
Expand source code
@char def terminate_all_alive_processes(self): """""" print("Terminating all alive processes") for process_num in list(self.dict_alive_processes_by_id): print("---> Terminate process: ", process_num + 1) process_obj = self.dict_alive_processes_by_id[process_num] process_obj.terminate() if not process_obj.is_alive(): self.dict_alive_processes_by_id.pop(process_num, None) print("------> Done")
def wait_till_all_processes_are_over(self, int_seconds_step=10, int_max_processes_to_show=20)
-
Expand source code
@char def wait_till_all_processes_are_over( self, int_seconds_step=10, int_max_processes_to_show=20 ): """""" clear_output(wait=True) self.print_info_about_running_processes( int_max_processes_to_show=int_max_processes_to_show) try: while True: if not self.dict_alive_processes_by_id: break with yaspin() as sp: for i in range(int_seconds_step): sp.text = "Updating in {} seconds".format( int_seconds_step - i) sleep(1) clear_output(wait=True) self.print_info_about_running_processes( int_max_processes_to_show=int_max_processes_to_show) except KeyboardInterrupt: clear_output(wait=True) print("Interrupting running processes") self.terminate_all_alive_processes() print("---> Done") print("All processes were finished")