Module src.jupyter_process_manager.class_processes_manager

Main module with class which helps in handing many processes

Expand source code
"""Main module with class which helps in handing many processes"""
# Standard library imports
import os
import sys
import logging
from collections import OrderedDict
from time import sleep
import datetime

# Third party imports
from char import char
from IPython.display import clear_output
from IPython.display import display
from IPython import get_ipython
from IPython.display import HTML
from tqdm.auto import tqdm
from tabulate import tabulate
from yaspin import yaspin

# Local imports
from .class_one_process import OneProcess
from .other import timedelta_nice_format
from . import widget


class JupyterProcessesManager(object):
    """Class to handle creation and showing output for many processes"""

    @char
    def __init__(
            self,
            str_dir_for_output,
            is_to_delete_previous_outputs=True
    ):
        """Initialize object

        Args:
            str_dir_for_output (str): Path to dir where to store output files
            is_to_delete_previous_outputs (bool, optional): \
                Flag if to delete previous output files
        """
        self.int_processes = 0
        # Create directory where to store processes output
        str_processes_output_dir = os.path.join(
            str_dir_for_output, "processes_output")
        if not os.path.exists(str_processes_output_dir):
            os.makedirs(str_processes_output_dir)
        self.str_dir_for_output = str_processes_output_dir
        if is_to_delete_previous_outputs:
            self._delete_all_previous_outputs()
        self.dict_all_processes_by_id = OrderedDict()
        self.dict_alive_processes_by_id = OrderedDict()
        self.dt_processes_started_at = None

    @char
    def add_function_to_processing(self, func_to_process, *args, **kwargs):
        """Start running function as process

        Args:
            func_to_process (function): Function to add for processing
        """
        new_process = OneProcess(self.str_dir_for_output)
        new_process.start_process(func_to_process, *args, **kwargs)
        if not self.dict_alive_processes_by_id:
            self.dt_processes_started_at = datetime.datetime.now()
        self.dict_all_processes_by_id[new_process.int_process_id] = new_process
        self.dict_alive_processes_by_id[new_process.int_process_id] = new_process

    @char
    def debug_run_of_1_function(self, func_to_process, *args, **kwargs):
        """"""
        new_process = OneProcess(self.str_dir_for_output)
        new_process.debug_run_of_the_func(func_to_process, *args, **kwargs)
        self.dict_all_processes_by_id[new_process.int_process_id] = new_process
        self.dict_alive_processes_by_id[new_process.int_process_id] = new_process

    @char
    def wait_till_all_processes_are_over(
            self,
            int_seconds_step=10,
            int_max_processes_to_show=20
    ):
        """"""
        clear_output(wait=True)
        self.print_info_about_running_processes(
            int_max_processes_to_show=int_max_processes_to_show)
        try:
            while True:
                if not self.dict_alive_processes_by_id:
                    break
                with yaspin() as sp:
                    for i in range(int_seconds_step):
                        sp.text = "Updating in {} seconds".format(
                            int_seconds_step - i)
                        sleep(1)
                clear_output(wait=True)
                self.print_info_about_running_processes(
                    int_max_processes_to_show=int_max_processes_to_show)
        except KeyboardInterrupt:
            clear_output(wait=True)
            print("Interrupting running processes")
            self.terminate_all_alive_processes()
            print("---> Done")
        print("All processes were finished")

    @char
    def terminate_all_alive_processes(self):
        """"""
        print("Terminating all alive processes")
        for process_num in list(self.dict_alive_processes_by_id):
            print("---> Terminate process: ", process_num + 1)
            process_obj = self.dict_alive_processes_by_id[process_num]
            process_obj.terminate()
            if not process_obj.is_alive():
                self.dict_alive_processes_by_id.pop(process_num, None)
                print("------> Done")

    @char
    def show_jupyter_widget(
            self,
            int_seconds_step=10,
            int_max_processes_to_show=20
    ):
        """"""
        clear_output(wait=True)
        APP_GUI = widget.create_jupyter_widget(self)
        display(APP_GUI)
        with widget.OUTPUT_PROCESSES_CONDITIONS:
            self.print_info_about_running_processes(
                int_max_processes_to_show=int_max_processes_to_show)
        try:
            while True:
                if not self.dict_alive_processes_by_id:
                    break
                for i in range(int_seconds_step):
                    sleep(1)
                    get_ipython().kernel.do_one_iteration()
                widget.OUTPUT_PROCESSES_CONDITIONS.clear_output(wait=True)
                with widget.OUTPUT_PROCESSES_CONDITIONS:
                    self.print_info_about_running_processes(
                        int_max_processes_to_show=int_max_processes_to_show)
        except KeyboardInterrupt:
            clear_output(wait=True)
            print("Interrupting running processes")
            self.terminate_all_alive_processes()
            print("---> Done")
        print("All processes were finished")

    @char
    def print_info_about_running_processes(self, int_max_processes_to_show=20):
        """"""
        display(HTML("<h2>Processes conditions:</h2>"))

        # print("Conditions of the processes:")
        if self.dt_processes_started_at:
            timedelta = datetime.datetime.now() - self.dt_processes_started_at
            print("Working for:", timedelta_nice_format(timedelta))
        self.print_table_with_conditions(int_max_processes_to_show=20)
        print(
            "ALIVE PROCESSES: ",
            len(self.dict_alive_processes_by_id), "/",
            len(self.dict_all_processes_by_id))

    @char
    def print_table_with_conditions(self, int_max_processes_to_show=20):
        """"""
        list_list_processes_info = []
        list_headers = ["Process Id", "Status", "Runtime"]
        for process_num in list(self.dict_all_processes_by_id):
            list_process_info = []
            process_obj = self.dict_all_processes_by_id[process_num]
            # Delete if process is not alive
            if process_num in list(self.dict_alive_processes_by_id):
                if not process_obj.is_alive():
                    self.dict_alive_processes_by_id.pop(process_num, None)
            # "Process Id"
            list_process_info.append(process_num)
            # "Status"
            list_process_info.append(process_obj.str_status)
            # "Runtime"
            str_runtime = process_obj.get_how_long_this_process_is_running()
            list_process_info.append(str_runtime)
            list_list_processes_info.append(list_process_info)
        if len(list_list_processes_info) > int_max_processes_to_show:
            list_list_additional_columns = []
            list_list_additional_columns.append(["---", "---", "---"])
            str_num = str(
                len(list_list_processes_info) - int_max_processes_to_show)
            list_list_additional_columns.append(
                ["Processes", "Hidden", str_num])
            list_list_additional_columns.append(["---", "---", "---"])
            list_list_processes_info = \
                list_list_additional_columns + list_list_processes_info[:20]
        # github  psql  orgtbl  pretty
        print(tabulate(
            list_list_processes_info, headers=list_headers, tablefmt="pretty"))

    def _delete_all_previous_outputs(self):
        """Delete outputs of previous processes"""
        for str_filename in os.listdir(self.str_dir_for_output):
            str_file_path = os.path.join(self.str_dir_for_output, str_filename)
            if not os.path.isfile(str_file_path):
                continue
            if "stdout_" in str_filename or "stderr_" in str_filename:
                try:
                    os.remove(str_file_path)
                except Exception as ex:
                    print("Cant DELETE previous thread file: ", str_filename)
                    print(ex)

Classes

class JupyterProcessesManager (str_dir_for_output, is_to_delete_previous_outputs=True)

Class to handle creation and showing output for many processes

Initialize object

Args

str_dir_for_output : str
Path to dir where to store output files
is_to_delete_previous_outputs : bool, optional
        Flag if to delete previous output files
Expand source code
class JupyterProcessesManager(object):
    """Class to handle creation and showing output for many processes"""

    @char
    def __init__(
            self,
            str_dir_for_output,
            is_to_delete_previous_outputs=True
    ):
        """Initialize object

        Args:
            str_dir_for_output (str): Path to dir where to store output files
            is_to_delete_previous_outputs (bool, optional): \
                Flag if to delete previous output files
        """
        self.int_processes = 0
        # Create directory where to store processes output
        str_processes_output_dir = os.path.join(
            str_dir_for_output, "processes_output")
        if not os.path.exists(str_processes_output_dir):
            os.makedirs(str_processes_output_dir)
        self.str_dir_for_output = str_processes_output_dir
        if is_to_delete_previous_outputs:
            self._delete_all_previous_outputs()
        self.dict_all_processes_by_id = OrderedDict()
        self.dict_alive_processes_by_id = OrderedDict()
        self.dt_processes_started_at = None

    @char
    def add_function_to_processing(self, func_to_process, *args, **kwargs):
        """Start running function as process

        Args:
            func_to_process (function): Function to add for processing
        """
        new_process = OneProcess(self.str_dir_for_output)
        new_process.start_process(func_to_process, *args, **kwargs)
        if not self.dict_alive_processes_by_id:
            self.dt_processes_started_at = datetime.datetime.now()
        self.dict_all_processes_by_id[new_process.int_process_id] = new_process
        self.dict_alive_processes_by_id[new_process.int_process_id] = new_process

    @char
    def debug_run_of_1_function(self, func_to_process, *args, **kwargs):
        """"""
        new_process = OneProcess(self.str_dir_for_output)
        new_process.debug_run_of_the_func(func_to_process, *args, **kwargs)
        self.dict_all_processes_by_id[new_process.int_process_id] = new_process
        self.dict_alive_processes_by_id[new_process.int_process_id] = new_process

    @char
    def wait_till_all_processes_are_over(
            self,
            int_seconds_step=10,
            int_max_processes_to_show=20
    ):
        """"""
        clear_output(wait=True)
        self.print_info_about_running_processes(
            int_max_processes_to_show=int_max_processes_to_show)
        try:
            while True:
                if not self.dict_alive_processes_by_id:
                    break
                with yaspin() as sp:
                    for i in range(int_seconds_step):
                        sp.text = "Updating in {} seconds".format(
                            int_seconds_step - i)
                        sleep(1)
                clear_output(wait=True)
                self.print_info_about_running_processes(
                    int_max_processes_to_show=int_max_processes_to_show)
        except KeyboardInterrupt:
            clear_output(wait=True)
            print("Interrupting running processes")
            self.terminate_all_alive_processes()
            print("---> Done")
        print("All processes were finished")

    @char
    def terminate_all_alive_processes(self):
        """"""
        print("Terminating all alive processes")
        for process_num in list(self.dict_alive_processes_by_id):
            print("---> Terminate process: ", process_num + 1)
            process_obj = self.dict_alive_processes_by_id[process_num]
            process_obj.terminate()
            if not process_obj.is_alive():
                self.dict_alive_processes_by_id.pop(process_num, None)
                print("------> Done")

    @char
    def show_jupyter_widget(
            self,
            int_seconds_step=10,
            int_max_processes_to_show=20
    ):
        """"""
        clear_output(wait=True)
        APP_GUI = widget.create_jupyter_widget(self)
        display(APP_GUI)
        with widget.OUTPUT_PROCESSES_CONDITIONS:
            self.print_info_about_running_processes(
                int_max_processes_to_show=int_max_processes_to_show)
        try:
            while True:
                if not self.dict_alive_processes_by_id:
                    break
                for i in range(int_seconds_step):
                    sleep(1)
                    get_ipython().kernel.do_one_iteration()
                widget.OUTPUT_PROCESSES_CONDITIONS.clear_output(wait=True)
                with widget.OUTPUT_PROCESSES_CONDITIONS:
                    self.print_info_about_running_processes(
                        int_max_processes_to_show=int_max_processes_to_show)
        except KeyboardInterrupt:
            clear_output(wait=True)
            print("Interrupting running processes")
            self.terminate_all_alive_processes()
            print("---> Done")
        print("All processes were finished")

    @char
    def print_info_about_running_processes(self, int_max_processes_to_show=20):
        """"""
        display(HTML("<h2>Processes conditions:</h2>"))

        # print("Conditions of the processes:")
        if self.dt_processes_started_at:
            timedelta = datetime.datetime.now() - self.dt_processes_started_at
            print("Working for:", timedelta_nice_format(timedelta))
        self.print_table_with_conditions(int_max_processes_to_show=20)
        print(
            "ALIVE PROCESSES: ",
            len(self.dict_alive_processes_by_id), "/",
            len(self.dict_all_processes_by_id))

    @char
    def print_table_with_conditions(self, int_max_processes_to_show=20):
        """"""
        list_list_processes_info = []
        list_headers = ["Process Id", "Status", "Runtime"]
        for process_num in list(self.dict_all_processes_by_id):
            list_process_info = []
            process_obj = self.dict_all_processes_by_id[process_num]
            # Delete if process is not alive
            if process_num in list(self.dict_alive_processes_by_id):
                if not process_obj.is_alive():
                    self.dict_alive_processes_by_id.pop(process_num, None)
            # "Process Id"
            list_process_info.append(process_num)
            # "Status"
            list_process_info.append(process_obj.str_status)
            # "Runtime"
            str_runtime = process_obj.get_how_long_this_process_is_running()
            list_process_info.append(str_runtime)
            list_list_processes_info.append(list_process_info)
        if len(list_list_processes_info) > int_max_processes_to_show:
            list_list_additional_columns = []
            list_list_additional_columns.append(["---", "---", "---"])
            str_num = str(
                len(list_list_processes_info) - int_max_processes_to_show)
            list_list_additional_columns.append(
                ["Processes", "Hidden", str_num])
            list_list_additional_columns.append(["---", "---", "---"])
            list_list_processes_info = \
                list_list_additional_columns + list_list_processes_info[:20]
        # github  psql  orgtbl  pretty
        print(tabulate(
            list_list_processes_info, headers=list_headers, tablefmt="pretty"))

    def _delete_all_previous_outputs(self):
        """Delete outputs of previous processes"""
        for str_filename in os.listdir(self.str_dir_for_output):
            str_file_path = os.path.join(self.str_dir_for_output, str_filename)
            if not os.path.isfile(str_file_path):
                continue
            if "stdout_" in str_filename or "stderr_" in str_filename:
                try:
                    os.remove(str_file_path)
                except Exception as ex:
                    print("Cant DELETE previous thread file: ", str_filename)
                    print(ex)

Methods

def add_function_to_processing(self, func_to_process, *args, **kwargs)

Start running function as process

Args

func_to_process : function
Function to add for processing
Expand source code
@char
def add_function_to_processing(self, func_to_process, *args, **kwargs):
    """Start running function as process

    Args:
        func_to_process (function): Function to add for processing
    """
    new_process = OneProcess(self.str_dir_for_output)
    new_process.start_process(func_to_process, *args, **kwargs)
    if not self.dict_alive_processes_by_id:
        self.dt_processes_started_at = datetime.datetime.now()
    self.dict_all_processes_by_id[new_process.int_process_id] = new_process
    self.dict_alive_processes_by_id[new_process.int_process_id] = new_process
def debug_run_of_1_function(self, func_to_process, *args, **kwargs)
Expand source code
@char
def debug_run_of_1_function(self, func_to_process, *args, **kwargs):
    """"""
    new_process = OneProcess(self.str_dir_for_output)
    new_process.debug_run_of_the_func(func_to_process, *args, **kwargs)
    self.dict_all_processes_by_id[new_process.int_process_id] = new_process
    self.dict_alive_processes_by_id[new_process.int_process_id] = new_process
def print_info_about_running_processes(self, int_max_processes_to_show=20)
Expand source code
@char
def print_info_about_running_processes(self, int_max_processes_to_show=20):
    """"""
    display(HTML("<h2>Processes conditions:</h2>"))

    # print("Conditions of the processes:")
    if self.dt_processes_started_at:
        timedelta = datetime.datetime.now() - self.dt_processes_started_at
        print("Working for:", timedelta_nice_format(timedelta))
    self.print_table_with_conditions(int_max_processes_to_show=20)
    print(
        "ALIVE PROCESSES: ",
        len(self.dict_alive_processes_by_id), "/",
        len(self.dict_all_processes_by_id))
def print_table_with_conditions(self, int_max_processes_to_show=20)
Expand source code
@char
def print_table_with_conditions(self, int_max_processes_to_show=20):
    """"""
    list_list_processes_info = []
    list_headers = ["Process Id", "Status", "Runtime"]
    for process_num in list(self.dict_all_processes_by_id):
        list_process_info = []
        process_obj = self.dict_all_processes_by_id[process_num]
        # Delete if process is not alive
        if process_num in list(self.dict_alive_processes_by_id):
            if not process_obj.is_alive():
                self.dict_alive_processes_by_id.pop(process_num, None)
        # "Process Id"
        list_process_info.append(process_num)
        # "Status"
        list_process_info.append(process_obj.str_status)
        # "Runtime"
        str_runtime = process_obj.get_how_long_this_process_is_running()
        list_process_info.append(str_runtime)
        list_list_processes_info.append(list_process_info)
    if len(list_list_processes_info) > int_max_processes_to_show:
        list_list_additional_columns = []
        list_list_additional_columns.append(["---", "---", "---"])
        str_num = str(
            len(list_list_processes_info) - int_max_processes_to_show)
        list_list_additional_columns.append(
            ["Processes", "Hidden", str_num])
        list_list_additional_columns.append(["---", "---", "---"])
        list_list_processes_info = \
            list_list_additional_columns + list_list_processes_info[:20]
    # github  psql  orgtbl  pretty
    print(tabulate(
        list_list_processes_info, headers=list_headers, tablefmt="pretty"))
def show_jupyter_widget(self, int_seconds_step=10, int_max_processes_to_show=20)
Expand source code
@char
def show_jupyter_widget(
        self,
        int_seconds_step=10,
        int_max_processes_to_show=20
):
    """"""
    clear_output(wait=True)
    APP_GUI = widget.create_jupyter_widget(self)
    display(APP_GUI)
    with widget.OUTPUT_PROCESSES_CONDITIONS:
        self.print_info_about_running_processes(
            int_max_processes_to_show=int_max_processes_to_show)
    try:
        while True:
            if not self.dict_alive_processes_by_id:
                break
            for i in range(int_seconds_step):
                sleep(1)
                get_ipython().kernel.do_one_iteration()
            widget.OUTPUT_PROCESSES_CONDITIONS.clear_output(wait=True)
            with widget.OUTPUT_PROCESSES_CONDITIONS:
                self.print_info_about_running_processes(
                    int_max_processes_to_show=int_max_processes_to_show)
    except KeyboardInterrupt:
        clear_output(wait=True)
        print("Interrupting running processes")
        self.terminate_all_alive_processes()
        print("---> Done")
    print("All processes were finished")
def terminate_all_alive_processes(self)
Expand source code
@char
def terminate_all_alive_processes(self):
    """"""
    print("Terminating all alive processes")
    for process_num in list(self.dict_alive_processes_by_id):
        print("---> Terminate process: ", process_num + 1)
        process_obj = self.dict_alive_processes_by_id[process_num]
        process_obj.terminate()
        if not process_obj.is_alive():
            self.dict_alive_processes_by_id.pop(process_num, None)
            print("------> Done")
def wait_till_all_processes_are_over(self, int_seconds_step=10, int_max_processes_to_show=20)
Expand source code
@char
def wait_till_all_processes_are_over(
        self,
        int_seconds_step=10,
        int_max_processes_to_show=20
):
    """"""
    clear_output(wait=True)
    self.print_info_about_running_processes(
        int_max_processes_to_show=int_max_processes_to_show)
    try:
        while True:
            if not self.dict_alive_processes_by_id:
                break
            with yaspin() as sp:
                for i in range(int_seconds_step):
                    sp.text = "Updating in {} seconds".format(
                        int_seconds_step - i)
                    sleep(1)
            clear_output(wait=True)
            self.print_info_about_running_processes(
                int_max_processes_to_show=int_max_processes_to_show)
    except KeyboardInterrupt:
        clear_output(wait=True)
        print("Interrupting running processes")
        self.terminate_all_alive_processes()
        print("---> Done")
    print("All processes were finished")