Source code for WORC.facade.helpers.configbuilder

from WORC.detectors.detectors import BigrClusterDetector, CartesiusClusterDetector, DebugDetector
import configparser
import fastr
import collections.abc


def _deep_update(d, u):
    for k, v in u.items():
        if isinstance(v, collections.abc.Mapping):
            d[k] = _deep_update(d.get(k, {}), v)
        else:
            d[k] = v
    return d


[docs]class ConfigBuilder():
[docs] def __init__(self): # initalize the main config object and the custom overrids self._config = configparser.ConfigParser() self._custom_overrides = {} # Detect when using a cluster and override relevant config fields self._cluster_config_overrides()
[docs] def build_config(self, defaultconfig): defaultconfig.read_dict({**self._config}) defaultconfig.read_dict({**self._custom_overrides}) defaultconfig.read_dict({**self._debug_config_overrides()}) self._config = defaultconfig return defaultconfig
[docs] def custom_config_overrides(self, config): _deep_update(self._custom_overrides, config)
def _cluster_config_overrides(self): if BigrClusterDetector().do_detection(): overrides = { 'General': {'Joblib_ncores': '1', 'Joblib_backend': 'threading'}, 'Classification': {'fastr': 'True', 'fastr_plugin': 'DRMAAExecution'}, 'HyperOptimization': {'n_jobspercore': '4000'} } elif CartesiusClusterDetector().do_detection(): overrides = { 'Classification': {'fastr': 'True', 'fastr_plugin': 'ProcessPoolExecution'}, 'HyperOptimization': {'n_jobspercore': '4000'} } else: overrides = {} # not a cluster or unsupported self.custom_config_overrides(overrides) return overrides
[docs] def estimator_scoring_overrides(self, estimators, scoring_method): overrides = { 'Classification': {'classifiers': ', '.join(estimators)}, 'HyperOptimization': {'scoring_method': scoring_method} } self.custom_config_overrides(overrides) return overrides
[docs] def coarse_overrides(self): overrides = { 'ImageFeatures': { 'texture_Gabor': 'False', 'vessel': 'False', 'log': 'False', 'phase': 'False', }, 'SelectFeatGroup': { 'texture_Gabor_features': 'False', 'log_features': 'False', 'vessel_features': 'False', 'phase_features': 'False', }, 'CrossValidation': {'N_iterations': '3'}, 'HyperOptimization': {'n_splits': '2', 'N_iterations': '1000', 'n_jobspercore': '500'}, 'Ensemble': {'Use': '1'} } self.custom_config_overrides(overrides) return overrides
[docs] def full_overrides(self): overrides = { # Compute all available features 'ImageFeatures': { 'texture_Gabor': 'True', 'vessel': 'True', 'log': 'True', 'phase': 'True', }, # Also take these features into account in the feature groupwise selection 'SelectFeatGroup': { 'texture_Gabor_features': 'True, False', 'log_features': 'True, False', 'vessel_features': 'True, False', 'phase_features': 'True, False', }, # Use some more feature selection methods 'Featsel': { 'UsePCA': '0.25', 'StatisticalTestUse': '0.25', 'ReliefUse': '0.25' }, # Extensive cross-validation and hyperoptimization 'CrossValidation': {'N_iterations': '100'}, 'HyperOptimization': {'N_iterations': '100000', 'n_jobspercore': '4000'}, # Make use of ensembling 'Ensemble': {'Use': '50'} } self.custom_config_overrides(overrides) return overrides
[docs] def fullprint(self): ''' Print the full contents of the config to the console. ''' for k, v in self._config.items(): print(f"{k}:") for k2, v2 in v.items(): print(f"\t {k2}: {v2}") print("\n")
def _debug_config_overrides(self): if DebugDetector().do_detection(): overrides = { 'ImageFeatures': { 'texture_Gabor': 'False', 'vessel': 'False', 'log': 'False', 'phase': 'False', 'texture_LBP': 'False', 'texture_GLCMMS': 'False', 'texture_GLRLM': 'False', 'texture_NGTDM': 'False', }, 'PyRadiomics': { 'Wavelet': 'False', 'LoG': 'False' }, 'SelectFeatGroup': { 'texture_Gabor_features': 'False', 'log_features': 'False', 'vessel_features': 'False', 'phase_features': 'False', }, 'CrossValidation': {'N_iterations': '2', 'fixed_seed': ' True'}, 'HyperOptimization': {'N_iterations': '10', 'n_jobspercore': '10', 'n_splits': '2'}, 'Ensemble': {'Use': '1'}, 'SampleProcessing': {'SMOTE': 'False'}, } # Additionally, turn queue reporting system on fastr.config.queue_report_interval = 120 else: overrides = {} # not a cluster or unsupported return overrides