[1]+ nohup singularity exec --nv ../images/uno-tensorflow:2.8.2-gpu-20220624.sif train.sh 0 $CANDLE_DATA_DIR /homes/brettin/Singularity/workspace/configs/uno_auc_model.txt &
(base) brettin@lambda7:~/Singularity/workspace/top21_uno$ tail -f nohup.out
'timeout': -1,
'train_bool': True,
'train_sources': ['CCLE', 'GDSC', 'CTRP', 'NCI60'],
'use_exported_data': 'top21_uno_v2.h5',
'use_filtered_genes': False,
'use_landmark_genes': True,
'val_split': 0.2,
'verbose': False,
'warmup_lr': True}
Params: {'train_sources': ['CCLE', 'GDSC', 'CTRP', 'NCI60'], 'use_exported_data': 'top21_uno_v2.h5', 'shuffle': True, 'test_sources': ['train'], 'cell_types': None, 'cell_features': ['rnaseq'], 'drug_features': ['descriptors'], 'dense': [1000, 1000, 1000, 1000, 1000], 'dense_feature_layers': [1000, 1000, 1000], 'activation': 'relu', 'loss': 'mse', 'optimizer': 'adamax', 'scaling': 'std', 'dropout': 0.1, 'epochs': 400, 'batch_size': 32, 'val_split': 0.2, 'cv': 1, 'max_val_loss': 1.0, 'learning_rate': 0.0001, 'base_lr': None, 'agg_dose': 'AUC', 'residual': False, 'reduce_lr': True, 'warmup_lr': True, 'batch_normalization': False, 'feature_subsample': 0, 'rng_seed': 2018, 'no_gen': False, 'verbose': False, 'preprocess_rnaseq': 'source_scale', 'gpus': [0], 'use_landmark_genes': True, 'no_feature_source': True, 'no_response_source': True, 'cp': True, 'save_path': 'save/uno', 'output_dir': '/homes/brettin/Singularity/workspace/top21_uno/output/uno/EXP000/RUN000', 'single': True, 'on_memory_loader': True, 'timeout': -1, 'train_bool': True, 'profiling': False, 'experiment_id': 'EXP000', 'run_id': 'RUN000', 'logfile': None, 'ckpt_restart_mode': 'auto', 'ckpt_checksum': False, 'ckpt_skip_epochs': 0, 'ckpt_directory': './save', 'ckpt_save_best': True, 'ckpt_save_best_metric': 'val_loss', 'ckpt_save_weights_only': False, 'ckpt_save_interval': 0, 'ckpt_keep_mode': 'linear', 'ckpt_keep_limit': 1000000, 'by_cell': None, 'by_drug': None, 'cell_subset_path': '', 'drug_subset_path': '', 'drug_median_response_min': -1, 'drug_median_response_max': 1, 'dense_cell_feature_layers': None, 'dense_drug_feature_layers': None, 'use_filtered_genes': False, 'feature_subset_path': '', 'cell_feature_subset_path': '', 'drug_feature_subset_path': '', 'es': False, 'tb': False, 'tb_prefix': 'tb', 'partition_by': None, 'cache': None, 'export_csv': None, 'export_data': None, 'growth_bins': 0, 'initial_weights': None, 'save_weights': None, 'config_file': '/homes/brettin/Singularity/workspace/configs/uno_auc_model.txt', 'data_type': <class 'numpy.float32'>}
/usr/local/Benchmarks/Pilot1/Uno/uno_baseline_keras2.py:14: DeprecationWarning: Please use pearsonr
from the scipy.stats
namespace, the scipy.stats.stats
namespace is deprecated.
from scipy.stats.stats import pearsonr
WARNING:tensorflow:From /usr/local/Benchmarks/Pilot1/Uno/uno_baseline_keras2.py:269: The name tf.keras.backend.set_session is deprecated. Please use tf.compat.v1.keras.backend.set_session instead.
Traceback (most recent call last):
File "/usr/local/Benchmarks/Pilot1/Uno/uno_baseline_keras2.py", line 676, in
main()
File "/usr/local/Benchmarks/Pilot1/Uno/uno_baseline_keras2.py", line 672, in main
run(params)
File "/usr/local/Benchmarks/Pilot1/Uno/uno_baseline_keras2.py", line 272, in run
loader.load(
File "/usr/local/Benchmarks/Pilot1/Uno/uno_data.py", line 1142, in load
with pd.HDFStore(use_exported_data, "r") as store:
File "/usr/local/lib/python3.8/dist-packages/pandas/io/pytables.py", line 591, in init
self.open(mode=mode, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/pandas/io/pytables.py", line 740, in open
self._handle = tables.open_file(self._path, self._mode, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/tables/file.py", line 300, in open_file
return File(filename, mode, title, root_uep, filters, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/tables/file.py", line 750, in init
self._g_new(filename, mode, **params)
File "tables/hdf5extension.pyx", line 368, in tables.hdf5extension.File._g_new
File "/usr/local/lib/python3.8/dist-packages/tables/utils.py", line 143, in check_file_access
raise OSError(f"{path}
does not exist")
OSError: /homes/brettin/Singularity/workspace/top21_uno/top21_uno_v2.h5
does not exist
(base) brettin@lambda7:/Singularity/workspace/top21_uno$ echo $CANDLE_DATA_DIR
/homes/brettin/Singularity/workspace/data_dir
(base) brettin@lambda7:/Singularity/workspace/top21_uno$ ls $CANDLE_DATA_DIR
Pilot1 top21_uno_v2.h5 uno_input_data.h5