changes.patch

Only in code/data/ratschlab_preproc/: changes.patch
diff -ur ../dpsom/eicu_preproc/classes/dynamic_endpoints.py code/data/ratschlab_preproc/classes/dynamic_endpoints.py
--- ../dpsom/eicu_preproc/classes/dynamic_endpoints.py	2025-08-28 12:08:16.218009982 +0200
+++ code/data/ratschlab_preproc/classes/dynamic_endpoints.py	2025-08-28 14:53:08.243829721 +0200
@@ -5,7 +5,7 @@
 import numpy as np
 import pandas as pd
 
-import functions.util_array as mlhc_array
+from ..functions import util_array as mlhc_array
 
 class DynamicEndpointExtractor():
 
diff -ur ../dpsom/eicu_preproc/classes/imputer.py code/data/ratschlab_preproc/classes/imputer.py
--- ../dpsom/eicu_preproc/classes/imputer.py	2025-08-28 12:08:16.218155477 +0200
+++ code/data/ratschlab_preproc/classes/imputer.py	2025-09-10 13:36:33.881303466 +0200
@@ -2,12 +2,11 @@
 Imputation on eICU
 """
 
-import json
 import numpy as np
 import pandas as pd
 
-import functions.util_impute as eicu_impute
-import functions.util_array as mlhc_array
+from ..functions import util_impute as eicu_impute
+from ..functions import util_array as mlhc_array
 
 class Timegridder():
     ''' Function transforming the input table from the eICU tables into imputed values'''
@@ -15,13 +14,14 @@
     def __init__(self, timegrid_step_mins=60.0):
 
         # List of selected vitalPeriodic variables
-        self.sel_vs_vars = ["temperature", "sao2", "heartrate", "respiration", "cvp", "etco2", "systemicsystolic",
-                            "systemicdiastolic", "systemicmean", "pasystolic", "padiastolic", "pamean",
-                            "st1", "st2", "st3"]
+        self.sel_vs_vars = ["cvp", "heartrate", "respiration", "sao2", "st1", "st2", "st3", "systemicdiastolic", "systemicmean", "systemicsystolic", "temperature",
+            # "etco2", "pasystolic", "padiastolic", "pamean", # Removed to get the 98 Features used in the paper
+        ]
 
         # List of selected vitalAperiodic variables
-        self.sel_avs_vars = ["noninvasivesystolic", "noninvasivediastolic", "noninvasivemean", "paop", "cardiacoutput",
-                             "cardiacinput", "svr", "svri", "pvr", "pvri"]
+        self.sel_avs_vars = ["noninvasivediastolic", "noninvasivemean", "noninvasivesystolic", 
+            # "paop", "cardiacoutput", "cardiacinput", "svr", "svri", "pvr", "pvri" # Removed to get the 98 Features used in the paper
+        ]
 
         # Time grid interval length in minutes
         self.timegrid_step_mins = timegrid_step_mins
Only in code/data/ratschlab_preproc/classes: __init__.py
Only in code/data/ratschlab_preproc/classes: __pycache__
diff -ur ../dpsom/eicu_preproc/compute_patient_batches.py code/data/ratschlab_preproc/compute_patient_batches.py
--- ../dpsom/eicu_preproc/compute_patient_batches.py	2025-08-28 12:08:16.218155477 +0200
+++ code/data/ratschlab_preproc/compute_patient_batches.py	2025-08-28 14:50:03.442459980 +0200
@@ -7,7 +7,7 @@
 import os.path
 import argparse
 
-import functions.util_io as mlhc_io
+from .functions import util_io as mlhc_io
 
 def compute_patient_batches(configs):
     pid_list = mlhc_io.read_list_from_file(configs["included_pid_path"])
diff -ur ../dpsom/eicu_preproc/compute_quantiles.py code/data/ratschlab_preproc/compute_quantiles.py
--- ../dpsom/eicu_preproc/compute_quantiles.py	2025-08-28 12:08:16.218155477 +0200
+++ code/data/ratschlab_preproc/compute_quantiles.py	2025-10-22 14:03:49.382591479 +0200
@@ -6,17 +6,18 @@
 import pandas as pd
 import numpy as np
 
-import functions.util_io as mlhc_io
+from .functions import util_io as mlhc_io
 
 def save_variable_quantiles(configs):
     ''' Saves the quantiles of all variables in the LAB/VITAL SIGN tables'''
-    all_pids=mlhc_io.read_list_from_file(configs["included_pid_path"])
+    all_pids=[int(pid) for pid in mlhc_io.read_list_from_file(configs["included_pid_path"])]
     vital_per_variables=mlhc_io.read_list_from_file(configs["list_per_variables"])
     vital_aper_variables=mlhc_io.read_list_from_file(configs["list_aper_variables"])
     var_quantiles={}
 
     print("Lab table...")
     df_lab=pd.read_hdf(configs["lab_table_path"],mode='r')
+    print(df_lab.shape)
     df_lab=df_lab[df_lab.patientunitstayid.isin(all_pids)]
     print("Loaded lab table with {} rows".format(df_lab.shape[0]))
     all_lab_vars=df_lab.labname.unique()
@@ -27,28 +28,30 @@
         var_quantiles["lab_"+lab_var]=[]
 
         for quantile in np.arange(0.01,1.00,0.01):
-            quant_val = df_var.labresult.quantile(quantile)
+            quant_val = float(df_var.labresult.quantile(quantile))
             var_quantiles["lab_"+lab_var].append(quant_val)
 
         print("List length: {}".format(len(var_quantiles["lab_"+lab_var])))
         
+    del df_lab
     gc.collect()
     print("Vital periodic table...")
-    df_vital_per=pd.read_hdf(configs["vital_per_path"],mode='r')
-    df_vital_per=df_vital_per[df_vital_per.patientunitstayid.isin(all_pids)]
-    print("Loaded vital periodic table with {} rows".format(df_vital_per.shape[0]))
-
     for per_var in vital_per_variables:
         print("Periodic variable: {}".format(per_var))
+        df_vital_per = pd.read_hdf(configs["vital_per_path"], mode='r', columns=[per_var, 'patientunitstayid'])
+        df_vital_per = df_vital_per[df_vital_per.patientunitstayid.isin(all_pids)]
+        print("Loaded with {} rows".format(df_vital_per.shape[0]))
+
         df_col=df_vital_per[per_var]
         var_quantiles["periodic_"+per_var]=[]
 
         for quantile in np.arange(0.01,1.00,0.01):
-            quant_val=df_col.quantile(quantile)
+            quant_val=float(df_col.quantile(quantile))
             var_quantiles["periodic_"+per_var].append(quant_val)
                 
         print("List length: {}".format(len(var_quantiles["periodic_"+per_var])))
 
+    del df_vital_per
     gc.collect()
     print("Vital aperiodic table...")
     df_vital_aper=pd.read_hdf(configs["vital_aper_path"],mode='r')
@@ -61,11 +64,12 @@
         var_quantiles["aperiodic_"+aper_var]=[]
         
         for quantile in np.arange(0.01,1.00,0.01):
-            quant_val=df_col.quantile(quantile)
+            quant_val=float(df_col.quantile(quantile))
             var_quantiles["aperiodic_"+aper_var].append(quant_val)
                         
         print("List length: {}".format(len(var_quantiles["aperiodic_"+aper_var])))
 
+    del df_vital_aper
     gc.collect()
     quantile_fp=open(configs["quantile_path"],mode='w')
     json.dump(var_quantiles,quantile_fp)
diff -ur ../dpsom/eicu_preproc/filter_patients.py code/data/ratschlab_preproc/filter_patients.py
--- ../dpsom/eicu_preproc/filter_patients.py	2025-08-28 12:08:16.218155477 +0200
+++ code/data/ratschlab_preproc/filter_patients.py	2025-08-28 14:50:03.442459980 +0200
@@ -2,13 +2,12 @@
 Generates the list of includes patient stays.
 """
 
-import ipdb
 import argparse
 
 import numpy as np
 import pandas as pd
 
-import functions.util_io as mlhc_io
+from .functions import util_io as mlhc_io
 
 def filter_patients(configs):
     pid_list = mlhc_io.read_list_from_file(configs["all_pid_stay_path"])
diff -ur ../dpsom/eicu_preproc/filter_variables.py code/data/ratschlab_preproc/filter_variables.py
--- ../dpsom/eicu_preproc/filter_variables.py	2025-08-28 12:08:16.218155477 +0200
+++ code/data/ratschlab_preproc/filter_variables.py	2025-08-28 14:50:03.455460076 +0200
@@ -4,7 +4,6 @@
 
 import os
 import os.path
-import datetime
 import gc
 import argparse
 
@@ -15,7 +14,7 @@
 matplotlib.use("pdf")
 import matplotlib.pyplot as plt
 
-import functions.util_io as mlhc_io
+from .functions import util_io as mlhc_io
 
 def filter_variables(configs):
 
Only in code/data/ratschlab_preproc/functions: __pycache__
diff -ur ../dpsom/eicu_preproc/hdf_convert.py code/data/ratschlab_preproc/hdf_convert.py
--- ../dpsom/eicu_preproc/hdf_convert.py	2025-08-28 12:08:16.218844935 +0200
+++ code/data/ratschlab_preproc/hdf_convert.py	2025-08-28 13:26:41.485311109 +0200
@@ -7,10 +7,7 @@
 import csv
 import os.path
 import pandas as pd
-import numpy as np
 import glob as glob
-import ipdb
-import gc
 
 TABLES=["admissionDrug", "admissionDx", "allergy", "apacheApsVar", "apachePatientResult", "apachePredVar", "carePlanCareProvider", 
         "carePlanEOL", "carePlanGeneral" , "carePlanGoal", "carePlanInfectiousDisease", "customLab", "diagnosis", "hospital", 
Only in code/data/ratschlab_preproc/: __init__.py
diff -ur ../dpsom/eicu_preproc/label_all_patients.py code/data/ratschlab_preproc/label_all_patients.py
--- ../dpsom/eicu_preproc/label_all_patients.py	2025-08-28 12:08:16.218844935 +0200
+++ code/data/ratschlab_preproc/label_all_patients.py	2025-08-28 14:50:59.848879490 +0200
@@ -9,7 +9,7 @@
 import os
 import sys
 
-import functions.util_filesystem as mlhc_fs
+from .functions import util_filesystem as mlhc_fs
 
 def label_all_patients(configs):
     job_index=0
diff -ur ../dpsom/eicu_preproc/label_data_one_batch.py code/data/ratschlab_preproc/label_data_one_batch.py
--- ../dpsom/eicu_preproc/label_data_one_batch.py	2025-08-28 12:08:16.218844935 +0200
+++ code/data/ratschlab_preproc/label_data_one_batch.py	2025-08-28 14:52:17.938458127 +0200
@@ -10,7 +10,7 @@
 
 import pandas as pd
 
-import classes.dynamic_endpoints as eicu_dynamic_tf
+from .classes import dynamic_endpoints as eicu_dynamic_tf
 
 def label_data_one_batch(configs):
     first_write = True
Only in code/data/ratschlab_preproc/: __pycache__
diff -ur ../dpsom/eicu_preproc/README.txt code/data/ratschlab_preproc/README.txt
--- ../dpsom/eicu_preproc/README.txt	2025-08-28 12:08:16.216941714 +0200
+++ code/data/ratschlab_preproc/README.txt	2025-08-29 14:17:44.379422210 +0200
@@ -1,2 +1,7 @@
 Computational pipeline used to reproduce the eICU pre-processed data
-used in the DPSOM project
+used in the DPSOM project.
+
+Source: https://github.com/ratschlab/dpsom/tree/master/eicu_preproc
+
+All changes made to the code where only done to get it working in this environment or do reduce memory footprint and improve speed. The changes can be seen in changes.patch (where ../dpsom is the original repository)
+
diff -ur ../dpsom/eicu_preproc/save_all_pids.py code/data/ratschlab_preproc/save_all_pids.py
--- ../dpsom/eicu_preproc/save_all_pids.py	2025-08-28 12:08:16.218844935 +0200
+++ code/data/ratschlab_preproc/save_all_pids.py	2025-08-29 13:06:43.435958318 +0200
@@ -5,7 +5,7 @@
 import pandas as pd
 import argparse
 
-import functions.util_io as mlhc_io
+from .functions import util_io as mlhc_io
 
 def save_all_pids(configs):
     
@@ -14,20 +14,20 @@
     stay_ids = list(df_patient["patientunitstayid"].unique())
     mlhc_io.write_list_to_file(configs["pid_stay_list"], stay_ids)
 
-    print("Async Vital Signs")
-    df_async = pd.read_hdf(configs["vital_aper_path"], configs["generic_dset_id"], mode='r')
-    for vs in configs["ASYNC_VITALS"]:
-        orig = df_async[vs]
-        finite = orig.dropna()
-        print("{}: Number of entries: {}".format(vs, finite.size))
-
-    print("Sync Vital Signs")
-    df_sync = pd.read_hdf(configs["vital_per_path"], configs["generic_dset_id"], mode='r')
-
-    for vs in configs["SYNC_VITALS"]:
-        orig = df_sync[vs]
-        finite = orig.dropna()
-        print("{}: Number of entries: {}".format(vs, finite.size))
+    # print("Async Vital Signs")
+    # df_async = pd.read_hdf(configs["vital_aper_path"], configs["generic_dset_id"], mode='r')
+    # for vs in configs["ASYNC_VITALS"]:
+    #     orig = df_async[vs]
+    #     finite = orig.dropna()
+    #     print("{}: Number of entries: {}".format(vs, finite.size))
+
+    # print("Sync Vital Signs")
+    # df_sync = pd.read_hdf(configs["vital_per_path"], configs["generic_dset_id"], mode='r')
+
+    # for vs in configs["SYNC_VITALS"]:
+    #     orig = df_sync[vs]
+    #     finite = orig.dropna()
+    #     print("{}: Number of entries: {}".format(vs, finite.size))
 
 
 if __name__ == "__main__":
diff -ur ../dpsom/eicu_preproc/save_model_inputs.py code/data/ratschlab_preproc/save_model_inputs.py
--- ../dpsom/eicu_preproc/save_model_inputs.py	2025-08-28 12:08:16.218844935 +0200
+++ code/data/ratschlab_preproc/save_model_inputs.py	2025-09-11 14:21:13.933804978 +0200
@@ -63,27 +63,28 @@
 
     return time_series_all, time_series_endpoint_all
 
-# *******************************************************************************************************************
+if __name__ == "__main__":
+    # *******************************************************************************************************************
 
-# path of the preprocessed data
-data_total = glob("../data/time_grid/batch_*.h5")
+    # path of the preprocessed data
+    data_total = glob("../data/time_grid/batch_*.h5")
 
-# path of the labels of the preprocessed data
-endpoints_total = glob("../data/labels/batch_*.h5")
+    # path of the labels of the preprocessed data
+    endpoints_total = glob("../data/labels/batch_*.h5")
 
-# path of the labels of the mins
-mins_dynamic = pd.read_hdf("../data/time_grid/normalization_values.h5","mins_dynamic")
-
-# path of the labels of the scales
-scales_dynamic = pd.read_hdf("../data/time_grid/normalization_values.h5", "scales_dynamic")
-
-# *******************************************************************************************************************
-
-# Create numpy arrays with the last 72 time-steps of each time-series.
-data, labels = parmap_batch_generator(data_total, endpoints_total, mins_dynamic, scales_dynamic, max_n_step=72)
-l = np.array(labels)
-d = np.array(data)
-hf = h5py.File('../data/eICU_data.csv', 'w')
-hf.create_dataset('x', data=d)
-hf.create_dataset('y', data=l)
-hf.close()
+    # path of the labels of the mins
+    mins_dynamic = pd.read_hdf("../data/time_grid/normalization_values.h5","mins_dynamic")
+
+    # path of the labels of the scales
+    scales_dynamic = pd.read_hdf("../data/time_grid/normalization_values.h5", "scales_dynamic")
+
+    # *******************************************************************************************************************
+
+    # Create numpy arrays with the last 72 time-steps of each time-series.
+    data, labels = parmap_batch_generator(data_total, endpoints_total, mins_dynamic, scales_dynamic, max_n_step=72)
+    l = np.array(labels)
+    d = np.array(data)
+    hf = h5py.File('../data/eICU_data.csv', 'w')
+    hf.create_dataset('x', data=d)
+    hf.create_dataset('y', data=l)
+    hf.close()
diff -ur ../dpsom/eicu_preproc/timegrid_all_patients.py code/data/ratschlab_preproc/timegrid_all_patients.py
--- ../dpsom/eicu_preproc/timegrid_all_patients.py	2025-08-28 12:08:16.218844935 +0200
+++ code/data/ratschlab_preproc/timegrid_all_patients.py	2025-08-28 14:50:59.850879505 +0200
@@ -9,7 +9,7 @@
 import os
 import sys
 
-import functions.util_filesystem as mlhc_fs
+from .functions import util_filesystem as mlhc_fs
 
 def timegrid_all_patients(configs):
 
diff -ur ../dpsom/eicu_preproc/timegrid_one_batch.py code/data/ratschlab_preproc/timegrid_one_batch.py
--- ../dpsom/eicu_preproc/timegrid_one_batch.py	2025-08-28 12:08:16.218844935 +0200
+++ code/data/ratschlab_preproc/timegrid_one_batch.py	2025-09-10 17:13:54.776479990 +0200
@@ -6,9 +6,6 @@
 import os
 import os.path
 import csv
-import time
-import timeit
-import psutil
 import pickle
 import sys
 import json
@@ -18,10 +15,9 @@
 import matplotlib as mpl
 mpl.use("PDF")
 
-import classes.imputer as eicu_tf_impute
-import classes.static_extractor as eicu_static_tf
+from .classes import imputer as eicu_tf_impute
+from .classes import static_extractor as eicu_static_tf
 
-import functions.util_io as mlhc_io
 
 def timegrid_one_batch(configs):
     batch_id=configs["batch_id"]    
@@ -29,7 +25,6 @@
     with open(configs["pid_batch_file"], 'rb') as fp:
         obj = pickle.load(fp)
         batch_to_lst = obj["batch_to_lst"]
-        batches = list(sorted(batch_to_lst.keys()))
         batch_idxs = batch_to_lst[batch_id]    
     
     first_write = True
@@ -56,7 +51,6 @@
 
             with open(configs["selected_lab_vars"], 'r') as fp:
                 csv_fp = csv.reader(fp, delimiter='\t')
-                next(csv_fp)
                 for lab_name in csv_fp:
                     lab_vars.append(lab_name[0].strip())