From 214b7093f0d4f4e1c8adbb38155ea6e6917e02e6 Mon Sep 17 00:00:00 2001 From: coolneng Date: Thu, 31 Dec 2020 01:56:54 +0100 Subject: [PATCH] Add basic preprocessing functions --- src/preprocessing.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 src/preprocessing.py diff --git a/src/preprocessing.py b/src/preprocessing.py new file mode 100644 index 0000000..b40305b --- /dev/null +++ b/src/preprocessing.py @@ -0,0 +1,29 @@ +from pandas import DataFrame, read_csv +from sklearn.preprocessing import normalize +from sklearn.model_selection import KFold + + +def replace_values(df) -> DataFrame: + for column in df.columns: + df[column].fillna(value=df[column].mean(), inplace=True) + return df + + +def remove_letters(df) -> DataFrame: + columns = ["Consumo", "Motor_CC", "Potencia"] + processed_df = df.copy() + for col in columns: + processed_df[col] = df[col].str.replace(pat="[^.0-9]", repl="").astype(float) + return processed_df + + +def split_k_sets(df): + k_fold = KFold(shuffle=True, random_state=42) + return k_fold.split(df) + + +def parse_data(source) -> DataFrame: + df = read_csv(filepath_or_buffer=source, na_values="?") + processed_df = df.dropna() + numeric_df = remove_letters(processed_df) + return numeric_df