From 3b08cb73f5acae5b174ee043e4b12588c1f97ef3 Mon Sep 17 00:00:00 2001 From: coolneng Date: Fri, 1 Jan 2021 23:52:43 +0100 Subject: [PATCH] Complete missing values in columns with less data --- src/preprocessing.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/preprocessing.py b/src/preprocessing.py index 32709a4..273fa9e 100644 --- a/src/preprocessing.py +++ b/src/preprocessing.py @@ -19,9 +19,17 @@ def rename_columns(df_list) -> DataFrame: return df_list -def drop_null_values(df_list): +def process_null_values(df_list): drop_columns = ["tipo_marchas", "descuento", "ciudad"] + fill_columns = ["asientos", "motor_cc", "potencia"] for df in df_list: + for column in fill_columns: + if column == "asientos": + df[column].fillna(value=df[column].median(), inplace=True) + else: + df[column].fillna( + value=df[column].str.extract("(\d+)").mean(), inplace=True + ) df.drop(columns=drop_columns, inplace=True) df.dropna(inplace=True) return df_list @@ -80,7 +88,7 @@ def split_k_sets(df): def parse_data(train, test): df_list = construct_dataframes(train=train, test=test) renamed_df_list = rename_columns(df_list) - processed_df_list = drop_null_values(renamed_df_list) + processed_df_list = process_null_values(renamed_df_list) encoded_df_list = encode_columns(processed_df_list) train_data, train_target = balance_training_data(encoded_df_list[0]) test_data, test_ids = split_data_target(encoded_df_list[1], dataset="test")