chavalitros
/
IN-P3
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Remove string trimming function

This commit is contained in:
coolneng 2021-01-01 21:54:05 +01:00
parent abfc877c7d
commit e05ccdabb9
Signed by: coolneng
GPG Key ID: 9893DA236405AF57
1 changed files with 21 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
src/preprocessing.py
View File

@ -21,38 +21,32 @@ def rename_columns(df_list) -> DataFrame:
def drop_null_values(df_list): def drop_null_values(df_list):
for df in df_list: for df in df_list:
df.dropna(inplace=True)
df.drop(columns="tipo_marchas", inplace=True) df.drop(columns="tipo_marchas", inplace=True)
df["descuento"].fillna(0) df.drop(columns="descuento", inplace=True)
return df_list df.dropna(inplace=True)
def trim_column_names(df_list) -> DataFrame:
columns = ["consumo", "motor_CC", "potencia"]
for df in df_list:
for col in columns:
df[col] = df[col].str.replace(pat="[^.0-9]", repl="").astype(float)
return df_list return df_list
def encode_columns(df_list): def encode_columns(df_list):
label_encoder = LabelEncoder() label_encoder = LabelEncoder()
files = [ files = [
"ao" "ao",
"asientos" "asientos",
"ciudad" "ciudad",
"combustible" "combustible",
"consumo" "consumo",
"descuento" "kilometros",
"kilometros" "mano",
"mano" "motor_cc",
"motor_cc" "nombre",
"nombre" "potencia",
"potencia"
] ]
for data in files: for data in files:
for df in df_list: for df in df_list:
label = label_encoder.fit(read_csv("data/" + data + ".csv", squeeze=True)) label = label_encoder.fit(read_csv("data/" + data + ".csv", squeeze=True))
if data == "ao":
df["año"] = label.transform(df["año"])
else:
df[data] = label.transform(df[data]) df[data] = label.transform(df[data])
return df_list return df_list
@ -72,8 +66,10 @@ def balance_training_data(df):
smote_tomek = SMOTETomek(random_state=42) smote_tomek = SMOTETomek(random_state=42)
data, target = split_data_target(df=df, dataset="data") data, target = split_data_target(df=df, dataset="data")
balanced_data, balanced_target = smote_tomek.fit_resample(data, target) balanced_data, balanced_target = smote_tomek.fit_resample(data, target)
balanced_data_df = DataFrame(balanced_data, columns=data.columns) balanced_data_df = DataFrame(
balanced_target_df = DataFrame(balanced_target, columns=target.columns) balanced_data, columns=df.columns.difference(["precio_cat"])
)
balanced_target_df = DataFrame(balanced_target, columns=["precio_cat"])
return balanced_data_df, balanced_target_df return balanced_data_df, balanced_target_df
@ -86,8 +82,7 @@ def parse_data(train, test):
df_list = construct_dataframes(train=train, test=test) df_list = construct_dataframes(train=train, test=test)
renamed_df_list = rename_columns(df_list) renamed_df_list = rename_columns(df_list)
processed_df_list = drop_null_values(renamed_df_list) processed_df_list = drop_null_values(renamed_df_list)
numeric_df_list = trim_column_names(processed_df_list) encoded_df_list = encode_columns(processed_df_list)
encoded_df_list = encode_columns(numeric_df_list)
train_data, train_target = balance_training_data(encoded_df_list[0]) train_data, train_target = balance_training_data(encoded_df_list[0])
test_data, test_ids = split_data_target(encoded_df_list[1], dataset="test") test_data, test_ids = split_data_target(encoded_df_list[1], dataset="test")
return train_data, train_target, test_data, test_ids return train_data, train_target, test_data, test_ids