coolneng
/
igdb
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Only parse and export data if needed

This commit is contained in:
coolneng 2020-01-08 22:14:16 +01:00
parent f84d71616a
commit 16a789cdf9
Signed by: coolneng
GPG Key ID: 9893DA236405AF57
2 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
code/database/export.py
View File

@ -25,6 +25,8 @@ def create_dataframes() -> DataFrame:
def insert_data(df_list, conn): def insert_data(df_list, conn):
if conn.table_names() is not None:
return
for key, value in df_list.items(): for key, value in df_list.items():
value.to_sql(key, con=conn, index=False, if_exists="append") value.to_sql(key, con=conn, index=False, if_exists="append")

5
code/database/parser.py
View File

@ -2,6 +2,7 @@ from iso3166 import countries as co
from pandas import DataFrame, concat, read_csv from pandas import DataFrame, concat, read_csv
from csv import QUOTE_NONNUMERIC from csv import QUOTE_NONNUMERIC
from database.constants import ADMIN_PW from database.constants import ADMIN_PW
from os import path
def country_conversion(political_unit) -> str: def country_conversion(political_unit) -> str:
@ -51,12 +52,12 @@ def rename_fields(df_list):
def create_databases(df): def create_databases(df):
user = {"uid": [7843], "username": ["admin"], "password": [ADMIN_PW]}
files = { files = {
"glacier": "../data/glacier.csv", "glacier": "../data/glacier.csv",
"annual_data": "../data/annual_data.csv", "annual_data": "../data/annual_data.csv",
"user": "../data/user.csv", "user": "../data/user.csv",
} }
user = {"uid": [7843], "username": ["admin"], "password": [ADMIN_PW]}
dataframes = { dataframes = {
"glacier": df[["POLITICAL_UNIT", "NAME", "WGMS_ID"]].drop_duplicates(), "glacier": df[["POLITICAL_UNIT", "NAME", "WGMS_ID"]].drop_duplicates(),
"annual_data": df[["WGMS_ID", "YEAR", "AREA", "MEDIAN_ELEVATION", "LENGTH"]], "annual_data": df[["WGMS_ID", "YEAR", "AREA", "MEDIAN_ELEVATION", "LENGTH"]],
@ -64,6 +65,8 @@ def create_databases(df):
} }
renamed_dfs = rename_fields(dataframes) renamed_dfs = rename_fields(dataframes)
for key, val in renamed_dfs.items(): for key, val in renamed_dfs.items():
if path.isfile(files[key]):
continue
val.to_csv(files[key], index=False, quoting=QUOTE_NONNUMERIC) val.to_csv(files[key], index=False, quoting=QUOTE_NONNUMERIC)