From 7b945e2b981e1a565920f44dd35c084600d93b87 Mon Sep 17 00:00:00 2001 From: coolneng Date: Wed, 8 Jan 2020 01:23:00 +0100 Subject: [PATCH] Add SQL insertion functionality --- Design.org | 1 + code/database/db_setup.py | 25 ++++++++++--------------- code/database/parser.py | 21 +++++++++++++++++++-- code/database/storage.py | 18 ++++++++++++++---- data/annual_change.csv | 2 +- data/annual_data.csv | 2 +- data/glacier.csv | 2 +- data/users.csv | 2 +- docs/Project.org | 6 +++--- 9 files changed, 51 insertions(+), 28 deletions(-) diff --git a/Design.org b/Design.org index 70c2cd3..004765b 100644 --- a/Design.org +++ b/Design.org @@ -62,3 +62,4 @@ CLOSED: [2020-01-03 Fri 00:44] *** NEXT Documentation *** INACTIVE Frontend [0/1] [0%] - [ ] [[https://adminlte.io/][Adminlte]] +- [ ] Bootstrap diff --git a/code/database/db_setup.py b/code/database/db_setup.py index 5a47c46..69d0655 100644 --- a/code/database/db_setup.py +++ b/code/database/db_setup.py @@ -29,9 +29,11 @@ db = create_connection() class Glacier(db.Model): - uid = db.Column(db.Integer, primary_key=True) + uid = db.Column(db.String(5), primary_key=True) country = db.Column(db.String(60)) name = db.Column(db.String(60)) + annual_data = db.relationship("Annual_Data") + annual_change = db.relationship("Annual_Change") def __init__(self, uid, country, name): self.uid = uid @@ -40,37 +42,30 @@ class Glacier(db.Model): class Annual_Data(db.Model): - glacier = db.relationship("Glacier", backref=db.backref("posts", lazy=True)) - uid = db.Column( - db.Integer, db.ForeignKey("glacier.uid"), nullable=False, primary_key=True - ) + year = db.Column(db.Integer, primary_key=True) + uid = db.Column(db.String(5), db.ForeignKey("glacier.uid"), primary_key=True) surface = db.Column(db.Integer) length = db.Column(db.Integer) elevation = db.Column(db.Integer) - year = db.Column(db.Integer, primary_key=True) - def __init__(self, surface, length, elevation, year): + def __init__(self, year, surface, length, elevation): + self.year = year self.surface = surface self.length = length self.elevation = elevation - self.year = year class Annual_Change(db.Model): - glacier = db.relationship("Glacier", backref=db.backref("posts", lazy=True)) - uid = db.Column( - db.Integer, db.ForeignKey("glacier.uid"), nullable=False, primary_key=True - ) + year = db.Column(db.Integer, db.ForeignKey("annual__data.year"), primary_key=True) + uid = db.Column(db.String(5), db.ForeignKey("glacier.uid"), primary_key=True) surface_change = db.Column(db.Integer) length_change = db.Column(db.Integer) elevation_change = db.Column(db.Integer) - year = db.Column(db.Integer, primary_key=True) - def __init__(self, surface_change, length_change, elevation_change, year): + def __init__(self, surface_change, length_change, elevation_change): self.surface_change = surface_change self.length_change = length_change self.elevation_change = elevation_change - self.year = year class User(db.Model): diff --git a/code/database/parser.py b/code/database/parser.py index 6039e39..9415465 100644 --- a/code/database/parser.py +++ b/code/database/parser.py @@ -34,8 +34,24 @@ def select_columns() -> DataFrame: return data +def rename_fields(df_list): + new_df_list = {} + new_fields = { + "POLITICAL_UNIT": "country", + "NAME": "name", + "WGMS_ID": "uid", + "YEAR": "year", + "MEDIAN_ELEVATION": "elevation", + "AREA": "surface", + "LENGTH": "length", + } + for key, value in df_list.items(): + new_df_list[key] = value.rename(columns=new_fields) + return new_df_list + + def create_databases(df): - users = {"UID": [7843], "USERNAME": ["admin"], "PASSWORD": [ADMIN_PW]} + users = {"uid": [7843], "username": ["admin"], "password": [ADMIN_PW]} files = { "glacier": "../data/glacier.csv", "annual__data": "../data/annual_data.csv", @@ -48,7 +64,8 @@ def create_databases(df): "annual__change": df[["WGMS_ID", "YEAR"]], "users": DataFrame(users), } - for key, val in dataframes.items(): + renamed_dfs = rename_fields(dataframes) + for key, val in renamed_dfs.items(): val.to_csv(files[key], index=False, quoting=QUOTE_NONNUMERIC) diff --git a/code/database/storage.py b/code/database/storage.py index 105cece..019105b 100644 --- a/code/database/storage.py +++ b/code/database/storage.py @@ -1,5 +1,15 @@ -from db_setup import create_connection -from pandas import DataFrame, read_csv, to_sql +from constants import DB_NAME, DB_PW, DB_USER +from pandas import DataFrame, read_csv +from sqlalchemy import create_engine, engine + + +def create_connection() -> engine: + host = "localhost:3306" + connection_uri = "mysql+pymysql://{user}:{pw}@{url}/{db}".format( + user=DB_USER, pw=DB_PW, url=host, db=DB_NAME + ) + engine = create_engine(connection_uri) + return engine def create_dataframes() -> DataFrame: @@ -11,13 +21,13 @@ def create_dataframes() -> DataFrame: } df_list = {} for csv in files.keys(): - df_list[csv] = read_csv(files[csv], index=False) + df_list[csv] = read_csv(files[csv]) return df_list def insert_data(df_list, conn): for key, value in df_list.items(): - value.to_sql(key, con=conn, if_exists="replace", index=False) + value.to_sql(key, con=conn, index=False, if_exists="append", chunksize=1) def main(): diff --git a/data/annual_change.csv b/data/annual_change.csv index 8488d8f..49c5c07 100644 --- a/data/annual_change.csv +++ b/data/annual_change.csv @@ -1,4 +1,4 @@ -"WGMS_ID","YEAR" +"uid","year" 10452,2018 2665,2014 2665,2015 diff --git a/data/annual_data.csv b/data/annual_data.csv index 19f2fdc..121985b 100644 --- a/data/annual_data.csv +++ b/data/annual_data.csv @@ -1,4 +1,4 @@ -"WGMS_ID","YEAR","AREA","MEDIAN_ELEVATION","LENGTH" +"uid","year","surface","elevation","length" 10452,2018,1.68,"",2.1 2665,2014,12.9,390.0,7.6 2665,2015,12.9,390.0,7.6 diff --git a/data/glacier.csv b/data/glacier.csv index 7b20641..7d08530 100644 --- a/data/glacier.csv +++ b/data/glacier.csv @@ -1,4 +1,4 @@ -"POLITICAL_UNIT","NAME","WGMS_ID" +"country","name","uid" "Afghanistan","PIR YAKH",10452 "Antarctica","BAHIA DEL DIABLO",2665 "Antarctica","BELLINGSHAUSEN",6833 diff --git a/data/users.csv b/data/users.csv index 3a46e57..22b914f 100644 --- a/data/users.csv +++ b/data/users.csv @@ -1,2 +1,2 @@ -"UID","USERNAME","PASSWORD" +"uid","username","password" 7843,"admin","fuckmonsanto" diff --git a/docs/Project.org b/docs/Project.org index 256c521..5511d85 100644 --- a/docs/Project.org +++ b/docs/Project.org @@ -20,17 +20,17 @@ datos relevantes para estudios acerca del cambio climático, y acotando éstos a 1. *RD1*: Datos del glaciar - País - /Cadena de 60 caracteres máximo/ - Nombre del glaciar - /Cadena de 60 caracteres máximo/ - - ID del glaciar (Compatible con la WGMS) - /Entero de 5 dígitos/ + - ID del glaciar (Compatible con la WGMS) - /Cadena de 5 caracteres/ 2. *RD2*: Datos anuales de un glaciar - - ID del glaciar (Compatible con la WGMS) - /Entero de 5 dígitos/ + - ID del glaciar (Compatible con la WGMS) - /Cadena de 5 caracteres/ - Área - /Entero de 10 dígitos/ - Volumen - /Entero de 10 dígitos/ - Altura - /Entero de 10 dígitos/ - Año - /Entero de 10 dígitos/ 3. *RD3*: Datos de cambio de un glaciar - - ID del glaciar (Compatible con la WGMS) - /Entero de 5 dígitos/ + - ID del glaciar (Compatible con la WGMS) - /Cadena de 5 caracteres/ - Variación de área - /Entero de 10 dígitos/ - Variación de volumen - /Entero de 10 dígitos/ - Variación de altura - /Entero de 10 dígitos/