Add SQL insertion functionality

Design.org

@@ -62,3 +62,4 @@ CLOSED: [2020-01-03 Fri 00:44]
 *** NEXT Documentation
 *** INACTIVE Frontend [0/1] [0%]
 - [ ] [[https://adminlte.io/][Adminlte]]
+- [ ] Bootstrap

code/database/db_setup.py

@@ -29,9 +29,11 @@ db = create_connection()
 
 
 class Glacier(db.Model):
-    uid = db.Column(db.Integer, primary_key=True)
+    uid = db.Column(db.String(5), primary_key=True)
     country = db.Column(db.String(60))
     name = db.Column(db.String(60))
+    annual_data = db.relationship("Annual_Data")
+    annual_change = db.relationship("Annual_Change")
 
     def __init__(self, uid, country, name):
         self.uid = uid
@@ -40,37 +42,30 @@ class Glacier(db.Model):
 
 
 class Annual_Data(db.Model):
-    glacier = db.relationship("Glacier", backref=db.backref("posts", lazy=True))
+    year = db.Column(db.Integer, primary_key=True)
-    uid = db.Column(
+    uid = db.Column(db.String(5), db.ForeignKey("glacier.uid"), primary_key=True)
-        db.Integer, db.ForeignKey("glacier.uid"), nullable=False, primary_key=True
-    )
     surface = db.Column(db.Integer)
     length = db.Column(db.Integer)
     elevation = db.Column(db.Integer)
-    year = db.Column(db.Integer, primary_key=True)
 
-    def __init__(self, surface, length, elevation, year):
+    def __init__(self, year, surface, length, elevation):
+        self.year = year
         self.surface = surface
         self.length = length
         self.elevation = elevation
-        self.year = year
 
 
 class Annual_Change(db.Model):
-    glacier = db.relationship("Glacier", backref=db.backref("posts", lazy=True))
+    year = db.Column(db.Integer, db.ForeignKey("annual__data.year"), primary_key=True)
-    uid = db.Column(
+    uid = db.Column(db.String(5), db.ForeignKey("glacier.uid"), primary_key=True)
-        db.Integer, db.ForeignKey("glacier.uid"), nullable=False, primary_key=True
-    )
     surface_change = db.Column(db.Integer)
     length_change = db.Column(db.Integer)
     elevation_change = db.Column(db.Integer)
-    year = db.Column(db.Integer, primary_key=True)
 
-    def __init__(self, surface_change, length_change, elevation_change, year):
+    def __init__(self, surface_change, length_change, elevation_change):
         self.surface_change = surface_change
         self.length_change = length_change
         self.elevation_change = elevation_change
-        self.year = year
 
 
 class User(db.Model):

code/database/parser.py

@@ -34,8 +34,24 @@ def select_columns() -> DataFrame:
     return data
 
 
+def rename_fields(df_list):
+    new_df_list = {}
+    new_fields = {
+        "POLITICAL_UNIT": "country",
+        "NAME": "name",
+        "WGMS_ID": "uid",
+        "YEAR": "year",
+        "MEDIAN_ELEVATION": "elevation",
+        "AREA": "surface",
+        "LENGTH": "length",
+    }
+    for key, value in df_list.items():
+        new_df_list[key] = value.rename(columns=new_fields)
+    return new_df_list
+
+
 def create_databases(df):
-    users = {"UID": [7843], "USERNAME": ["admin"], "PASSWORD": [ADMIN_PW]}
+    users = {"uid": [7843], "username": ["admin"], "password": [ADMIN_PW]}
     files = {
         "glacier": "../data/glacier.csv",
         "annual__data": "../data/annual_data.csv",
@@ -48,7 +64,8 @@ def create_databases(df):
         "annual__change": df[["WGMS_ID", "YEAR"]],
         "users": DataFrame(users),
     }
-    for key, val in dataframes.items():
+    renamed_dfs = rename_fields(dataframes)
+    for key, val in renamed_dfs.items():
         val.to_csv(files[key], index=False, quoting=QUOTE_NONNUMERIC)
 
 

code/database/storage.py

@@ -1,5 +1,15 @@
-from db_setup import create_connection
+from constants import DB_NAME, DB_PW, DB_USER
-from pandas import DataFrame, read_csv, to_sql
+from pandas import DataFrame, read_csv
+from sqlalchemy import create_engine, engine
+
+
+def create_connection() -> engine:
+    host = "localhost:3306"
+    connection_uri = "mysql+pymysql://{user}:{pw}@{url}/{db}".format(
+        user=DB_USER, pw=DB_PW, url=host, db=DB_NAME
+    )
+    engine = create_engine(connection_uri)
+    return engine
 
 
 def create_dataframes() -> DataFrame:
@@ -11,13 +21,13 @@ def create_dataframes() -> DataFrame:
     }
     df_list = {}
     for csv in files.keys():
-        df_list[csv] = read_csv(files[csv], index=False)
+        df_list[csv] = read_csv(files[csv])
     return df_list
 
 
 def insert_data(df_list, conn):
     for key, value in df_list.items():
-        value.to_sql(key, con=conn, if_exists="replace", index=False)
+        value.to_sql(key, con=conn, index=False, if_exists="append", chunksize=1)
 
 
 def main():

data/annual_change.csv

@@ -1,4 +1,4 @@
-"WGMS_ID","YEAR"
+"uid","year"
 10452,2018
 2665,2014
 2665,2015

data/annual_data.csv

@@ -1,4 +1,4 @@
-"WGMS_ID","YEAR","AREA","MEDIAN_ELEVATION","LENGTH"
+"uid","year","surface","elevation","length"
 10452,2018,1.68,"",2.1
 2665,2014,12.9,390.0,7.6
 2665,2015,12.9,390.0,7.6

data/glacier.csv

@@ -1,4 +1,4 @@
-"POLITICAL_UNIT","NAME","WGMS_ID"
+"country","name","uid"
 "Afghanistan","PIR YAKH",10452
 "Antarctica","BAHIA DEL DIABLO",2665
 "Antarctica","BELLINGSHAUSEN",6833

data/users.csv

@@ -1,2 +1,2 @@
-"UID","USERNAME","PASSWORD"
+"uid","username","password"
 7843,"admin","fuckmonsanto"

docs/Project.org

@@ -20,17 +20,17 @@ datos relevantes para estudios acerca del cambio climático, y acotando éstos a
 1. *RD1*: Datos del glaciar
    - País - /Cadena de 60 caracteres máximo/
    - Nombre del glaciar - /Cadena de 60 caracteres máximo/
-   - ID del glaciar (Compatible con la WGMS) - /Entero de 5 dígitos/
+   - ID del glaciar (Compatible con la WGMS) - /Cadena de 5 caracteres/
 
 2. *RD2*: Datos anuales de un glaciar
-   - ID del glaciar (Compatible con la WGMS) - /Entero de 5 dígitos/
+   - ID del glaciar (Compatible con la WGMS) - /Cadena de 5 caracteres/
    - Área - /Entero de 10 dígitos/
    - Volumen - /Entero de 10 dígitos/
    - Altura - /Entero de 10 dígitos/
    - Año - /Entero de 10 dígitos/
         
 3. *RD3*: Datos de cambio de un glaciar
-   - ID del glaciar (Compatible con la WGMS) - /Entero de 5 dígitos/
+   - ID del glaciar (Compatible con la WGMS) - /Cadena de 5 caracteres/
    - Variación de área - /Entero de 10 dígitos/
    - Variación de volumen - /Entero de 10 dígitos/
    - Variación de altura - /Entero de 10 dígitos/