From 8f31f6ff8409c1d161114e4d1ad7a32d094f7426 Mon Sep 17 00:00:00 2001 From: coolneng Date: Sun, 5 Jan 2020 02:51:14 +0100 Subject: [PATCH] Create constants file and move tables to db_setup --- Code/database/db_setup.py | 49 ------------------ Code/database/parse_csv.py | 30 ----------- code/database/columns.py | 52 ------------------- code/database/constants.py | 3 ++ code/database/db_setup.py | 94 +++++++++++++++++++++++++--------- code/database/mariadb_setup.sh | 2 +- code/database/parse_csv.py | 22 ++++++-- 7 files changed, 92 insertions(+), 160 deletions(-) delete mode 100644 Code/database/db_setup.py delete mode 100644 Code/database/parse_csv.py delete mode 100644 code/database/columns.py create mode 100644 code/database/constants.py mode change 100644 => 100755 code/database/mariadb_setup.sh diff --git a/Code/database/db_setup.py b/Code/database/db_setup.py deleted file mode 100644 index 00eaf77..0000000 --- a/Code/database/db_setup.py +++ /dev/null @@ -1,49 +0,0 @@ -from subprocess import run -from sys import argv -from flask_sqlalchemy import SQLAlchemy -from flask import Flask -from fields import Glacier, Annual_Data, Annual_Change, User - -global db - - -def create_database(db_name, db_user, db_pw): - script = "mariadb_setup.sh" - output = run([script + db_name + db_user + db_pw]) - if output.returncode != 0: - print("Error: couldn't create database") - exit() - - -def create_connection(db_name, db_user, db_pw): - host = "localhost:3306" - connection_uri = "mysql+pymysql://{user}:{pw}@{url}/{db}".format( - user=db_user, pw=db_pw, url=host, db=db_name - ) - app = Flask(__name__) - app.config["SQLALCHEMY_DATABASE_URI"] = connection_uri - app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False - global db - db = SQLAlchemy(app) - - -def create_tables(db): - db.create_all() - - -def main(): - if len(argv) != 3: - print("Usage: " + argv[0] + " ") - exit() - - db_name = str(argv[1]) - db_user = str(argv[2]) - db_password = str(argv[3]) - - create_database(db_name, db_user, db_password) - create_connection(db_name, db_user, db_password) - create_tables(db) - - -if __name__ == "__main__": - main() diff --git a/Code/database/parse_csv.py b/Code/database/parse_csv.py deleted file mode 100644 index ecbb551..0000000 --- a/Code/database/parse_csv.py +++ /dev/null @@ -1,30 +0,0 @@ -from pandas import read_csv, concat, DataFrame -from iso3166 import countries as co - - -def country_conversion(political_unit) -> str: - codes = co.get(political_unit) - return codes.name - - -def select_columns() -> DataFrame: - min_year = 2010 - fields = [ - "POLITICAL_UNIT", - "WGMS_ID", - "YEAR", - "AREA_SURVEY_YEAR", - "AREA_CHANGE", - "THICKNESS CHANGE", - "VOLUME_CHANGE", - ] - iter_csv = read_csv( - "../../data/WGMS-FoG-2019-12-D-CHANGE.csv", - skipinitialspace=True, - usecols=fields, - iterator=True, - chunksize=100, - converters={"YEAR": country_conversion}, - ) - data = concat([chunk[chunk["YEAR"] > min_year] for chunk in iter_csv]) - return data diff --git a/code/database/columns.py b/code/database/columns.py deleted file mode 100644 index d326065..0000000 --- a/code/database/columns.py +++ /dev/null @@ -1,52 +0,0 @@ -from db_setup import db -from datetime import datetime - - -class Glacier(db.Model): - uid = db.Column(db.Integer, primary_key=True) - country = db.Column(db.String(30)) - name = db.Column(db.String(30)) - - def __init__(self, uid, country, name): - self.uid = uid - self.country = country - self.name = name - - -class Annual_Data(db.Model): - glacier = db.relationship("Glacier", backref=db.backref("posts", lazy=True)) - uid = db.Column(db.Integer, db.ForeignKey("glacier.uid"), nullable=False) - surface = db.Column(db.Integer) - volume = db.Column(db.Integer) - thickness = db.Column(db.Integer) - year = db.Column(db.Integer) - - def __init__(self, surface, volume, thickness, year): - self.surface = surface - self.volume = volume - self.thickness = thickness - self.year = year - - -class Annual_Change(db.Model): - glacier = db.relationship("Glacier", backref=db.backref("posts", lazy=True)) - uid = db.Column(db.Integer, db.ForeignKey("glacier.uid"), nullable=False) - surface_change = db.Column(db.Integer) - volume_change = db.Column(db.Integer) - thickness_change = db.Column(db.Integer) - year = db.Column(db.Integer) - - def __init__(self, surface_change, volume_change, thickness_change, year): - self.surface_change = surface_change - self.volume_change = volume_change - self.thickness_change = thickness_change - self.year = year - - -class Users(db.Model): - uid = db.Column(db.Integer, primary_key=True) - registration_date = db.Column(db.DateTime, nullable=False) - - def __init__(self, uid): - self.uid = uid - self.registration_date = datetime.today() diff --git a/code/database/constants.py b/code/database/constants.py new file mode 100644 index 0000000..fb59dbf --- /dev/null +++ b/code/database/constants.py @@ -0,0 +1,3 @@ +DB_NAME = "igdb" +DB_USER = "igdb" +DB_PW = "agentorange" diff --git a/code/database/db_setup.py b/code/database/db_setup.py index 85e712d..c50c8f6 100644 --- a/code/database/db_setup.py +++ b/code/database/db_setup.py @@ -1,48 +1,94 @@ -from subprocess import call -from sys import argv +from subprocess import run from flask_sqlalchemy import SQLAlchemy from flask import Flask -from fields import Glacier, Annual_Data, Annual_Change, User - -global db +from datetime import datetime +from constants import DB_NAME, DB_USER, DB_PW -def create_database(db_name, db_user, db_pw): - script = "mariadb_setup.sh" - output = call([script + db_name + db_user + db_pw]) - if output != 0: +def create_database(): + script = "database/mariadb_setup.sh" + output = run([script, DB_NAME, DB_USER, DB_PW]) + if output.returncode != 0: print("Error: couldn't create database") exit() -def create_connection(db_name, db_user, db_pw): +def create_connection(): host = "localhost:3306" connection_uri = "mysql+pymysql://{user}:{pw}@{url}/{db}".format( - user=db_user, pw=db_pw, url=host, db=db_name + user=DB_USER, pw=DB_PW, url=host, db=DB_NAME ) app = Flask(__name__) app.config["SQLALCHEMY_DATABASE_URI"] = connection_uri app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False - global db db = SQLAlchemy(app) + return db -def create_tables(db): +db = create_connection() + + +class Glacier(db.Model): + uid = db.Column(db.Integer, primary_key=True) + country = db.Column(db.String(30)) + name = db.Column(db.String(30)) + + def __init__(self, uid, country, name): + self.uid = uid + self.country = country + self.name = name + + +class Annual_Data(db.Model): + glacier = db.relationship("Glacier", backref=db.backref("posts", lazy=True)) + uid = db.Column( + db.Integer, db.ForeignKey("glacier.uid"), nullable=False, primary_key=True + ) + surface = db.Column(db.Integer) + volume = db.Column(db.Integer) + thickness = db.Column(db.Integer) + year = db.Column(db.Integer, primary_key=True) + + def __init__(self, surface, volume, thickness, year): + self.surface = surface + self.volume = volume + self.thickness = thickness + self.year = year + + +class Annual_Change(db.Model): + glacier = db.relationship("Glacier", backref=db.backref("posts", lazy=True)) + uid = db.Column( + db.Integer, db.ForeignKey("glacier.uid"), nullable=False, primary_key=True + ) + surface_change = db.Column(db.Integer) + volume_change = db.Column(db.Integer) + thickness_change = db.Column(db.Integer) + year = db.Column(db.Integer, primary_key=True) + + def __init__(self, surface_change, volume_change, thickness_change, year): + self.surface_change = surface_change + self.volume_change = volume_change + self.thickness_change = thickness_change + self.year = year + + +class Users(db.Model): + uid = db.Column(db.Integer, primary_key=True) + registration_date = db.Column(db.DateTime, nullable=False) + + def __init__(self, uid): + self.uid = uid + self.registration_date = datetime.today() + + +def create_tables(): db.create_all() def main(): - if len(argv) != 3: - print("Usage: " + argv[0] + " ") - exit() - - db_name = str(argv[1]) - db_user = str(argv[2]) - db_password = str(argv[3]) - - create_database(db_name, db_user, db_password) - create_connection(db_name, db_user, db_password) - create_tables(db) + create_database() + create_tables() if __name__ == "__main__": diff --git a/code/database/mariadb_setup.sh b/code/database/mariadb_setup.sh old mode 100644 new mode 100755 index ad2f5f2..2e7208c --- a/code/database/mariadb_setup.sh +++ b/code/database/mariadb_setup.sh @@ -2,7 +2,7 @@ EXPECTED_ARGS=3 ERROR=1 -MYSQL=$(command -v which) +MYSQL=$(command -v mysql) if [ $# -ne $EXPECTED_ARGS ] then diff --git a/code/database/parse_csv.py b/code/database/parse_csv.py index 869059b..ecbb551 100644 --- a/code/database/parse_csv.py +++ b/code/database/parse_csv.py @@ -1,16 +1,30 @@ from pandas import read_csv, concat, DataFrame -from iso3166 import countries +from iso3166 import countries as co -def select_columns() -> [DataFrame]: +def country_conversion(political_unit) -> str: + codes = co.get(political_unit) + return codes.name + + +def select_columns() -> DataFrame: min_year = 2010 - fields = ["POLITICAL_UNIT", "WGMS_ID", "YEAR"] + fields = [ + "POLITICAL_UNIT", + "WGMS_ID", + "YEAR", + "AREA_SURVEY_YEAR", + "AREA_CHANGE", + "THICKNESS CHANGE", + "VOLUME_CHANGE", + ] iter_csv = read_csv( - "../../Assets/WGMS-FoG-2019-12-D-CHANGE.csv", + "../../data/WGMS-FoG-2019-12-D-CHANGE.csv", skipinitialspace=True, usecols=fields, iterator=True, chunksize=100, + converters={"YEAR": country_conversion}, ) data = concat([chunk[chunk["YEAR"] > min_year] for chunk in iter_csv]) return data