coolneng
/
igdb
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Create constants file and move tables to db_setup

This commit is contained in:
coolneng 2020-01-05 02:51:14 +01:00
parent c2d6c2363b
commit 8f31f6ff84
Signed by: coolneng
GPG Key ID: 9893DA236405AF57
7 changed files with 92 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
Code/database/db_setup.py
View File

@ -1,49 +0,0 @@
from subprocess import run
from sys import argv
from flask_sqlalchemy import SQLAlchemy
from flask import Flask
from fields import Glacier, Annual_Data, Annual_Change, User
global db
def create_database(db_name, db_user, db_pw):
script = "mariadb_setup.sh"
output = run([script + db_name + db_user + db_pw])
if output.returncode != 0:
print("Error: couldn't create database")
exit()
def create_connection(db_name, db_user, db_pw):
host = "localhost:3306"
connection_uri = "mysql+pymysql://{user}:{pw}@{url}/{db}".format(
user=db_user, pw=db_pw, url=host, db=db_name
)
app = Flask(__name__)
app.config["SQLALCHEMY_DATABASE_URI"] = connection_uri
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
global db
db = SQLAlchemy(app)
def create_tables(db):
db.create_all()
def main():
if len(argv) != 3:
print("Usage: " + argv[0] + " <db name> <db user> <db password>")
exit()
db_name = str(argv[1])
db_user = str(argv[2])
db_password = str(argv[3])
create_database(db_name, db_user, db_password)
create_connection(db_name, db_user, db_password)
create_tables(db)
if __name__ == "__main__":
main()

30
Code/database/parse_csv.py
View File

@ -1,30 +0,0 @@
from pandas import read_csv, concat, DataFrame
from iso3166 import countries as co
def country_conversion(political_unit) -> str:
codes = co.get(political_unit)
return codes.name
def select_columns() -> DataFrame:
min_year = 2010
fields = [
"POLITICAL_UNIT",
"WGMS_ID",
"YEAR",
"AREA_SURVEY_YEAR",
"AREA_CHANGE",
"THICKNESS CHANGE",
"VOLUME_CHANGE",
]
iter_csv = read_csv(
"../../data/WGMS-FoG-2019-12-D-CHANGE.csv",
skipinitialspace=True,
usecols=fields,
iterator=True,
chunksize=100,
converters={"YEAR": country_conversion},
)
data = concat([chunk[chunk["YEAR"] > min_year] for chunk in iter_csv])
return data

52
code/database/columns.py
View File

@ -1,52 +0,0 @@
from db_setup import db
from datetime import datetime
class Glacier(db.Model):
uid = db.Column(db.Integer, primary_key=True)
country = db.Column(db.String(30))
name = db.Column(db.String(30))
def __init__(self, uid, country, name):
self.uid = uid
self.country = country
self.name = name
class Annual_Data(db.Model):
glacier = db.relationship("Glacier", backref=db.backref("posts", lazy=True))
uid = db.Column(db.Integer, db.ForeignKey("glacier.uid"), nullable=False)
surface = db.Column(db.Integer)
volume = db.Column(db.Integer)
thickness = db.Column(db.Integer)
year = db.Column(db.Integer)
def __init__(self, surface, volume, thickness, year):
self.surface = surface
self.volume = volume
self.thickness = thickness
self.year = year
class Annual_Change(db.Model):
glacier = db.relationship("Glacier", backref=db.backref("posts", lazy=True))
uid = db.Column(db.Integer, db.ForeignKey("glacier.uid"), nullable=False)
surface_change = db.Column(db.Integer)
volume_change = db.Column(db.Integer)
thickness_change = db.Column(db.Integer)
year = db.Column(db.Integer)
def __init__(self, surface_change, volume_change, thickness_change, year):
self.surface_change = surface_change
self.volume_change = volume_change
self.thickness_change = thickness_change
self.year = year
class Users(db.Model):
uid = db.Column(db.Integer, primary_key=True)
registration_date = db.Column(db.DateTime, nullable=False)
def __init__(self, uid):
self.uid = uid
self.registration_date = datetime.today()

3
code/database/constants.py Normal file
View File

@ -0,0 +1,3 @@
DB_NAME = "igdb"
DB_USER = "igdb"
DB_PW = "agentorange"

94
code/database/db_setup.py
View File

@ -1,48 +1,94 @@
from subprocess import call
from sys import argv
from subprocess import run
from flask_sqlalchemy import SQLAlchemy
from flask import Flask
from fields import Glacier, Annual_Data, Annual_Change, User
global db
from datetime import datetime
from constants import DB_NAME, DB_USER, DB_PW
def create_database(db_name, db_user, db_pw):
script = "mariadb_setup.sh"
output = call([script + db_name + db_user + db_pw])
if output != 0:
def create_database():
script = "database/mariadb_setup.sh"
output = run([script, DB_NAME, DB_USER, DB_PW])
if output.returncode != 0:
print("Error: couldn't create database")
exit()
def create_connection(db_name, db_user, db_pw):
def create_connection():
host = "localhost:3306"
connection_uri = "mysql+pymysql://{user}:{pw}@{url}/{db}".format(
user=db_user, pw=db_pw, url=host, db=db_name
user=DB_USER, pw=DB_PW, url=host, db=DB_NAME
)
app = Flask(__name__)
app.config["SQLALCHEMY_DATABASE_URI"] = connection_uri
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
global db
db = SQLAlchemy(app)
return db
def create_tables(db):
db = create_connection()
class Glacier(db.Model):
uid = db.Column(db.Integer, primary_key=True)
country = db.Column(db.String(30))
name = db.Column(db.String(30))
def __init__(self, uid, country, name):
self.uid = uid
self.country = country
self.name = name
class Annual_Data(db.Model):
glacier = db.relationship("Glacier", backref=db.backref("posts", lazy=True))
uid = db.Column(
db.Integer, db.ForeignKey("glacier.uid"), nullable=False, primary_key=True
)
surface = db.Column(db.Integer)
volume = db.Column(db.Integer)
thickness = db.Column(db.Integer)
year = db.Column(db.Integer, primary_key=True)
def __init__(self, surface, volume, thickness, year):
self.surface = surface
self.volume = volume
self.thickness = thickness
self.year = year
class Annual_Change(db.Model):
glacier = db.relationship("Glacier", backref=db.backref("posts", lazy=True))
uid = db.Column(
db.Integer, db.ForeignKey("glacier.uid"), nullable=False, primary_key=True
)
surface_change = db.Column(db.Integer)
volume_change = db.Column(db.Integer)
thickness_change = db.Column(db.Integer)
year = db.Column(db.Integer, primary_key=True)
def __init__(self, surface_change, volume_change, thickness_change, year):
self.surface_change = surface_change
self.volume_change = volume_change
self.thickness_change = thickness_change
self.year = year
class Users(db.Model):
uid = db.Column(db.Integer, primary_key=True)
registration_date = db.Column(db.DateTime, nullable=False)
def __init__(self, uid):
self.uid = uid
self.registration_date = datetime.today()
def create_tables():
db.create_all()
def main():
if len(argv) != 3:
print("Usage: " + argv[0] + " <db name> <db user> <db password>")
exit()
db_name = str(argv[1])
db_user = str(argv[2])
db_password = str(argv[3])
create_database(db_name, db_user, db_password)
create_connection(db_name, db_user, db_password)
create_tables(db)
create_database()
create_tables()
if __name__ == "__main__":

2
code/database/mariadb_setup.sh Normal file → Executable file
View File

@ -2,7 +2,7 @@
EXPECTED_ARGS=3
ERROR=1
MYSQL=$(command -v which)
MYSQL=$(command -v mysql)
if [ $# -ne $EXPECTED_ARGS ]
then

22
code/database/parse_csv.py
View File

@ -1,16 +1,30 @@
from pandas import read_csv, concat, DataFrame
from iso3166 import countries
from iso3166 import countries as co
def select_columns() -> [DataFrame]:
def country_conversion(political_unit) -> str:
codes = co.get(political_unit)
return codes.name
def select_columns() -> DataFrame:
min_year = 2010
fields = ["POLITICAL_UNIT", "WGMS_ID", "YEAR"]
fields = [
"POLITICAL_UNIT",
"WGMS_ID",
"YEAR",
"AREA_SURVEY_YEAR",
"AREA_CHANGE",
"THICKNESS CHANGE",
"VOLUME_CHANGE",
]
iter_csv = read_csv(
"../../Assets/WGMS-FoG-2019-12-D-CHANGE.csv",
"../../data/WGMS-FoG-2019-12-D-CHANGE.csv",
skipinitialspace=True,
usecols=fields,
iterator=True,
chunksize=100,
converters={"YEAR": country_conversion},
)
data = concat([chunk[chunk["YEAR"] > min_year] for chunk in iter_csv])
return data