Fix duplicate 'users' table insertion

This commit is contained in:
coolneng 2020-01-08 01:47:12 +01:00
parent 7b945e2b98
commit cbc298a629
Signed by: coolneng
GPG Key ID: 9893DA236405AF57
5 changed files with 29 additions and 15 deletions

11
Pipfile Normal file
View File

@ -0,0 +1,11 @@
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
[packages]
[requires]
python_version = "3.8"

View File

@ -1,7 +1,7 @@
from subprocess import run from subprocess import run
from flask_sqlalchemy import SQLAlchemy from flask_sqlalchemy import SQLAlchemy
from flask import Flask
from datetime import datetime from datetime import datetime
from flask import Flask
from constants import DB_NAME, DB_USER, DB_PW from constants import DB_NAME, DB_USER, DB_PW
@ -42,6 +42,7 @@ class Glacier(db.Model):
class Annual_Data(db.Model): class Annual_Data(db.Model):
__tablename__ = "annual_data"
year = db.Column(db.Integer, primary_key=True) year = db.Column(db.Integer, primary_key=True)
uid = db.Column(db.String(5), db.ForeignKey("glacier.uid"), primary_key=True) uid = db.Column(db.String(5), db.ForeignKey("glacier.uid"), primary_key=True)
surface = db.Column(db.Integer) surface = db.Column(db.Integer)
@ -56,7 +57,8 @@ class Annual_Data(db.Model):
class Annual_Change(db.Model): class Annual_Change(db.Model):
year = db.Column(db.Integer, db.ForeignKey("annual__data.year"), primary_key=True) __tablename__ = "annual_change"
year = db.Column(db.Integer, db.ForeignKey("annual_data.year"), primary_key=True)
uid = db.Column(db.String(5), db.ForeignKey("glacier.uid"), primary_key=True) uid = db.Column(db.String(5), db.ForeignKey("glacier.uid"), primary_key=True)
surface_change = db.Column(db.Integer) surface_change = db.Column(db.Integer)
length_change = db.Column(db.Integer) length_change = db.Column(db.Integer)
@ -70,13 +72,14 @@ class Annual_Change(db.Model):
class User(db.Model): class User(db.Model):
uid = db.Column(db.Integer, primary_key=True) uid = db.Column(db.Integer, primary_key=True)
registration_date = db.Column(db.DateTime, nullable=False) registration_date = db.Column(
db.DateTime, nullable=False, server_default=db.func.now()
)
username = db.Column(db.String(20), nullable=False, unique=True) username = db.Column(db.String(20), nullable=False, unique=True)
password = db.Column(db.String(60)) password = db.Column(db.String(60))
def __init__(self, uid, username, password): def __init__(self, uid, username, password):
self.uid = uid self.uid = uid
self.registration_date = datetime.today()
self.username = username self.username = username
self.password = password self.password = password

View File

@ -15,9 +15,9 @@ def create_connection() -> engine:
def create_dataframes() -> DataFrame: def create_dataframes() -> DataFrame:
files = { files = {
"glacier": "../data/glacier.csv", "glacier": "../data/glacier.csv",
"annual__data": "../data/annual_data.csv", "annual_data": "../data/annual_data.csv",
"annual__change": "../data/annual_change.csv", "annual_change": "../data/annual_change.csv",
"users": "../data/users.csv", "user": "../data/user.csv",
} }
df_list = {} df_list = {}
for csv in files.keys(): for csv in files.keys():
@ -27,7 +27,7 @@ def create_dataframes() -> DataFrame:
def insert_data(df_list, conn): def insert_data(df_list, conn):
for key, value in df_list.items(): for key, value in df_list.items():
value.to_sql(key, con=conn, index=False, if_exists="append", chunksize=1) value.to_sql(key, con=conn, index=False, if_exists="append")
def main(): def main():

View File

@ -51,18 +51,18 @@ def rename_fields(df_list):
def create_databases(df): def create_databases(df):
users = {"uid": [7843], "username": ["admin"], "password": [ADMIN_PW]} user = {"uid": [7843], "username": ["admin"], "password": [ADMIN_PW]}
files = { files = {
"glacier": "../data/glacier.csv", "glacier": "../data/glacier.csv",
"annual__data": "../data/annual_data.csv", "annual_data": "../data/annual_data.csv",
"annual__change": "../data/annual_change.csv", "annual_change": "../data/annual_change.csv",
"users": "../data/users.csv", "user": "../data/user.csv",
} }
dataframes = { dataframes = {
"glacier": df[["POLITICAL_UNIT", "NAME", "WGMS_ID"]].drop_duplicates(), "glacier": df[["POLITICAL_UNIT", "NAME", "WGMS_ID"]].drop_duplicates(),
"annual__data": df[["WGMS_ID", "YEAR", "AREA", "MEDIAN_ELEVATION", "LENGTH"]], "annual_data": df[["WGMS_ID", "YEAR", "AREA", "MEDIAN_ELEVATION", "LENGTH"]],
"annual__change": df[["WGMS_ID", "YEAR"]], "annual_change": df[["WGMS_ID", "YEAR"]],
"users": DataFrame(users), "user": DataFrame(user),
} }
renamed_dfs = rename_fields(dataframes) renamed_dfs = rename_fields(dataframes)
for key, val in renamed_dfs.items(): for key, val in renamed_dfs.items():