Rename folders, add DB parser and improve DB setup

Assets/auto/Citations.el

@@ -1,7 +0,0 @@
-(TeX-add-style-hook
- "Citations"
- (lambda ()
-   (LaTeX-add-bibitems
-    "wgms-db"))
- :bibtex)
-

Code/Pipfile

@@ -9,6 +9,8 @@ verify_ssl = true
 flask = "*"
 pymysql = "*"
 flask-sqlalchemy = "*"
+pandas = "*"
+iso3166 = "*"
 
 [requires]
 python_version = "3.8"

Code/Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "77464f98cb5a9b5d12663323acdbbc44b448b5d9d2fee7c8e0e6302987ef8bfe"
+            "sha256": "180a68bb151e2c8ab26c87081302da4bd38107869f77507d721f7502631c4684"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -39,6 +39,14 @@
             "index": "pypi",
             "version": "==2.4.1"
         },
+        "iso3166": {
+            "hashes": [
+                "sha256:b07208703bd881a4f974e39fa013c4498dddd64913ada15f24be75d02ae68a44",
+                "sha256:b1e58dbcf50fbb2c9c418ec7a6057f0cdb30b8f822ac852f72e71ba769dae8c5"
+            ],
+            "index": "pypi",
+            "version": "==1.0.1"
+        },
         "itsdangerous": {
             "hashes": [
                 "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19",
@@ -86,6 +94,57 @@
             ],
             "version": "==1.1.1"
         },
+        "numpy": {
+            "hashes": [
+                "sha256:03bbde29ac8fba860bb2c53a1525b3604a9b60417855ac3119d89868ec6041c3",
+                "sha256:1baefd1fb4695e7f2e305467dbd876d765e6edd30c522894df76f8301efaee36",
+                "sha256:1c35fb1131362e6090d30286cfda52ddd42e69d3e2bf1fea190a0fad83ea3a18",
+                "sha256:3c68c827689ca0ca713dba598335073ce0966850ec0b30715527dce4ecd84055",
+                "sha256:443ab93fc35b31f01db8704681eb2fd82f3a1b2fa08eed2dd0e71f1f57423d4a",
+                "sha256:56710a756c5009af9f35b91a22790701420406d9ac24cf6b652b0e22cfbbb7ff",
+                "sha256:62506e9e4d2a39c87984f081a2651d4282a1d706b1a82fe9d50a559bb58e705a",
+                "sha256:6f8113c8dbfc192b58996ee77333696469ea121d1c44ea429d8fd266e4c6be51",
+                "sha256:712f0c32555132f4b641b918bdb1fd3c692909ae916a233ce7f50eac2de87e37",
+                "sha256:854f6ed4fa91fa6da5d764558804ba5b0f43a51e5fe9fc4fdc93270b052f188a",
+                "sha256:88c5ccbc4cadf39f32193a5ef22e3f84674418a9fd877c63322917ae8f295a56",
+                "sha256:905cd6fa6ac14654a6a32b21fad34670e97881d832e24a3ca32e19b455edb4a8",
+                "sha256:9d6de2ad782aae68f7ed0e0e616477fbf693d6d7cc5f0f1505833ff12f84a673",
+                "sha256:a30f5c3e1b1b5d16ec1f03f4df28e08b8a7529d8c920bbed657f4fde61f1fbcd",
+                "sha256:a9d72d9abaf65628f0f31bbb573b7d9304e43b1e6bbae43149c17737a42764c4",
+                "sha256:ac3cf835c334fcc6b74dc4e630f9b5ff7b4c43f7fb2a7813208d95d4e10b5623",
+                "sha256:b091e5d4cbbe79f0e8b6b6b522346e54a282eadb06e3fd761e9b6fafc2ca91ad",
+                "sha256:cc070fc43a494e42732d6ae2f6621db040611c1dde64762a40c8418023af56d7",
+                "sha256:e1080e37c090534adb2dd7ae1c59ee883e5d8c3e63d2a4d43c20ee348d0459c5",
+                "sha256:f084d513de729ff10cd72a1f80db468cff464fedb1ef2fea030221a0f62d7ff4",
+                "sha256:f6a7421da632fc01e8a3ecd19c3f7350258d82501a646747664bae9c6a87c731"
+            ],
+            "version": "==1.18.0"
+        },
+        "pandas": {
+            "hashes": [
+                "sha256:00dff3a8e337f5ed7ad295d98a31821d3d0fe7792da82d78d7fd79b89c03ea9d",
+                "sha256:22361b1597c8c2ffd697aa9bf85423afa9e1fcfa6b1ea821054a244d5f24d75e",
+                "sha256:255920e63850dc512ce356233081098554d641ba99c3767dde9e9f35630f994b",
+                "sha256:26382aab9c119735908d94d2c5c08020a4a0a82969b7e5eefb92f902b3b30ad7",
+                "sha256:33970f4cacdd9a0ddb8f21e151bfb9f178afb7c36eb7c25b9094c02876f385c2",
+                "sha256:4545467a637e0e1393f7d05d61dace89689ad6d6f66f267f86fff737b702cce9",
+                "sha256:52da74df8a9c9a103af0a72c9d5fdc8e0183a90884278db7f386b5692a2220a4",
+                "sha256:61741f5aeb252f39c3031d11405305b6d10ce663c53bc3112705d7ad66c013d0",
+                "sha256:6a3ac2c87e4e32a969921d1428525f09462770c349147aa8e9ab95f88c71ec71",
+                "sha256:7458c48e3d15b8aaa7d575be60e1e4dd70348efcd9376656b72fecd55c59a4c3",
+                "sha256:78bf638993219311377ce9836b3dc05f627a666d0dbc8cec37c0ff3c9ada673b",
+                "sha256:8153705d6545fd9eb6dd2bc79301bff08825d2e2f716d5dced48daafc2d0b81f",
+                "sha256:975c461accd14e89d71772e89108a050fa824c0b87a67d34cedf245f6681fc17",
+                "sha256:9962957a27bfb70ab64103d0a7b42fa59c642fb4ed4cb75d0227b7bb9228535d",
+                "sha256:adc3d3a3f9e59a38d923e90e20c4922fc62d1e5a03d083440468c6d8f3f1ae0a",
+                "sha256:bbe3eb765a0b1e578833d243e2814b60c825b7fdbf4cdfe8e8aae8a08ed56ecf",
+                "sha256:df8864824b1fe488cf778c3650ee59c3a0d8f42e53707de167ba6b4f7d35f133",
+                "sha256:e45055c30a608076e31a9fcd780a956ed3b1fa20db61561b8d88b79259f526f7",
+                "sha256:ee50c2142cdcf41995655d499a157d0a812fce55c97d9aad13bc1eef837ed36c"
+            ],
+            "index": "pypi",
+            "version": "==0.25.3"
+        },
         "pymysql": {
             "hashes": [
                 "sha256:3943fbbbc1e902f41daf7f9165519f140c4451c179380677e6a848587042561a",
@@ -94,6 +153,27 @@
             "index": "pypi",
             "version": "==0.9.3"
         },
+        "python-dateutil": {
+            "hashes": [
+                "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
+                "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"
+            ],
+            "version": "==2.8.1"
+        },
+        "pytz": {
+            "hashes": [
+                "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d",
+                "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be"
+            ],
+            "version": "==2019.3"
+        },
+        "six": {
+            "hashes": [
+                "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd",
+                "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"
+            ],
+            "version": "==1.13.0"
+        },
         "sqlalchemy": {
             "hashes": [
                 "sha256:bfb8f464a5000b567ac1d350b9090cf081180ec1ab4aa87e7bca12dab25320ec"

Code/database/db_setup.py

@@ -1,25 +1,49 @@
-from os import environ
-
-from sqlalchemy import create_engine
+from subprocess import call
+from sys import argv
+from flask_sqlalchemy import SQLAlchemy
+from flask import Flask
+from fields import Glacier, Annual_Data, Annual_Change, User
 
 
-def create_connection():
-    db_name = get_env_variable("DB_NAME")
-    db_user = get_env_variable("DB_USER")
-    db_password = get_env_variable("DB_PASSWORD")
-    db_url = get_env_variable("DB_URL")
-    db_connection_uri = "mysql+pymysql://{user}:{pw}@{url}/{db}".format(
-        user=db_user, pw=db_password, url=db_url, db=db_name
+def create_database(db_name, db_user, db_pw):
+    script = "mariadb_setup.sh"
+    output = call([script + db_name + db_user + db_pw])
+    if output != 0:
+        print("Error: couldn't create database")
+        exit()
+
+
+def create_connection(db_name, db_user, db_pw):
+    host = "localhost:3306"
+    connection_uri = "mysql+pymysql://{user}:{pw}@{url}/{db}".format(
+        user=db_user, pw=db_pw, url=host, db=db_name
     )
-    engine = create_engine(db_connection_uri, echo=True)
+    app = Flask(__name__)
+    app.config["SQLALCHEMY_DATABASE_URI"] = connection_uri
+    app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
+    global db
+    db = SQLAlchemy(app)
 
 
-def create_tables():
+def create_tables(db):
+    db.create_all()
 
 
-def get_env_variable(name):
-    try:
-        return environ[name]
-    except KeyError:
-        message = "Expected environment variable '{}' not set.".format(name)
-        raise Exception(message)
+def main():
+    if len(argv) != 3:
+        print(
+            "Usage: " + argv[0] + " <database name> <database user> <database password>"
+        )
+        exit()
+
+    db_name = str(argv[1])
+    db_user = str(argv[2])
+    db_password = str(argv[3])
+
+    create_database(db_name, db_user, db_password)
+    create_connection(db_name, db_user, db_password)
+    create_tables(db)
+
+
+if __name__ == "__main__":
+    main()

Code/database/fields.py

@@ -0,0 +1,52 @@
+from db_setup import db
+from datetime import datetime
+
+
+class Glacier(db.Model):
+    uid = db.Column(db.Integer, primary_key=True)
+    country = db.Column(db.String(30))
+    name = db.Column(db.String(30))
+
+    def __init__(self, uid, country, name):
+        self.uid = uid
+        self.country = country
+        self.name = name
+
+
+class Annual_Data(db.Model):
+    glacier = db.relationship("Glacier", backref=db.backref("posts", lazy=True))
+    uid = db.Column(db.Integer, db.ForeignKey("glacier.uid"), nullable=False)
+    surface = db.Column(db.Integer)
+    volume = db.Column(db.Integer)
+    thickness = db.Column(db.Integer)
+    year = db.Column(db.Integer)
+
+    def __init__(self, surface, volume, thickness, year):
+        self.surface = surface
+        self.volume = volume
+        self.thickness = thickness
+        self.year = year
+
+
+class Annual_Change(db.Model):
+    glacier = db.relationship("Glacier", backref=db.backref("posts", lazy=True))
+    uid = db.Column(db.Integer, db.ForeignKey("glacier.uid"), nullable=False)
+    surface_change = db.Column(db.Integer)
+    volume_change = db.Column(db.Integer)
+    thickness_change = db.Column(db.Integer)
+    year = db.Column(db.Integer)
+
+    def __init__(self, surface_change, volume_change, thickness_change, year):
+        self.surface_change = surface_change
+        self.volume_change = volume_change
+        self.thickness_change = thickness_change
+        self.year = year
+
+
+class Users(db.Model):
+    uid = db.Column(db.Integer, primary_key=True)
+    registration_date = db.Column(db.DateTime, nullable=False)
+
+    def __init__(self, uid):
+        self.uid = uid
+        self.registration_date = datetime.today()

Code/database/parse_csv.py

@@ -0,0 +1,16 @@
+from pandas import read_csv, concat, DataFrame
+from iso3166 import countries
+
+
+def select_columns() -> [DataFrame]:
+    min_year = 2010
+    fields = ["POLITICAL_UNIT", "WGMS_ID", "YEAR"]
+    iter_csv = read_csv(
+        "../../Assets/WGMS-FoG-2019-12-D-CHANGE.csv",
+        skipinitialspace=True,
+        usecols=fields,
+        iterator=True,
+        chunksize=100,
+    )
+    data = concat([chunk[chunk["YEAR"] > min_year] for chunk in iter_csv])
+    return data

Design.org

@@ -45,15 +45,18 @@ CLOSED: [2019-11-01 Fri 00:34]
 - [X] Black box
 - [X] Entity-Relationship
 ** Implementation
-*** TODO Backend [0/2] [0%]
-**** TODO Database [2/3] [66%]
+*** TODO Backend [1/3] [33%]
+**** DONE Database [3/3] [100%]
+CLOSED: [2020-01-03 Fri 00:44]
 - [X] Connection
 - [X] Creation from script
-- [ ] Creation of tables via class
+- [X] Creation of tables via class
+**** TODO Parser [0/4] [0%]
+- [ ] Select useful fiels
+- [ ] Convert PU to Country (ISO 3166)
+- [ ] Arithmetic operations for yearly changes
+- [ ] Insert into database
 **** NEXT Flask framework
 -[[https://flask.palletsprojects.com/en/1.1.x/patterns/#patterns][ Patterns]]
-*** NEXT Parsing script [0/2] [0%]
-- [ ] Select useful fiels with awk
-- [ ] Arithmetic operations for comparisons
 *** INACTIVE Frontend [0/1] [0%]
 - [ ] [[https://adminlte.io/][Adminlte]]

Docs/auto/Citations.el

@@ -1,7 +0,0 @@
-(TeX-add-style-hook
- "Citations"
- (lambda ()
-   (LaTeX-add-bibitems
-    "wgms-db"))
- :bibtex)
-

Makefile

@@ -1,10 +1,10 @@
 ## Source files
 ## (Adjust to your needs. Order of markdown files in $(SRC) matters!)
 PANDOC 									=	pandoc
-DOC 										= Docs
-META                    = Assets/metadata.yaml
+DOC 										= docs
+META                    = assets/metadata.yaml
 SRC                     = $(DOC)/Project.md
-BIBFILE                 = Assets/Citations.bib
+BIBFILE                 = assets/Citations.bib
 TARGET                  = $(DOC)/Project.pdf
 
 
@@ -17,7 +17,7 @@ TARGET                  = $(DOC)/Project.pdf
 ## Auxiliary files
 ## (Do not change!)
 TITLEPAGE               = titlepage.tex
-TMP                     = Assets/$(TITLEPAGE:%.tex=__%.filled.tex)
+TMP                     = assets/$(TITLEPAGE:%.tex=__%.filled.tex)
 
 
 ## Pandoc options
@@ -30,13 +30,13 @@ OPTIONS                += --metadata-file=$(META)
 OPTIONS                += -M bibliography=$(BIBFILE)
 OPTIONS                += --listings
 OPTIONS                += --include-in-header=$(TMP)
-OPTIONS                += --resource-path=./Assets/Diagrams
+OPTIONS                += --resource-path=./assets/Diagrams
 
 
 ## Template variables
 TEMPLATE_DL_DIR         = .tmp_template_dl
 
-CLEANTHESIS_TEMPLATE    = Assets/cleanthesis.sty
+CLEANTHESIS_TEMPLATE    = assets/cleanthesis.sty
 
 TEMPLATE_FILES          = $(CLEANTHESIS_TEMPLATE)
 
@@ -52,7 +52,7 @@ cleanthesis: TEMPLATE_FILE    += $(CLEANTHESIS_TEMPLATE)
 cleanthesis: TEMPLATE_REPO    += $(CLEANTHESIS_REPO)
 cleanthesis: TEMPLATE_VERSION += $(CLEANTHESIS_VERSION)
 cleanthesis: AUX_OPTS         += -M cleanthesis=true -M cleanthesisbibfile=$(BIBFILE:%.bib=%)
-cleanthesis: OPTIONS          += --include-in-header=Assets/include-header.tex $(AUX_OPTS)
+cleanthesis: OPTIONS          += --include-in-header=assets/include-header.tex $(AUX_OPTS)
 cleanthesis: $(CLEANTHESIS_TEMPLATE) $(TARGET)
 
 
@@ -80,7 +80,7 @@ ${TARGET}: $(SRC) $(META) $(BIBFILE) $(TMP)
 
 
 ## Build auxiliary files (title page, references)
-$(TMP): Assets/__%.filled.tex: Assets/%.tex $(META)
+$(TMP): assets/__%.filled.tex: assets/%.tex $(META)
 	$(PANDOC) $(AUX_OPTS) --template=$< --metadata-file=$(META) -o $@ $<
 
 

assets/__titlepage.filled.tex

@@ -0,0 +1,54 @@
+
+%------------------------------------- Custom Title Page ---------------------------
+\renewcommand{\maketitle}{
+\thispagestyle{empty}
+
+\parindent=0pt
+
+\vspace{20mm}
+\hrule
+\vspace{5mm}
+
+\begin{center}
+    \LARGE
+    \textbf{\textsc{ IGDB: Base de datos internacional de glaciares }}
+\end{center}
+
+\vspace{5mm}
+\hrule
+\vspace{10mm}
+
+\begin{center}
+    \Large
+    \textsc{ Diseño y Desarrollo de Sistemas de Información }
+\end{center}
+
+\vfill
+
+\noindent
+\includegraphics[width=120pt, center]{/home/coolneng/Pictures/Logos/UGR.png}
+
+\vspace{\baselineskip}\noindent
+\large
+\begin{tabular}{lp{\textwidth}}
+  Autor: & \texttt{Amin Kasrou Aouam}\\
+  Fecha: & \texttt{18/10/2019}\\
+                 &\\[24pt]
+\end{tabular}
+}
+%------------------------------------- Custom Title Page ---------------------------
+
+
+\usepackage[export]{adjustbox}
+\usepackage{graphicx}
+
+%------------------------------------- Workaround for CleanStyle -------------------
+%% cleanthesis.sty *will* check the bibfile, even if `configurebiblatex=false` ...
+%% So we need to set it appropriately using our metadata variable "cleanthesisbibfile"
+\PassOptionsToPackage{
+    figuresep=colon,
+    configurelistings=true,
+    configurebiblatex=false,
+    bibfile=Assets/Citations
+}{cleanthesis}
+%------------------------------------- Workaround for CleanStyle -------------------