from iso3166 import countries as co from pandas import DataFrame, concat, read_csv from csv import QUOTE_NONNUMERIC from constants import ADMIN_PW def country_conversion(political_unit) -> str: if political_unit == "99": return "99" codes = co.get(political_unit) return codes.name def select_columns() -> DataFrame: min_year = 2010 fields = [ "POLITICAL_UNIT", "NAME", "WGMS_ID", "YEAR", "MEDIAN_ELEVATION", "AREA", "LENGTH", ] iter_csv = read_csv( "../data/WGMS-FoG-2019-12-B-STATE.csv", skipinitialspace=True, usecols=fields, iterator=True, chunksize=100, converters={"POLITICAL_UNIT": country_conversion}, ) data = concat([chunk[chunk["YEAR"] > min_year] for chunk in iter_csv]) return data def create_databases(df): users = {"UID": [7843], "USERNAME": ["admin"], "PASSWORD": [ADMIN_PW]} files = { "glacier": "../data/glacier.csv", "annual__data": "../data/annual_data.csv", "annual__change": "../data/annual_change.csv", "users": "../data/users.csv", } dataframes = { "glacier": df[["POLITICAL_UNIT", "NAME", "WGMS_ID"]].drop_duplicates(), "annual__data": df[["WGMS_ID", "YEAR", "AREA", "MEDIAN_ELEVATION", "LENGTH"]], "annual__change": df[["WGMS_ID", "YEAR"]], "users": DataFrame(users), } for key, val in dataframes.items(): val.to_csv(files[key], index=False, quoting=QUOTE_NONNUMERIC) def main(): df = select_columns() create_databases(df) if __name__ == "__main__": main()