igdb/code/database/parse_csv.py

31 lines
756 B
Python

from pandas import read_csv, concat, DataFrame
from iso3166 import countries as co
def country_conversion(political_unit) -> str:
codes = co.get(political_unit)
return codes.name
def select_columns() -> DataFrame:
min_year = 2010
fields = [
"POLITICAL_UNIT",
"WGMS_ID",
"YEAR",
"AREA_SURVEY_YEAR",
"AREA_CHANGE",
"THICKNESS CHANGE",
"VOLUME_CHANGE",
]
iter_csv = read_csv(
"../../data/WGMS-FoG-2019-12-D-CHANGE.csv",
skipinitialspace=True,
usecols=fields,
iterator=True,
chunksize=100,
converters={"YEAR": country_conversion},
)
data = concat([chunk[chunk["YEAR"] > min_year] for chunk in iter_csv])
return data