igdb/code/database/parse_csv.py

from pandas import read_csv, concat, DataFrame
from iso3166 import countries as co


def country_conversion(political_unit) -> str:
    codes = co.get(political_unit)
    return codes.name


def select_columns() -> DataFrame:
    min_year = 2010
    fields = [
        "POLITICAL_UNIT",
        "WGMS_ID",
        "YEAR",
        "AREA_SURVEY_YEAR",
        "AREA_CHANGE",
        "THICKNESS CHANGE",
        "VOLUME_CHANGE",
    ]
    iter_csv = read_csv(
        "../../data/WGMS-FoG-2019-12-D-CHANGE.csv",
        skipinitialspace=True,
        usecols=fields,
        iterator=True,
        chunksize=100,
        converters={"YEAR": country_conversion},
    )
    data = concat([chunk[chunk["YEAR"] > min_year] for chunk in iter_csv])
    return data