From a20dab0053f8826355c7ade417b9602a513adc91 Mon Sep 17 00:00:00 2001
From: coolneng <akasroua@gmail.com>
Date: Fri, 12 Jun 2020 19:21:50 +0200
Subject: [PATCH] Add dataframe column assertion

---
 app/{request_datasets.py => data_request.py} |  3 +++
 app/preprocessing.py                         |  6 ++++--
 constants.py                                 |  1 -
 tests/preprocessing_test.py                  | 11 +++++++----
 4 files changed, 14 insertions(+), 7 deletions(-)
 rename app/{request_datasets.py => data_request.py} (92%)

diff --git a/app/request_datasets.py b/app/data_request.py
similarity index 92%
rename from app/request_datasets.py
rename to app/data_request.py
index 946195d..8d5069c 100644
--- a/app/request_datasets.py
+++ b/app/data_request.py
@@ -1,5 +1,7 @@
 from json import dump
+
 from requests import get
+
 from constants import FILES, URL
 
 
@@ -22,6 +24,7 @@ def save_json(data, dataset):
 def request_dataset(dataset):
     """
     Fetches the requested dataset from opendata's API
+    Raises an exception if there's an HTTP error
     """
     url = format_url(dataset)
     response = get(url)
diff --git a/app/preprocessing.py b/app/preprocessing.py
index c9456c7..b600d3e 100644
--- a/app/preprocessing.py
+++ b/app/preprocessing.py
@@ -1,6 +1,8 @@
 from json import load
-from pandas import json_normalize, DataFrame
-from constants import FILES, COLUMNS
+
+from pandas import DataFrame, json_normalize
+
+from constants import COLUMNS, FILES
 
 
 def open_json(dataset) -> dict:
diff --git a/constants.py b/constants.py
index 1675fc2..187d00d 100644
--- a/constants.py
+++ b/constants.py
@@ -4,7 +4,6 @@ DATASETS = [
     "deconfinement-parking-relais-doublement-des-places",
 ]
 URL = "https://opendata.paris.fr/api/records/1.0/search/?dataset={}&q=&rows=-1"
-TEST_URL = "https://opendata.paris.fr/api/records/1.0/search/?dataset=deconfinement-pistes-cyclables-temporaires&rows=-1"
 FILES = {
     "deconfinement-pistes-cyclables-temporaires": "data/cycling-paths.json",
     "deconfinement-parking-relais-doublement-des-places": "data/relay-parking.json",
diff --git a/tests/preprocessing_test.py b/tests/preprocessing_test.py
index 8fa6a43..15c0f2b 100644
--- a/tests/preprocessing_test.py
+++ b/tests/preprocessing_test.py
@@ -1,9 +1,11 @@
-from constants import FILES, DATASETS, URL
-from app.preprocessing import create_dataframe
-from app.request_datasets import request_dataset
+from os import remove
+
 from pandas import DataFrame
 from requests import get
-from os import remove
+
+from app.preprocessing import create_dataframe
+from app.data_request import request_dataset
+from constants import COLUMNS, DATASETS, FILES, URL
 
 
 def test_dataset_request():
@@ -24,3 +26,4 @@ def test_dataframe_creation():
         df = create_dataframe(dataset)
         remove(FILES[dataset])
         assert isinstance(df, DataFrame)
+        assert all(df.columns == COLUMNS[dataset])