From 7eba74b3efd6491b7118e9471f2751107dbd06f8 Mon Sep 17 00:00:00 2001
From: coolneng <akasroua@gmail.com>
Date: Fri, 5 Jun 2020 15:24:16 +0200
Subject: [PATCH] Flatten the JSON into a DataFrame

---
 app/constants.py        |  7 ++++++-
 app/preprocessing.py    | 10 ++--------
 app/request_datasets.py |  3 ---
 3 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/app/constants.py b/app/constants.py
index ae6a9bb..9527b12 100644
--- a/app/constants.py
+++ b/app/constants.py
@@ -15,7 +15,12 @@ FILES = {
     "home-delivery": "data/home-delivery.json",
 }
 COLUMNS = {
-    "cycling-paths": ["geo_shape", "statut", "record_timestamp", "complement"],
+    "cycling-paths": [
+        ["fields", "geo_shape", "coordinates"],
+        "statut",
+        "record_timestamp",
+        "complement",
+    ],
     "relay-parking": [
         "societe",
         "nb_places_dispositif_environ",
diff --git a/app/preprocessing.py b/app/preprocessing.py
index bd451dc..400473c 100644
--- a/app/preprocessing.py
+++ b/app/preprocessing.py
@@ -1,5 +1,5 @@
 from json import load
-from pandas import json_normalize, DataFrame
+from pandas import json_normalize, DataFrame, set_option
 from constants import FILES, COLUMNS
 
 
@@ -17,11 +17,5 @@ def create_dataframe(dataset) -> DataFrame:
     Creates a DataFrame from a JSON file
     """
     json = open_json(dataset)
-    df = json_normalize(
-        data=json, record_path=["records"], meta=COLUMNS[dataset], errors="ignore",
-    )
-    print(df)
+    df = json_normalize(data=json, record_path=["records"], errors="ignore",)
     return df
-
-
-create_dataframe("cycling-paths")
diff --git a/app/request_datasets.py b/app/request_datasets.py
index 8b35600..2a4eb2e 100644
--- a/app/request_datasets.py
+++ b/app/request_datasets.py
@@ -29,6 +29,3 @@ def request_dataset(dataset):
     response.raise_for_status()
     data = response.json()
     save_json(data=data, dataset=dataset)
-
-
-request_dataset("deconfinement-pistes-cyclables-temporaires")