Add dataframes filtering

app/constants.py

@@ -14,3 +14,27 @@ files = {
     "relay-parking": "data/relay-parking.json",
     "home-delivery": "data/home-delivery.json",
 }
+columns = {
+    "cycling-paths": ["geo_shape", "statut", "record_timestamp", "complement"],
+    "relay-parking": [
+        "societe",
+        "nb_places_dispositif_environ",
+        "parcs",
+        "geo_shape",
+        "cp",
+        "ville",
+        "adresse",
+    ],
+    "home-delivery": [
+        "geo_shape",
+        "adresse",
+        "code_postal",
+        "nom_du_commerce",
+        "type_du_commerce",
+        "site_internet",
+        "record_timestamp",
+        "precisions",
+        "telephone",
+        "mail",
+    ],
+}

app/preprocessing.py

@@ -1,6 +1,6 @@
 from json import load
 from pandas import json_normalize, DataFrame
-from app.constants import files
+from constants import files, columns
 
 
 def open_json(dataset):
@@ -11,6 +11,10 @@ def open_json(dataset):
 
 def create_dataframe(dataset):
     json = open_json(dataset)
-    data = json_normalize(data=json["records"])
-    df = DataFrame.from_dict(data=data)
+    df = json_normalize(
+        data=json["records"],
+        record_path=["fields"],
+        meta=columns[dataset],
+        errors="ignore",
+    )
     return df