coolneng
/
graphPaname
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Flatten the JSON into a DataFrame

This commit is contained in:
coolneng 2020-06-05 15:24:16 +02:00
parent 7909def8e2
commit 7eba74b3ef
Signed by: coolneng
GPG Key ID: 9893DA236405AF57
3 changed files with 8 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
app/constants.py
View File

@ -15,7 +15,12 @@ FILES = {
"home-delivery": "data/home-delivery.json", "home-delivery": "data/home-delivery.json",
} }
COLUMNS = { COLUMNS = {
"cycling-paths": ["geo_shape", "statut", "record_timestamp", "complement"], "cycling-paths": [
["fields", "geo_shape", "coordinates"],
"statut",
"record_timestamp",
"complement",
],
"relay-parking": [ "relay-parking": [
"societe", "societe",
"nb_places_dispositif_environ", "nb_places_dispositif_environ",

10
app/preprocessing.py
View File

@ -1,5 +1,5 @@
from json import load from json import load
from pandas import json_normalize, DataFrame from pandas import json_normalize, DataFrame, set_option
from constants import FILES, COLUMNS from constants import FILES, COLUMNS
@ -17,11 +17,5 @@ def create_dataframe(dataset) -> DataFrame:
Creates a DataFrame from a JSON file Creates a DataFrame from a JSON file
""" """
json = open_json(dataset) json = open_json(dataset)
df = json_normalize( df = json_normalize(data=json, record_path=["records"], errors="ignore",)
data=json, record_path=["records"], meta=COLUMNS[dataset], errors="ignore",
)
print(df)
return df return df
create_dataframe("cycling-paths")

3
app/request_datasets.py
View File

@ -29,6 +29,3 @@ def request_dataset(dataset):
response.raise_for_status() response.raise_for_status()
data = response.json() data = response.json()
save_json(data=data, dataset=dataset) save_json(data=data, dataset=dataset)
request_dataset("deconfinement-pistes-cyclables-temporaires")