coolneng
/
locimend
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Compare commits

base: coolneng:e9582d08833807f6157a3ec658d093cf72411666
Branches Tags
coolneng:master
coolneng:0.1.1
coolneng:0.1.0
..
compare: coolneng:b7049f5b4710874aca0b72ca837b1eb75850cde5
Branches Tags
coolneng:master
coolneng:0.1.1
coolneng:0.1.0

1 Commits
e9582d0883 ... b7049f5b47

Author SHA1 Message Date
coolneng b7049f5b47
Parallelize dataset transformations 2021-06-24 19:30:46 +02:00
1 changed files with 1 additions and 1 deletions
Show Stats Expand all files Collapse all files

2
src/preprocessing.py
View File

@ -105,8 +105,8 @@ def read_dataset(filepath) -> TFRecordDataset:
Read TFRecords files and generate a dataset
"""
data_input = TFRecordDataset(filenames=filepath)
dataset = data_input.map(map_func=process_input, num_parallel_calls=AUTOTUNE)
shuffled_dataset = dataset.shuffle(buffer_size=10000, seed=42)
dataset = data_input.map(map_func=process_input, num_parallel_calls=AUTOTUNE)
batched_dataset = shuffled_dataset.batch(batch_size=BATCH_SIZE).repeat(count=EPOCHS)
return batched_dataset