sander/rowsandall
Private
Public Access
1
0
Fork 0
Code Activity

using dask

Browse Source
This commit is contained in:
Sander Roosendaal
2019-10-22 21:40:35 +02:00
parent 7f2c68a903
commit 622ae44ea6
1 changed files with 9 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
rowers/dataprep.py
View File

@@ -1779,11 +1779,15 @@ def getsmallrowdata_db(columns, ids=[], doclean=True,workstrokesonly=True):
data = [] data = []
columns = [c for c in columns if c != 'None'] columns = [c for c in columns if c != 'None']
if len(ids)>1:
for f in csvfilenames: for f in csvfilenames:
df = dd.read_parquet(f,columns=columns,engine='pyarrow') df = dd.read_parquet(f,columns=columns,engine='pyarrow')
data.append(df) data.append(df)
df = dd.concat(data,axis=0) df = dd.concat(data,axis=0)
else:
df = dd.read_parquet(csvfilenames[0],columns=columns,engine='pyarrow')
data = df.compute() data = df.compute()
data = data.loc[:,~data.columns.duplicated()] data = data.loc[:,~data.columns.duplicated()]
@@ -2443,7 +2447,7 @@ def dataprep(rowdatadf, id=0, bands=True, barchart=True, otwpower=True,
data['workoutid'] = id data['workoutid'] = id
filename = 'media/strokedata_{id}.parquet'.format(id=id) filename = 'media/strokedata_{id}.parquet'.format(id=id)
# df = dd.from_pandas(data,npartitions=1) # df = dd.from_pandas(data,npartitions=1)
data.to_parquet(filename,engine='pyarrow') data.to_parquet(filename,engine='pyarrow',compression='gzip')
return data return data