From eccb20b82fb9d4415fd6507d077363c328c3a683 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Wed, 10 Apr 2024 22:17:07 +0200 Subject: [PATCH] fixed dataplep --- rowers/dataroutines.py | 33 ++++++++++++++++---------- rowers/tests/testdata/testdata.tcx.gz | Bin 4002 -> 4002 bytes 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/rowers/dataroutines.py b/rowers/dataroutines.py index 38f2ccce..aa562ba2 100644 --- a/rowers/dataroutines.py +++ b/rowers/dataroutines.py @@ -569,6 +569,7 @@ def clean_df_stats(datadf, workstrokesonly=True, ignorehr=True, except KeyError: pass + if not ignoreadvanced: try: mask = datadf['rhythm'] < 0 @@ -1418,7 +1419,7 @@ def getrowdata_pl(id=0, doclean=False, convertnewtons=True, checkefficiency=True, for_chart=False): data = read_df_sql(id,polars=True) try: - data = data.with_columns(pl.col(data['time'].diff()).alias("deltat")) # data['time'].diff() + data = data.with_columns((pl.col('time').diff()).alias("deltat")) # data['time'].diff() except KeyError: # pragma: no cover data = pl.DataFrame() @@ -1781,7 +1782,7 @@ def read_df_sql(id, polars=False): if polars: try: f = 'media/strokedata_{id}.parquet.gz'.format(id=id) - df = pd.read_parquet(f) + df = pl.read_parquet(f) except (IsADirectoryError, FileNotFoundError, OSError, ArrowInvalid, IndexError): # pragma: no cover rowdata, row = getrowdata(id=id) try: @@ -2018,11 +2019,15 @@ def dataplep(rowdatadf, id=0, inboard=0.88, forceunit='lbs', bands=True, barchar if 'driveenergy' not in df.columns: if forceunit == 'lbs': df = df.with_columns((pl.col(" DriveLength (meters)") * pl.col(" AverageDriveForce (lbs)") * lbstoN).alias("driveenergy")) - df = df.with_columns((pl.col(" AverageDriveForce (lbs)") * lbstoN).alias(" AverageDriveForce (lbs)")) - df = df.with_columns((pl.col(" PeakDriveForce (lbs)") * lbstoN).alias(" PeakDriveForce (lbs)")) else: df = df.with_columns((pl.col(" DriveLength (meters)") * pl.col(" AverageDriveForce (lbs)")).alias("driveenergy")) + + if forceunit == 'lbs': + df = df.with_columns((pl.col(" AverageDriveForce (lbs)") * lbstoN).alias(" AverageDriveForce (lbs)")) + df = df.with_columns((pl.col(" PeakDriveForce (lbs)") * lbstoN).alias(" PeakDriveForce (lbs)")) + + if df["driveenergy"].mean() == 0 and df["driveenergy"].std() == 0: df = df.with_columns((0.0*pl.col("driveenergy")+100).alias("driveenergy")) @@ -2067,14 +2072,14 @@ def dataplep(rowdatadf, id=0, inboard=0.88, forceunit='lbs', bands=True, barchar if 'wash' not in df.columns: data = data.with_columns( - wash = pl.lit("0"), - catch = pl.lit("0"), - peakforceangle = pl.lit("0"), - finish = pl.lit("0"), - slip = pl.lit("0"), - totalangle = pl.lit("0"), - effectiveangle = pl.lit("0"), - efficiency = pl.lit("0"), + wash = pl.lit(0.0), + catch = pl.lit(0.0), + peakforceangle = pl.lit(0.0), + finish = pl.lit(0.0), + slip = pl.lit(0.0), + totalangle = pl.lit(0.0), + effectiveangle = pl.lit(0.0), + efficiency = pl.lit(0.0), ) else: wash = df['wash'] @@ -2145,6 +2150,10 @@ def dataplep(rowdatadf, id=0, inboard=0.88, forceunit='lbs', bands=True, barchar data = data.with_columns( workoutid = pl.lit(id) ) + # cast data + for k, v in dtypes.items(): + if v == 'int': + data = data.cast({k: pl.Int64}) filename = 'media/strokedata_{id}.parquet.gz'.format(id=id) try: data.write_parquet(filename, compression='gzip') diff --git a/rowers/tests/testdata/testdata.tcx.gz b/rowers/tests/testdata/testdata.tcx.gz index 62b2f5931143a4b2872cff35338cfaf73f70a23d..4623d9967757276973f7be061272a75a8eb56885 100644 GIT binary patch delta 3803 zcmV<14kYoSAEF-yABzYGb@7n~D-1S3(k{c`i9 z>$_jOk%d(jouukyPnYSa?doWM-e-~UB!64ipYHcx-dqa*+NA~o-S^P_ zZ#+DCbZ~I|?g6Bqkw3oQ@cY3}*H@eVtHXcxFPEFc{g>(Mr)MVw=)M5?g5U-6k;Nl& zCOYB>e*-uII9{Zk{@DLncYW|v-);IAtIOqAhkthL&4tpTFRuFTA|LkC^7LwbwmlK{ z$BR$4z6d|y$!h-wJztZX0V99x`sKIRm%CT9|NT1slAFy>*WWI?e*436^L({_)i3|u z-}({hG(UG2ua+lh(?#}wzQ4laj!WG7`g!+ub@}S@|8xW5?zn+FE^+JYr|aFT+W&sC zdn=ZS?9GOXD83Q^{eh8QT%^k>+QX})Bk>O z_q#3ESF3e@dvpFGy#s&YUEJ;V;gKJ`5@PJ|YqC4wfdp<-ApZ%4J&C&!eDHwbj_14E z;b!#Oesb9tKm78SXM#J&@Q=@!&iSvd?~~ZX-F=JsMYs#EmO6t; z(k@m(9ogVCu;&L|5qE!b6>(1x5nNDkr=&L(SISAon;M z?#@=kJtld#;x*i3NqJ|O4R>D=cT2l*-YK}-V5lH>d;oW>ihF>HxJ%-mgi52$d*dd; zy)JnVToHF~f(NJ3=bbBQ#xc!@yRV14fbjvXM&7*`nF#mF#`%AW>^NZ)!A1@DASLbh zImvsdh`Tcm2sCRWBE?RkVb-~65z3P12au<_$Wt{_JRLzY^U3%x z>ACasndb#6BF}%uG74+R7xvx78Ih-o$UB2h=fpwhOV0Ba+Gsxt`CuZ6)r9BJhm!ug z2a#{1{UCWl;|RQJv=3#;&nH5MQjt|h)(9Y~G`tu~+VgWFZ$m}olb+6ms^lw(C8Knf zW;EJkMdYIi2;QpF?i%?SkY_S3VpNgO$+-o|&&Y3I(`bLsV%UM#%=2W+lD9J=Us1${ zg8=|98u9`qqx1G5etT|K-4OXCzt*8D4T~g}^x@5ld__Jy%Z3rHCQ3`bXoh_wKN5Lo zz$(cHn2gNVHqTcSrM1EM^eU?C`3S~04R7l6yr4-Uud?SEplHNqMx%XAGB4JoXA7hu zk2o3KzD0j&hsitR1%oC^%Y;Sk`8koV$ezy@8e7#o&x|Hcggmx+{%-P|b8i`tG@hLF zqLj?Kwyyc@MqmXw7BZ^Aqqb=&aZ8Cp94*6Z~t*Ql@En89fAoAFJcthlK z?yXprZg0z0RNByFeiZW7q@g})bbCv#sLY3E)r~^l8;a;O-doGAXuf@Dqx~4<8FnqP zG`hXDuBdJwT6}&K@?;!CR5ihqvgGGfQRyloFUAV9YMysRYiVaiUMeCVOyaJ}z_ev6 zDg}R<%#T6dqDdO3BUQ`OqP=cqMpe{eDGso?d9fxW-Cia0wrtjQY_+tbkoQJd z7qJm3OTPK=Mj&q!Y=hpavaZE7^7kcgDw!5A`Ctf9)j2_8*%}+)bo(*LBbbQ98t<(GY`M47mw99(fk+djbvPaJ+&nphy+8K}+10bp{=Pq)K(oTQ0XKzGwoe(`V^7kcQk#29nuosml$D=K&;XNo1uW5M0 zI7$G&sP+mx4~Fs)a2y&^tRZEkgv^$ z7wRD|m}_ZOYg%|IT0cK0nXgI16ElH zQIuAq0VYt5_Q6d?w{PCt0rGMLXuQ&g=VQ@T$Y&+xJMJ~FIjo2A=Kdp&;G{Iq1$$a5#EZh3t zjzQj%aiXiPiiNYW=Cm$tMN!%$zrsr0-npVy-JHl*`I}{LSK<9M`BL6 zhoC}F(9qwr|7kVjy~~|%UDS>D2#iHj?jM34Dnl<7p_kk+1#|Uv~D?-m{t|w5<^$1}y=xZDBYm$0r7?4zRJ!08% zn>nH9iqOZb;YXd+mtD26p&ti*?u6=Cm6?TU#%db=4A4s+@Kt{yTSTFv(fT>5c@-IV z91M93Dh(f9^XA>xxC`~5Ph&T@+#gSa-WTnI`=E?WXvWC zb5gVGhtYdOL4t0;_fWL39-Eps26{{atSq=g3OJdLxY*J%2hp=Zr|X)96pJogI;#Pn zDnjoJSkh(WGVo;9xM_3!F#2GkD64N6ZBds_o9jnGPiTK45vUThn5MI)S@Yo!qqioo zvigSEO{QsD%hDM}?~S*@Dh;1)*>!HH)!dDNK4(}8>gwxR%C=LXrsj=;o{a!Toz%N9 znE@YLg4Q5{M z@U0jSW#NC7MVL&yzP4oSih{N_X>3R_s?Za)xqkZS1x!>fT#S~fXymT>@W(;VCORY? zddrhp-mE!thtcm^CVG{cXT>&`&h#C*yOxRQb#uLw$s}mCw9EnYVb?MdbW7f#_0o*H zsoV7-w@kDmsB+}m(5i3li#}9@KI9Hsx~>{FmJNU3a2xQ)K#ypc#i^uTpzLz>JR>y^ zDnf5}trb*io)m7QG^hG{TM>G1$SQ-j!cf$kXWM{326_U6#K*DniY*7ur z1#N%FL2r#iL{(P}iMDz3rtipQZ#?Uc?0NUZkY&(QD@gjYH67>K+h(C z(wpa@>~hA?dg+XU-evSrW!HP;qBVEWjJr|LN8?eMhwrIu2?n&F?HK5hazhoR)N|Pq z3}`{yQP4|ns6slaw^B562Tjcz1${6q7<5uEu4t-8Q}aeaPsz9=WorfRTyx}3-KBpM zbIU}~u4(JNpUhlOEoeK4J{MUtsKPfs_-5D70DUx)tUI05M?V?6o|>9Bgr3QWwYu<4 zfY9vv8K9S3cm+E2SQLF;DyR+kBcKlk7lC^G~yGUh&w$SGKQPBHbeWOIrelmZnTei7= z5Pe?BnvqoAyl8#V>Kkf(^F~3>yG6`Z#$A-yR=1qEH!q`ix$sJf-o>I)-x{TkgFfcM zE1g{*1GMI86ZhsN^t+X;m65v`v316t0eWlFOs`r1g#cQQ-20;UW1)Yy^^Xn5i)Y>S z&4=%=`sMcIvy1-A?=F^?%XNQ$@cC}j=LcPX0B`+o(fL!?ot^y(pPlz#9xwI>Jldb> zkKdR--+Wr_zdwGwNgS^>@Amz^OK;ES>dm%WpLG`(yQh^tuGZ&gUEihicKf$0QF(vd z@i(vfmsguxugN#p*B7hPu1`Px=Eb-<_a`Sy-+lXPb#Zp{bAL|nV>*BI*28(X<$3p? zbjj>2xc&Ll-B#h delta 3803 zcmV<14kYoSAEF-yABzYGoad1SD-5{c`i9 z>$~5&k%d(jpFRC~cZx4o>uukyPnYSa?doWM-lvi9B!8moPxt#TZ!U#@?NWn)?tAF| zHy)lmIyg9f_W;t*$RFQt`2FDL>#I%w)!{$;m&?uJ{>$|B_h%;r=)M5?g5Wvwk;Nl& zCOYB>e*-uII9{Zk{@DL%cYW}4-);KmtIOqAhkthL&4tpT&#(IKA|Lki^7LwbwmlK{ z$BR$4z6d|y$!h-wJzJBT0V99v`sKIRm%CT9|NT1slAF!nufJV({r1P@=Gkifs$c%Q zzx5;1X@2Q0UM)}1ri<+Ve1C<-9hbQE^|S8n>hjg)|LF$A-EjkVT;kT(->-MCYXAGm z?yXoRvNv09KghFXdcY@y`7!SQzD04fIzLahw(MVCot-@E)~~vYMDc%vt+)5?PXGJC z-S4(sU#-^t?ald%^bY)mcX7Adhev+&N{F$;ugUI!2NJkVf&3>F_9X5`@WBIyJD%@u zhnvxB`^jaW|M=@)o(b+8!#_S_I_JN-zE5Hkj~DyDKECE6UG(FhclRyk7vV0vTIviY zNxN7Db!3Cnz@8s;McjYMRm43(L~udDos!;EkbAg4cdAR?tyIL_8Ddn8@@a_Rg52Y5 zxI0@B_n73}iq~+DCFPxAHr#zh+%4_Kd8goRgQ0@l@d4biD((R);x37M5-N>0?~R)X z_qyaga7Em`2_BqApLedL8OJmq?!F%G0>%fl8hQ6(WFp)v8|Qy3vg3qJ1RFKngOs%2 z=Opi;BJR#OAkgS_QHtvG^O1LiiWEDEhFRyPMJP+2A3&b!B2U#!@pJ^qloOpPhbSd| zH*Du4^DGsSwA4>Y~ z9z?#4_JiaJjU(`?(LR(VKc5I4N<~&3StEd`((qy|Y0uAzybTqRPkK5Ns*qG%S)>(uX%I@)h~;EE`6&nkX&#q8av${7B@T z0jne*U@|ga+dN-Ul-360)2pbm=OY;3G`y+L^MWRcyvm+ufT9tb8IATe$-G#To-L4u zJmO?@`xbwt9VYLL7Yv#xEfW^C=jTMeB6~htXlzyUJTsa&5%SpP`Mb$;&b?(o(s**x zi&8S{+PdbqABDUaUt%y7u?{raG8)M0woH>!Nr^beJ?yd8Pn$cbm zCL{CBo*yP34F!lnMLss&e)bMxd3l6N`l zTGa{B5f;_$=S04y(LNeL44U}7V=7uj?OS|)4DvbmHek^7y>*s~=HAYUd`0#=Swnvg zSvAiKO(yr2TmAeXd2i4nsYZK8MeX?+k#|%P`P?~ySkvF$c`iE7&xw3Rx_#2tIHGFR zG8ec-X=gxQ41lP*oV&;^N<010p1l##bwc#e$lsTIMY_EO!(LRL9FMl7hWDU6yr$s| zljq$1ROP+($r6k$19S4^#dzyb6RUsoNjE4Ms}r1&Jzr6L-UeflQInJ7)7xSvLcTU1 zUZ{t>V6LTAt!d$REruysL=3 zHwXk8t1e)ejLbLk!{noJX|UIf_JKlCdwx!%eW-^#lCc(}ChIzIY%+gO^06ZFVjw6- zds}p+!dZE8DjMz48!w{jmJ(di)k){%z4cTNd4k*5Z*CT|ez!f#_sCo0oK%bQLh`O>Hr0n=_Qa^e0Vw~5ar?Rcm z<|Oq{5&B?6NY#G;8!1_YzM-GCOULKVnAVA?j5|sSf=r}+4z^|K2hm%@Ue`V+FN`sn z&PmO#A4cyD1qr$V-$T*DdTeUm80awxu(IF|Dd1!};$ln797N9sovv#ZQY^Y~>8u8P zstCO^U`dye%fORaAGszST=ur!)?GH13jW)7N?SWfwIfh^NiFy zs0h8?wN_B6c~ZEI(wyq+ZAIw4A*&493PVwEo^1pE80ZNM5+ke7TbRt+hRvHdjDA;S zO{#c(vIts{^$eum<*pi0hu%?9!mXadY92hm0j6!gKcV9-guxT2{VP0bqxJtgCgl&uxKbIp-Eb(eom z%q@F`dnnqpbFpk;G11P1N6~IvhH+JAN^$PdTMIk5PBvf*6PAH z0YbCuXMkRE;T7o6V^Q>Zsh~FCkAOZHVAS=)4>@^%BIv1k^M=spm8=1S4n4=opl|Ef z52I%jgmjU+^rDp1@N*}MhJFO}&Uk;eQRUJJ4okXpW`$lVLLc**wxE-Gk3|Q3-%2pX zKrdvR#h}X43884E?Ss&_n!922lB;imDkC?P~1A3JsH=<0&r30;R-XQwjO4hn{?jn^9+CrP_M?vp%^^Foe`^kT-ZrSGg zLG*bgYerIe^P=@dt8b|F%^L+h?-nsv8Fx`)TitTv-n@+7<-#i^dKZgIeQT6D4*Hl2 zuXJ{O4A7dVP28K8(C=2VRz~h(#MT*m2I#FxGrej76ar{Ha_@`YkA?o-);~5JFP?VS zHy?hu>X+M-&o25ezq?poF4uqk!RNb8pC5Gn0lf7uURaP9r)SF-@Y#9)bANKO^xe0wRu^YCKlhjPKBj*|Z#|rMTb_0Q zNtevdg4>@z-EFl_o4vf>Z@1y*(SCIPb^NqjpMU@4 zx=!9-#w<2M?2K3b$Sku6Uyc0NqQQ; RrAwbY`XAY=VbH{Y0RXz*z1{!-