passing tests

rowers/dataroutines.py

@@ -32,6 +32,7 @@ import zipfile
 import os
 from rowers.models import strokedatafields
 import polars as pl
+import polars.selectors as cs
 from polars.exceptions import (
     ColumnNotFoundError, SchemaError, ComputeError,
     InvalidOperationError
@@ -176,6 +177,19 @@ columndict = {
     'cumdist': 'cum_dist',
 }
 
+def remove_nulls_pl(data):
+    data = data.lazy().fill_nan(None)
+    data = data.select(cs.by_dtype(pl.NUMERIC_DTYPES)).collect()
+    data = data[[s.name for s in data if not s.is_infinite().sum()]]
+    data = data[[s.name for s in data if not (s.null_count() == data.height)]]
+    if not data.is_empty():
+        try:
+            data = data.drop_nulls()
+        except:
+            pass
+
+    return data
+
 
 def get_video_data(w, groups=['basic'], mode='water'):
     modes = [mode, 'both', 'basic']
@@ -798,14 +812,11 @@ def clean_df_stats_pl(datadf, workstrokesonly=True, ignorehr=True,
     
     datadf = datadf.filter(
         pl.col("spm") >=0,
-        pl.col("efficiency")<=200,
         pl.col("spm")>=10,
         pl.col("pace")<=300*1000.,
-        pl.col("efficiency")>=0,
         pl.col("pace")>=60*1000,
         pl.col("power")<=5000,
         pl.col("spm")<=120,
-        pl.col("wash")>=1
     )
 
 
@@ -823,7 +834,10 @@ def clean_df_stats_pl(datadf, workstrokesonly=True, ignorehr=True,
         datadf = datadf.filter(pl.col("rhythm")>=0,
                                pl.col("rhythm")<=70,
                                pl.col("power")>=20,
+                               pl.col("efficiency")<=200,
                                pl.col("drivelength")>=0.5,
+                               pl.col("wash")>=1,
+                               pl.col("efficiency")>=0,
                                pl.col("forceratio")>=0.2,
                                pl.col("forceratio")<=1.0,
                                pl.col("drivespeed")>=0.5,
@@ -833,6 +847,7 @@ def clean_df_stats_pl(datadf, workstrokesonly=True, ignorehr=True,
                                pl.col("catch")<=-30)
 
 
+
     # workoutstateswork = [1, 4, 5, 8, 9, 6, 7]
     workoutstatesrest = [3]
     # workoutstatetransition = [0, 2, 10, 11, 12, 13]
@@ -850,6 +865,8 @@ def clean_df_stats_pl(datadf, workstrokesonly=True, ignorehr=True,
         if ratio < 0.01 or after[workoutid] < 2:
             return data_orig
 
+
+
     return datadf
 
 
@@ -1541,9 +1558,8 @@ def getsmallrowdata_pl(columns, ids=[], doclean=True, workstrokesonly=True, comp
                                      workstrokesonly=workstrokesonly,
                                      for_chart=for_chart)
 
-        data = data.fill_nan(None).drop_nulls()
 
-        return data
+        data = remove_nulls_pl(data)
 
     if not df.is_empty():
         df = df.fill_nan(None).drop_nulls()