improved filtering for Colin

rowers/dataprep.py

@@ -483,6 +483,9 @@ def df_resample(datadf):
 def clean_df_stats(datadf, workstrokesonly=True, ignorehr=True,
                    ignoreadvanced=False):
     # clean data remove zeros and negative values
+    before = len(datadf)
+
+    data_orig = datadf.copy()
 
     # bring metrics which have negative values to positive domain
     if len(datadf)==0:
@@ -706,6 +709,12 @@ def clean_df_stats(datadf, workstrokesonly=True, ignorehr=True,
         except:
             pass
 
+    after = len(datadf.dropna())
+    ratio = float(after)/float(before)
+
+    if ratio < 0.001 or after < 2:
+        return data_orig
+
     return datadf
 
 def getstatsfields():