Merge pull request #50 from IBM/fix_na_remover

Fix na remover

Author: ppalmes

src/naremover.jl

@@ -21,7 +21,9 @@ export NARemover
     )
 
 Removes columns with NAs greater than acceptance rate.
-Remove remaining NAs by rows and return the Dataframe.
+This assumes that it processes columns of features. 
+The output column should not be part of input to avoid
+it being excluded if it fails the acceptance critera.
 
 Implements `fit!` and `transform!`.
 """
@@ -55,6 +57,7 @@ function fit!(nad::NARemover, features::DataFrame, labels::Vector=[])
     nad.model = nad.args
 end
 
+
 function transform!(nad::NARemover, nfeatures::DataFrame)
     features = deepcopy(nfeatures) 
     if features == DataFrame()
@@ -69,11 +72,7 @@ function transform!(nad::NARemover, nfeatures::DataFrame)
 	end
     end
     xtr =  features[:,colnames]
-    if xtr != DataFrame()
-	return xtr[completecases(xtr),:]
-    else
-	return DataFrame()
-    end
+    return xtr
 end
 
 end

test/test_naremover.jl

@@ -10,9 +10,9 @@ function test_naremover()
   Random.seed!(123)
   df  = DataFrame(a=rand([1:3...,missing],100),b=rand([1:9...,missing],100),c=rand([1:20...,missing],100))
   nara = NARemover(0.25)
-  @test fit_transform!(nara,df) |> Matrix |> sum == 1425
+  @test fit_transform!(nara,df) |> Matrix |> x->sum(skipmissing(x)) == 1546
   narb = NARemover(0.05)
-  @test fit_transform!(narb,df) |> Matrix |>sum == 1086
+  @test fit_transform!(narb,df) |> Matrix |> x->sum(skipmissing(x)) == 1086
   narc = NARemover(0.01)
   @test fit_transform!(narc,df)  == DataFrame()
 end