OPENNLP-1122: Leipzig sample should allow skip initial entries

Author: wcolen

opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/LeipzigLanguageSampleStreamFactory.java

@@ -21,6 +21,7 @@
 import java.io.IOException;
 
 import opennlp.tools.cmdline.ArgumentParser;
+import opennlp.tools.cmdline.ArgumentParser.OptionalParameter;
 import opennlp.tools.cmdline.ArgumentParser.ParameterDescription;
 import opennlp.tools.cmdline.StreamFactoryRegistry;
 import opennlp.tools.cmdline.TerminateToolException;
@@ -47,6 +48,11 @@ interface Parameters extends EncodingParameter {
     @ParameterDescription(valueName = "samplesPerLanguage",
         description = "number of samples per language")
     String getSamplesPerLanguage();
+
+    @ParameterDescription(valueName = "samplesToSkip",
+        description = "number of samples to skip before returning")
+    @OptionalParameter(defaultValue = "0")
+    String getSamplesToSkip();
   }
 
   protected <P> LeipzigLanguageSampleStreamFactory(Class<P> params) {
@@ -64,9 +70,11 @@ public ObjectStream<LanguageSample> create(String[] args) {
     File sentencesFileDir = params.getSentencesDir();
 
     try {
-      return new SampleShuffleStream(new LeipzigLanguageSampleStream(sentencesFileDir,
+      return new SampleSkipStream(new SampleShuffleStream(
+          new LeipzigLanguageSampleStream(sentencesFileDir,
           Integer.parseInt(params.getSentencesPerSample()),
-          Integer.parseInt(params.getSamplesPerLanguage())));
+          Integer.parseInt(params.getSamplesPerLanguage()) + Integer.parseInt(params.getSamplesToSkip()))),
+          Integer.parseInt(params.getSamplesToSkip()));
     } catch (IOException e) {
       throw new TerminateToolException(-1, "IO error while opening sample data.", e);
     }

opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/SampleSkipStream.java

@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.leipzig;
+
+import java.io.IOException;
+
+import opennlp.tools.util.ObjectStream;
+
+class SampleSkipStream<T> implements ObjectStream<T> {
+
+
+  private final ObjectStream<T> samples;
+  private final int samplesToSkip;
+
+  SampleSkipStream(ObjectStream<T> samples, int samplesToSkip) throws IOException {
+    this.samples = samples;
+    this.samplesToSkip = samplesToSkip;
+
+    skipSamples();
+  }
+
+  @Override
+  public T read() throws IOException {
+    return samples.read();
+  }
+
+  @Override
+  public void reset() throws IOException, UnsupportedOperationException {
+    this.samples.reset();
+    skipSamples();
+  }
+
+  private void skipSamples() throws IOException {
+    int i = 0;
+
+    while (i < samplesToSkip && (samples.read()) != null) {
+      i++;
+    }
+  }
+}