Skip to content

Commit 7099571

Browse files
foster33foster33
authored
Merge pull request #39 from NationalSecurityAgency/bugfix/issue-37
Ensure Last Updated dates for Index Only fields are correct
2 parents 05c02c3 + b668f5f commit 7099571

File tree

2 files changed

+69
-6
lines changed

2 files changed

+69
-6
lines changed

‎service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -196,9 +196,8 @@ private void transformEntry(Map.Entry<Key,Value> currEntry) throws IOException,
196196
if (currField.getFieldName() == null) {
197197
setFieldNameAndAlias();
198198
}
199-
200199
// Determine the lastUpdated value for index-only fields without including timestamps from description rows.
201-
if (currField.isIndexOnly() && currField.getLastUpdated() == null && !isColumnFamly(ColumnFamilyConstants.COLF_DESC)) {
200+
if (currField.isIndexOnly() && !isColumnFamly(ColumnFamilyConstants.COLF_DESC)) {
202201
setLastUpdated();
203202
}
204203
}
@@ -277,8 +276,15 @@ private void setType() {
277276

278277
// Set the last updated date for the current {@link DefaultMetadataField} based on the timestamp of the current entry.
279278
private void setLastUpdated() {
280-
currField.setLastUpdated(Instant.ofEpochMilli(currKey.getTimestamp()).atZone(ZoneId.systemDefault()).toLocalDateTime()
281-
.format(DateTimeFormatter.ofPattern(TIMESTAMP_FORMAT)));
279+
String formattedCurrentKeyTimeStamp = Instant.ofEpochMilli(currKey.getTimestamp()).atZone(ZoneId.systemDefault()).toLocalDateTime()
280+
.format(DateTimeFormatter.ofPattern(TIMESTAMP_FORMAT));
281+
if (currField.getLastUpdated() != null) {
282+
if (Long.parseLong(currField.getLastUpdated()) < Long.parseLong(formattedCurrentKeyTimeStamp)) {
283+
currField.setLastUpdated(formattedCurrentKeyTimeStamp);
284+
}
285+
} else {
286+
currField.setLastUpdated(formattedCurrentKeyTimeStamp);
287+
}
282288
}
283289
}
284290
}

‎service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java‎

Lines changed: 59 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,11 @@
22

33
import static org.assertj.core.api.Assertions.assertThat;
44

5+
import java.time.Instant;
56
import java.time.LocalDateTime;
67
import java.time.ZoneId;
78
import java.time.ZonedDateTime;
9+
import java.time.format.DateTimeFormatter;
810
import java.util.Collection;
911
import java.util.Collections;
1012
import java.util.HashMap;
@@ -42,6 +44,7 @@ public class DefaultMetadataFieldScannerTest {
4244

4345
private static final String DATE = "20200115051230";
4446
private static final long TIMESTAMP = ZonedDateTime.of(LocalDateTime.of(2020, 1, 15, 5, 12, 30), ZoneId.systemDefault()).toInstant().toEpochMilli();
47+
private static final long DAY_AS_MILLISECONDS = 86400000;
4548
private static final String MODEL_TABLE = "modelTable";
4649
private static final String METADATA_TABLE = "metadataTable";
4750
private static final String[] AUTH = {"PRIVATE"};
@@ -66,6 +69,8 @@ public DefaultFields getFields() {
6669

6770
private AccumuloClient connector;
6871

72+
private Map<String,String> expectedTimestamps;
73+
6974
private DefaultMetadataFieldScanner scanner;
7075

7176
@BeforeEach
@@ -75,6 +80,9 @@ public void setUp() throws Exception {
7580
connector.securityOperations().changeUserAuthorizations("root", new Authorizations(AUTH));
7681
connector.tableOperations().create(METADATA_TABLE);
7782
connector.tableOperations().create(MODEL_TABLE);
83+
84+
expectedTimestamps = new HashMap<>();
85+
7886
populateMetadataTable();
7987

8088
Map<String,String> normalizerMapping = new HashMap<>();
@@ -117,8 +125,18 @@ public void whenRetrievingFields_givenNoDataTypeFilters_shouldReturnUnfilteredRe
117125
name.setTypes(Collections.singletonList("Unknown"));
118126
name.setLastUpdated(DATE);
119127

128+
DefaultMetadataField fooToken = new DefaultMetadataField();
129+
fooToken.setFieldName("FOO_TOKEN");
130+
fooToken.setDataType("tvmaze");
131+
fooToken.setForwardIndexed(true);
132+
fooToken.setReverseIndexed(false);
133+
fooToken.setTokenized(true);
134+
fooToken.setIndexOnly(true);
135+
fooToken.setLastUpdated("20200120051230");
136+
fooToken.setTypes(Collections.singletonList("Text"));
137+
120138
Collection<DefaultMetadataField> fields = scanner.getFields(Collections.emptyMap(), Collections.emptySet());
121-
assertThat(fields).containsExactlyInAnyOrder(barField, contributorId, name);
139+
assertThat(fields).containsExactlyInAnyOrder(barField, contributorId, name, fooToken);
122140
}
123141

124142
@Test
@@ -178,11 +196,34 @@ public void whenRetrievingFields_givenAliases_shouldReturnResultsWithAliases() t
178196
name.setTypes(Collections.singletonList("Unknown"));
179197
name.setLastUpdated(DATE);
180198

199+
DefaultMetadataField fooToken = new DefaultMetadataField();
200+
fooToken.setFieldName("FOO_TOKEN");
201+
fooToken.setDataType("tvmaze");
202+
fooToken.setForwardIndexed(true);
203+
fooToken.setReverseIndexed(false);
204+
fooToken.setTokenized(true);
205+
fooToken.setIndexOnly(true);
206+
fooToken.setLastUpdated("20200120051230");
207+
fooToken.setTypes(Collections.singletonList("Text"));
208+
181209
Map<String,String> aliases = new HashMap<>();
182210
aliases.put("BAR_FIELD", "bar_field_alias");
183211
aliases.put("CONTRIBUTOR_ID", "contributor_id_alias");
184212
Collection<DefaultMetadataField> fields = scanner.getFields(aliases, Collections.emptySet());
185-
assertThat(fields).containsExactlyInAnyOrder(barField, contributorId, name);
213+
assertThat(fields).containsExactlyInAnyOrder(barField, contributorId, name, fooToken);
214+
}
215+
216+
@Test
217+
public void lastUpdatedTimeIsCorrect() throws Exception {
218+
Collection<DefaultMetadataField> fields = scanner.getFields(Collections.emptyMap(), Collections.emptySet());
219+
220+
for (DefaultMetadataField field : fields) {
221+
assertThat(expectedTimestamps).containsEntry(field.getFieldName(), field.getLastUpdated());
222+
}
223+
}
224+
225+
private String formatTimestamp(Long timestamp) {
226+
return Instant.ofEpochMilli(timestamp).atZone(ZoneId.systemDefault()).toLocalDateTime().format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss"));
186227
}
187228

188229
private void populateMetadataTable() throws TableNotFoundException, MutationsRejectedException {
@@ -193,25 +234,41 @@ private void populateMetadataTable() throws TableNotFoundException, MutationsRej
193234
barField.put(new Text(ColumnFamilyConstants.COLF_TF), new Text("csv"), TIMESTAMP, new Value());
194235
barField.put(new Text(ColumnFamilyConstants.COLF_T), new Text("csv\0datawave.data.type.LcNoDiacriticsType"), TIMESTAMP, new Value());
195236
barField.put(new Text(ColumnFamilyConstants.COLF_DESC), new Text("csv"), new ColumnVisibility("PRIVATE"), TIMESTAMP, new Value("Barfield Description"));
237+
expectedTimestamps.put("BAR_FIELD", formatTimestamp(TIMESTAMP));
196238

197239
Mutation contributorId = new Mutation(new Text("CONTRIBUTOR_ID"));
198240
contributorId.put(new Text(ColumnFamilyConstants.COLF_E), new Text("enwiki"), TIMESTAMP, new Value());
199241
contributorId.put(new Text(ColumnFamilyConstants.COLF_I), new Text("enwiki"), TIMESTAMP, new Value());
200242
contributorId.put(new Text(ColumnFamilyConstants.COLF_T), new Text("enwiki\0datawave.data.type.NumberType"), TIMESTAMP, new Value());
201243
contributorId.put(new Text(ColumnFamilyConstants.COLF_DESC), new Text("enwiki"), new ColumnVisibility("PRIVATE"), TIMESTAMP,
202244
new Value("ContributorId Description"));
245+
expectedTimestamps.put("CONTRIBUTOR_ID", formatTimestamp(TIMESTAMP));
203246

204247
Mutation name = new Mutation(new Text("NAME"));
205248
name.put(new Text(ColumnFamilyConstants.COLF_E), new Text("tvmaze"), TIMESTAMP, new Value());
206249
name.put(new Text(ColumnFamilyConstants.COLF_I), new Text("tvmaze"), TIMESTAMP, new Value());
207250
name.put(new Text(ColumnFamilyConstants.COLF_RI), new Text("tvmaze"), TIMESTAMP, new Value());
208251
name.put(new Text(ColumnFamilyConstants.COLF_T), new Text("tvmaze\0not.a.known.type"), TIMESTAMP, new Value());
252+
expectedTimestamps.put("NAME", formatTimestamp(TIMESTAMP));
253+
254+
Mutation fooToken = new Mutation(new Text("FOO_TOKEN"));
255+
fooToken.put(new Text(ColumnFamilyConstants.COLF_I), new Text("tvmaze"), TIMESTAMP, new Value());
256+
fooToken.put(new Text(ColumnFamilyConstants.COLF_I), new Text("tvmaze"), TIMESTAMP + (DAY_AS_MILLISECONDS * 2), new Value());
257+
fooToken.put(new Text(ColumnFamilyConstants.COLF_I), new Text("tvmaze"), TIMESTAMP + (DAY_AS_MILLISECONDS * 3), new Value());
258+
fooToken.put(new Text(ColumnFamilyConstants.COLF_I), new Text("tvmaze"), TIMESTAMP - (DAY_AS_MILLISECONDS * 2), new Value());
259+
fooToken.put(new Text(ColumnFamilyConstants.COLF_I), new Text("tvmaze"), TIMESTAMP - (DAY_AS_MILLISECONDS * 3), new Value());
260+
fooToken.put(new Text(ColumnFamilyConstants.COLF_I), new Text("tvmaze"), TIMESTAMP + (DAY_AS_MILLISECONDS * 4), new Value());
261+
fooToken.put(new Text(ColumnFamilyConstants.COLF_I), new Text("tvmaze"), TIMESTAMP + (DAY_AS_MILLISECONDS * 5), new Value());
262+
fooToken.put(new Text(ColumnFamilyConstants.COLF_TF), new Text("tvmaze"), TIMESTAMP, new Value());
263+
fooToken.put(new Text(ColumnFamilyConstants.COLF_T), new Text("tvmaze\0datawave.data.type.LcNoDiacriticsType"), TIMESTAMP, new Value());
264+
expectedTimestamps.put("FOO_TOKEN", formatTimestamp(TIMESTAMP + (DAY_AS_MILLISECONDS * 5)));
209265

210266
BatchWriterConfig bwConfig = new BatchWriterConfig().setMaxMemory(10L).setMaxLatency(1, TimeUnit.SECONDS).setMaxWriteThreads(1);
211267
BatchWriter writer = connector.createBatchWriter(METADATA_TABLE, bwConfig);
212268
writer.addMutation(barField);
213269
writer.addMutation(contributorId);
214270
writer.addMutation(name);
271+
writer.addMutation(fooToken);
215272
writer.flush();
216273
writer.close();
217274
}

0 commit comments

Comments
 (0)