Skip to content

Commit d334b7b

Browse files
jo-elimujo-elimu
authored
fix: number format exception during data batch processing (#2272)
2 parents 94cac55 + 56f8148 commit d334b7b

File tree

5 files changed

+141
-50
lines changed

5 files changed

+141
-50
lines changed

‎pom-dependency-tree.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
ai.elimu:webapp:war:2.6.66-SNAPSHOT
1+
ai.elimu:webapp:war:2.6.69-SNAPSHOT
22
+- ai.elimu:model:jar:model-2.0.113:compile
33
| \- com.google.code.gson:gson:jar:2.13.1:compile
44
| \- com.google.errorprone:error_prone_annotations:jar:2.38.0:compile

‎src/main/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelper.java

Lines changed: 77 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -83,13 +83,17 @@ public static List<LetterSoundAssessmentEvent> extractLetterSoundAssessmentEvent
8383
if (versionCode >= 3005009) {
8484
// https://github.com/elimu-ai/analytics/releases/tag/3.5.9
8585

86-
int researchExperimentOrdinal = Integer.valueOf(csvRecord.get("research_experiment"));
87-
ResearchExperiment researchExperiment = ResearchExperiment.values()[researchExperimentOrdinal];
88-
letterSoundAssessmentEvent.setResearchExperiment(researchExperiment);
89-
90-
int experimentGroupOrdinal = Integer.valueOf(csvRecord.get("experiment_group"));
91-
ExperimentGroup experimentGroup = ExperimentGroup.values()[experimentGroupOrdinal];
92-
letterSoundAssessmentEvent.setExperimentGroup(experimentGroup);
86+
if (StringUtils.isNotBlank(csvRecord.get("research_experiment"))) {
87+
int researchExperimentOrdinal = Integer.valueOf(csvRecord.get("research_experiment"));
88+
ResearchExperiment researchExperiment = ResearchExperiment.values()[researchExperimentOrdinal];
89+
letterSoundAssessmentEvent.setResearchExperiment(researchExperiment);
90+
}
91+
92+
if (StringUtils.isNotBlank(csvRecord.get("experiment_group"))) {
93+
int experimentGroupOrdinal = Integer.valueOf(csvRecord.get("experiment_group"));
94+
ExperimentGroup experimentGroup = ExperimentGroup.values()[experimentGroupOrdinal];
95+
letterSoundAssessmentEvent.setExperimentGroup(experimentGroup);
96+
}
9397
}
9498

9599
String letterSoundLetters = csvRecord.get("letter_sound_letters");
@@ -156,14 +160,18 @@ public static List<LetterSoundLearningEvent> extractLetterSoundLearningEvents(Fi
156160

157161
if (versionCode >= 3005009) {
158162
// https://github.com/elimu-ai/analytics/releases/tag/3.5.9
159-
160-
int researchExperimentOrdinal = Integer.valueOf(csvRecord.get("research_experiment"));
161-
ResearchExperiment researchExperiment = ResearchExperiment.values()[researchExperimentOrdinal];
162-
letterSoundLearningEvent.setResearchExperiment(researchExperiment);
163163

164-
int experimentGroupOrdinal = Integer.valueOf(csvRecord.get("experiment_group"));
165-
ExperimentGroup experimentGroup = ExperimentGroup.values()[experimentGroupOrdinal];
166-
letterSoundLearningEvent.setExperimentGroup(experimentGroup);
164+
if (StringUtils.isNotBlank(csvRecord.get("research_experiment"))) {
165+
int researchExperimentOrdinal = Integer.valueOf(csvRecord.get("research_experiment"));
166+
ResearchExperiment researchExperiment = ResearchExperiment.values()[researchExperimentOrdinal];
167+
letterSoundLearningEvent.setResearchExperiment(researchExperiment);
168+
}
169+
170+
if (StringUtils.isNotBlank(csvRecord.get("experiment_group"))) {
171+
int experimentGroupOrdinal = Integer.valueOf(csvRecord.get("experiment_group"));
172+
ExperimentGroup experimentGroup = ExperimentGroup.values()[experimentGroupOrdinal];
173+
letterSoundLearningEvent.setExperimentGroup(experimentGroup);
174+
}
167175
}
168176

169177
// TODO: letterSoundLetters
@@ -231,13 +239,17 @@ public static List<NumberLearningEvent> extractNumberLearningEvents(File csvFile
231239
if (versionCode >= 3005009) {
232240
// https://github.com/elimu-ai/analytics/releases/tag/3.5.9
233241

234-
int researchExperimentOrdinal = Integer.valueOf(csvRecord.get("research_experiment"));
235-
ResearchExperiment researchExperiment = ResearchExperiment.values()[researchExperimentOrdinal];
236-
numberLearningEvent.setResearchExperiment(researchExperiment);
237-
238-
int experimentGroupOrdinal = Integer.valueOf(csvRecord.get("experiment_group"));
239-
ExperimentGroup experimentGroup = ExperimentGroup.values()[experimentGroupOrdinal];
240-
numberLearningEvent.setExperimentGroup(experimentGroup);
242+
if (StringUtils.isNotBlank(csvRecord.get("research_experiment"))) {
243+
int researchExperimentOrdinal = Integer.valueOf(csvRecord.get("research_experiment"));
244+
ResearchExperiment researchExperiment = ResearchExperiment.values()[researchExperimentOrdinal];
245+
numberLearningEvent.setResearchExperiment(researchExperiment);
246+
}
247+
248+
if (StringUtils.isNotBlank(csvRecord.get("experiment_group"))) {
249+
int experimentGroupOrdinal = Integer.valueOf(csvRecord.get("experiment_group"));
250+
ExperimentGroup experimentGroup = ExperimentGroup.values()[experimentGroupOrdinal];
251+
numberLearningEvent.setExperimentGroup(experimentGroup);
252+
}
241253
}
242254

243255
Integer numberValue = Integer.valueOf(csvRecord.get("number_value"));
@@ -317,13 +329,17 @@ public static List<WordAssessmentEvent> extractWordAssessmentEvents(File csvFile
317329
if (versionCode >= 3005009) {
318330
// https://github.com/elimu-ai/analytics/releases/tag/3.5.9
319331

320-
int researchExperimentOrdinal = Integer.valueOf(csvRecord.get("research_experiment"));
321-
ResearchExperiment researchExperiment = ResearchExperiment.values()[researchExperimentOrdinal];
322-
wordAssessmentEvent.setResearchExperiment(researchExperiment);
323-
324-
int experimentGroupOrdinal = Integer.valueOf(csvRecord.get("experiment_group"));
325-
ExperimentGroup experimentGroup = ExperimentGroup.values()[experimentGroupOrdinal];
326-
wordAssessmentEvent.setExperimentGroup(experimentGroup);
332+
if (StringUtils.isNotBlank(csvRecord.get("research_experiment"))) {
333+
int researchExperimentOrdinal = Integer.valueOf(csvRecord.get("research_experiment"));
334+
ResearchExperiment researchExperiment = ResearchExperiment.values()[researchExperimentOrdinal];
335+
wordAssessmentEvent.setResearchExperiment(researchExperiment);
336+
}
337+
338+
if (StringUtils.isNotBlank(csvRecord.get("experiment_group"))) {
339+
int experimentGroupOrdinal = Integer.valueOf(csvRecord.get("experiment_group"));
340+
ExperimentGroup experimentGroup = ExperimentGroup.values()[experimentGroupOrdinal];
341+
wordAssessmentEvent.setExperimentGroup(experimentGroup);
342+
}
327343
}
328344

329345
String wordText = csvRecord.get("word_text");
@@ -396,13 +412,17 @@ public static List<WordLearningEvent> extractWordLearningEvents(File csvFile) {
396412
if (versionCode >= 3005009) {
397413
// https://github.com/elimu-ai/analytics/releases/tag/3.5.9
398414

399-
int researchExperimentOrdinal = Integer.valueOf(csvRecord.get("research_experiment"));
400-
ResearchExperiment researchExperiment = ResearchExperiment.values()[researchExperimentOrdinal];
401-
wordLearningEvent.setResearchExperiment(researchExperiment);
402-
403-
int experimentGroupOrdinal = Integer.valueOf(csvRecord.get("experiment_group"));
404-
ExperimentGroup experimentGroup = ExperimentGroup.values()[experimentGroupOrdinal];
405-
wordLearningEvent.setExperimentGroup(experimentGroup);
415+
if (StringUtils.isNotBlank(csvRecord.get("research_experiment"))) {
416+
int researchExperimentOrdinal = Integer.valueOf(csvRecord.get("research_experiment"));
417+
ResearchExperiment researchExperiment = ResearchExperiment.values()[researchExperimentOrdinal];
418+
wordLearningEvent.setResearchExperiment(researchExperiment);
419+
}
420+
421+
if (StringUtils.isNotBlank(csvRecord.get("experiment_group"))) {
422+
int experimentGroupOrdinal = Integer.valueOf(csvRecord.get("experiment_group"));
423+
ExperimentGroup experimentGroup = ExperimentGroup.values()[experimentGroupOrdinal];
424+
wordLearningEvent.setExperimentGroup(experimentGroup);
425+
}
406426
}
407427

408428
String wordText = csvRecord.get("word_text");
@@ -476,13 +496,17 @@ public static List<StoryBookLearningEvent> extractStoryBookLearningEvents(File c
476496
if (versionCode >= 3005009) {
477497
// https://github.com/elimu-ai/analytics/releases/tag/3.5.9
478498

479-
int researchExperimentOrdinal = Integer.valueOf(csvRecord.get("research_experiment"));
480-
ResearchExperiment researchExperiment = ResearchExperiment.values()[researchExperimentOrdinal];
481-
storyBookLearningEvent.setResearchExperiment(researchExperiment);
482-
483-
int experimentGroupOrdinal = Integer.valueOf(csvRecord.get("experiment_group"));
484-
ExperimentGroup experimentGroup = ExperimentGroup.values()[experimentGroupOrdinal];
485-
storyBookLearningEvent.setExperimentGroup(experimentGroup);
499+
if (StringUtils.isNotBlank(csvRecord.get("research_experiment"))) {
500+
int researchExperimentOrdinal = Integer.valueOf(csvRecord.get("research_experiment"));
501+
ResearchExperiment researchExperiment = ResearchExperiment.values()[researchExperimentOrdinal];
502+
storyBookLearningEvent.setResearchExperiment(researchExperiment);
503+
}
504+
505+
if (StringUtils.isNotBlank(csvRecord.get("experiment_group"))) {
506+
int experimentGroupOrdinal = Integer.valueOf(csvRecord.get("experiment_group"));
507+
ExperimentGroup experimentGroup = ExperimentGroup.values()[experimentGroupOrdinal];
508+
storyBookLearningEvent.setExperimentGroup(experimentGroup);
509+
}
486510
}
487511

488512
if (versionCode < 3003000) {
@@ -552,13 +576,17 @@ public static List<VideoLearningEvent> extractVideoLearningEvents(File csvFile)
552576
if (versionCode >= 3005009) {
553577
// https://github.com/elimu-ai/analytics/releases/tag/3.5.9
554578

555-
int researchExperimentOrdinal = Integer.valueOf(csvRecord.get("research_experiment"));
556-
ResearchExperiment researchExperiment = ResearchExperiment.values()[researchExperimentOrdinal];
557-
videoLearningEvent.setResearchExperiment(researchExperiment);
558-
559-
int experimentGroupOrdinal = Integer.valueOf(csvRecord.get("experiment_group"));
560-
ExperimentGroup experimentGroup = ExperimentGroup.values()[experimentGroupOrdinal];
561-
videoLearningEvent.setExperimentGroup(experimentGroup);
579+
if (StringUtils.isNotBlank(csvRecord.get("research_experiment"))) {
580+
int researchExperimentOrdinal = Integer.valueOf(csvRecord.get("research_experiment"));
581+
ResearchExperiment researchExperiment = ResearchExperiment.values()[researchExperimentOrdinal];
582+
videoLearningEvent.setResearchExperiment(researchExperiment);
583+
}
584+
585+
if (StringUtils.isNotBlank(csvRecord.get("experiment_group"))) {
586+
int experimentGroupOrdinal = Integer.valueOf(csvRecord.get("experiment_group"));
587+
ExperimentGroup experimentGroup = ExperimentGroup.values()[experimentGroupOrdinal];
588+
videoLearningEvent.setExperimentGroup(experimentGroup);
589+
}
562590
}
563591

564592
String videoTitle = csvRecord.get("video_title");

‎src/test/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelperTest.java

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,28 @@ public void testExtractLetterSoundLearningEvents_v3005009() throws IOException {
6767
assertEquals(149, letterSoundLearningEvent.getLetterSoundId());
6868
}
6969

70+
/**
71+
* Test extraction of data from CSV files generated by version 3005013 of the Analytics app:
72+
* https://github.com/elimu-ai/analytics/releases/tag/3.5.13
73+
*/
74+
@Test
75+
public void testExtractLetterSoundLearningEvents_v3005013() throws IOException {
76+
ResourceLoader resourceLoader = new ClassRelativeResourceLoader(CsvAnalyticsExtractionHelper.class);
77+
Resource resource = resourceLoader.getResource("7e89c8e7f4c68405_3005013_letter-sound-learning-events_2025-05-14.csv");
78+
File csvFile = resource.getFile();
79+
80+
List<LetterSoundLearningEvent> letterSoundLearningEvents = CsvAnalyticsExtractionHelper.extractLetterSoundLearningEvents(csvFile);
81+
assertEquals(5, letterSoundLearningEvents.size());
82+
83+
LetterSoundLearningEvent letterSoundLearningEvent = letterSoundLearningEvents.get(0);
84+
assertEquals(1747200557 * 1_000L, letterSoundLearningEvent.getTimestamp().getTimeInMillis());
85+
assertEquals("7e89c8e7f4c68405", letterSoundLearningEvent.getAndroidId());
86+
assertEquals("ai.elimu.herufi", letterSoundLearningEvent.getPackageName());
87+
assertNull(letterSoundLearningEvent.getResearchExperiment());
88+
assertNull(letterSoundLearningEvent.getExperimentGroup());
89+
assertEquals(1, letterSoundLearningEvent.getLetterSoundId());
90+
}
91+
7092

7193
// TODO: number assessment events
7294

@@ -306,4 +328,29 @@ public void testExtractVideoLearningEvents_v3005009() throws IOException {
306328
assertEquals("เพลง นับเลข 1-10 ｜ เพลงเด็กอนุบาล ｜ นับเล", videoLearningEvent.getVideoTitle());
307329
assertEquals(9, videoLearningEvent.getVideoId());
308330
}
331+
332+
/**
333+
* Test extraction of data from CSV files generated by version 3005013 of the Analytics app:
334+
* https://github.com/elimu-ai/analytics/releases/tag/3.5.13
335+
*/
336+
@Test
337+
public void testExtractVideoLearningEvents_v3005013() throws IOException {
338+
ResourceLoader resourceLoader = new ClassRelativeResourceLoader(CsvAnalyticsExtractionHelper.class);
339+
Resource resource = resourceLoader.getResource("7e89c8e7f4c68405_3005013_video-learning-events_2025-06-13.csv");
340+
File csvFile = resource.getFile();
341+
342+
List<VideoLearningEvent> videoLearningEvents = CsvAnalyticsExtractionHelper.extractVideoLearningEvents(csvFile);
343+
assertEquals(9, videoLearningEvents.size());
344+
345+
VideoLearningEvent videoLearningEvent = videoLearningEvents.get(0);
346+
assertEquals(1749775644 * 1_000L, videoLearningEvent.getTimestamp().getTimeInMillis());
347+
assertEquals("7e89c8e7f4c68405", videoLearningEvent.getAndroidId());
348+
assertEquals("ai.elimu.filamu", videoLearningEvent.getPackageName());
349+
assertNull(videoLearningEvent.getAdditionalData());
350+
assertEquals(LearningEventType.VIDEO_OPENED, videoLearningEvent.getLearningEventType());
351+
assertNull(videoLearningEvent.getResearchExperiment());
352+
assertNull(videoLearningEvent.getExperimentGroup());
353+
assertEquals("akili and me - letter b", videoLearningEvent.getVideoTitle());
354+
assertEquals(2, videoLearningEvent.getVideoId());
355+
}
309356
}

‎src/test/resources/ai/elimu/util/csv/7e89c8e7f4c68405_3005013_letter-sound-learning-events_2025-05-14.csv

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
id,timestamp,package_name,additional_data,research_experiment,experiment_group,letter_sound_letter_texts,letter_sound_sound_values_ipa,letter_sound_id
2+
1,1747200557,ai.elimu.herufi,,,,,,1
3+
2,1747200559,ai.elimu.herufi,,,,,,2
4+
3,1747200562,ai.elimu.herufi,,,,,,3
5+
4,1747200563,ai.elimu.herufi,,,,,,4
6+
5,1747200566,ai.elimu.herufi,,,,,,5

‎src/test/resources/ai/elimu/util/csv/7e89c8e7f4c68405_3005013_video-learning-events_2025-06-13.csv

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
id,timestamp,package_name,additional_data,learning_event_type,research_experiment,experiment_group,video_title,video_id
2+
1,1749775644,ai.elimu.filamu,,VIDEO_OPENED,,,akili and me - letter b,2
3+
2,1749775646,ai.elimu.filamu,,VIDEO_CLOSED_BEFORE_COMPLETION,,,,2
4+
3,1749775647,ai.elimu.filamu,,VIDEO_OPENED,,,[test] a test video,44
5+
4,1749775648,ai.elimu.filamu,,VIDEO_CLOSED_BEFORE_COMPLETION,,,,44
6+
5,1749775649,ai.elimu.filamu,,VIDEO_OPENED,,,[test] a test video,44
7+
6,1749775653,ai.elimu.filamu,,VIDEO_CLOSED_BEFORE_COMPLETION,,,,44
8+
7,1749799858,ai.elimu.filamu,,VIDEO_OPENED,,,[test] a test video,44
9+
8,1749799867,ai.elimu.filamu,,VIDEO_COMPLETED,,,[test] a test video,44
10+
9,1749800215,ai.elimu.filamu,,VIDEO_OPENED,,,akili and me - the rectangle song,13

0 commit comments

Comments
 (0)