Merge pull request #457 from pixlise/feature/em-import-fixes

EM importer: Better logging, logs BGT output too, don't overwrite RSI…

Author: pnemere

api/dataimport/pre-import.go

@@ -78,7 +78,7 @@ func ProcessEM(importId string, zipReader *zip.Reader, zippedData []byte, destBu
 	isCalTarget := strings.Contains(lowerId, "cal-target") || strings.Contains(lowerId, "cal_target") || strings.Contains(lowerId, "caltarget")
 
 	// Create an RSI file from the sdf_raw file
-	genFiles, rtts, err := sdfToRSI.ConvertSDFtoRSIs(sdfLocalPath, localTemp)
+	genFiles, rtts, err := sdfToRSI.ConvertSDFtoRSIs(sdfLocalPath, localTemp, logger)
 
 	if err != nil {
 		return fmt.Errorf("Failed to scan %v for RSI creation: %v", sdfLocalPath, err)
@@ -100,7 +100,7 @@ func ProcessEM(importId string, zipReader *zip.Reader, zippedData []byte, destBu
 
 		// Every second file is a HK file not an actual RSI file... make sure we have the right prefix here
 		if !strings.HasPrefix(f, "RSI-") || !strings.HasPrefix(hkFile, "HK-") {
-			logger.Errorf("ConvertSDFtoRSIs generated : %v. Error: %v", f, err)
+			logger.Errorf("ConvertSDFtoRSIs generated: \"%v\". Error: %v", f, err)
 			continue
 		}
 
@@ -112,7 +112,7 @@ func ProcessEM(importId string, zipReader *zip.Reader, zippedData []byte, destBu
 			continue
 		}
 
-		// Upoad the output files (beam locations, log and surface)
+		// Upload the output files (beam locations, log and surface)
 		files := []string{filepath.Join(localTemp, hkFile), rxlPath, logPath, surfPath, rsiLocalPath}
 		name := []string{"housekeeping", "beam location", "log", "surface", "rsi"}
 		for i, file := range files {
@@ -234,24 +234,45 @@ func createBeamLocation(isCalTarget bool, rsiPath string, rtt int64, outputBeamL
 	if _, err := os.Stat(bgtPath + "BGT"); err != nil {
 		// Try the path used in local testing
 		bgtPath = ".." + string(os.PathSeparator) + ".." + string(os.PathSeparator) + "beam-tool" + string(os.PathSeparator)
+		if _, err = os.Stat(bgtPath + "BGT"); err != nil {
+			return "", "", "", errors.New("BGT tool not found")
+		}
+	}
+
+	// Ensure output files don't exist yet!
+	toDel := []string{outSurfaceTop, outRXL, outLog}
+	for _, f := range toDel {
+		err := os.Remove(f)
+		if err != nil {
+			logger.Errorf("Error deleting existing \"%v\": %v", f, err)
+		}
+	}
+
+	// Find an absolute path because after Go 1.19 exec.Command doesn't work with relative paths... strange this wasn't noticed earlier!
+	wd, err := os.Getwd()
+	if err != nil {
+		return "", "", "", fmt.Errorf("Failed to get working dir: %v", err)
 	}
 
-	if _, err := os.Stat(bgtPath + "Geometry_PIXL_EM_Landing_25Jan2021.csv"); err != nil {
+	calibPath := path.Join(wd, bgtPath, "Geometry_PIXL_EM_Landing_25Jan2021.csv")
+
+	if _, err := os.Stat(calibPath); err != nil {
 		return "", "", "", errors.New("Calibration file not found")
 	}
 	if _, err := os.Stat(rsiPath); err != nil {
 		return "", "", "", errors.New("RSI not found")
 	}
 
-	args := []string{bgtPath + "Geometry_PIXL_EM_Landing_25Jan2021.csv", rsiPath, outSurfaceTop, outRXL}
+	args := []string{calibPath, rsiPath, outSurfaceTop, outRXL}
 	if isCalTarget {
 		args = append(args, "-t")
 	}
 	args = append(args, outLog)
 
-	fmt.Printf("Executing: %v %v\n", bgtPath+"BGT", strings.Join(args, " "))
+	exePath := path.Join(wd, bgtPath, "BGT")
 
-	cmd := exec.Command(bgtPath+"BGT", args...)
+	fmt.Printf("Executing: %v %v\n", exePath, strings.Join(args, " "))
+	cmd := exec.Command(exePath, args...)
 
 	// var out bytes.Buffer
 	// var stderr bytes.Buffer
@@ -262,17 +283,30 @@ func createBeamLocation(isCalTarget bool, rsiPath string, rtt int64, outputBeamL
 	// cmd.Stderr = os.Stderr
 	cmd.Dir = bgtPath
 
-	if out, err := cmd.CombinedOutput(); err != nil {
+	errOut := false
+	if out, cmdErr := cmd.CombinedOutput(); cmdErr != nil {
 		logger.Infof("CombinedOutput:\n%s", out)
+		err = cmdErr
+		errOut = true // Don't return just yet, we want to print the log file if it exists..
 		//if err := cmd.Run(); err != nil {
 		// Dump std out
 		// logger.Infof("BGT stdout:\n" + out.String())
 		// logger.Errorf("BGT stderr:\n" + stderr.String())
-		return "", "", "", fmt.Errorf("BGT tool error: %v", err)
 	} else {
 		logger.Infof("CombinedOutput:\n%s", out)
 	}
 
+	// Dump the log file to the log here
+	if bgtLog, logErr := os.ReadFile(outLog); logErr != nil {
+		logger.Infof("Failed to read BGT log \"%v\": %v", outLog, logErr)
+	} else {
+		logger.Infof("BGT Log Output:\n%s", bgtLog)
+	}
+
+	if errOut {
+		return "", "", "", fmt.Errorf("BGT tool error: %v", err)
+	}
+
 	/*
 		procAttr := new(os.ProcAttr)
 		procAttr.Files = []*os.File{nil, nil, nil}

api/dataimport/pre-import_test.go

@@ -41,7 +41,7 @@ func Example_dataimport_ProcessEM() {
 
 			localTemp, sdfLocalPath, _, _, err := startEMProcess("123", z, zFile, &l)
 			fmt.Printf("startEMProcess err=%v\n", err)
-			genFiles, rtts, err := sdfToRSI.ConvertSDFtoRSIs(sdfLocalPath, localTemp)
+			genFiles, rtts, err := sdfToRSI.ConvertSDFtoRSIs(sdfLocalPath, localTemp, &l)
 			fmt.Printf("genFiles: %v\nrtts: %v\nerr: %v\n", genFiles, rtts, err)
 		}
 	}

api/dataimport/sdfToRSI/scanSDF.go

@@ -141,6 +141,7 @@ func scanSDF(sdfPath string) ([]EventEntry, error) {
 		pos := strings.Index(lineData, sciPlace)
 		if pos > -1 {
 			lineData = lineData[pos+len(sciPlace):]
+			lineData = strings.TrimRight(lineData, "\"")
 			refs = append(refs, EventEntry{Line: lineNo, What: "sci-place", Value: lineData})
 		}
 	}

api/dataimport/sdfToRSI/sdfToRSI.go

@@ -7,11 +7,14 @@ import (
 	"path"
 	"strconv"
 	"strings"
+
+	"github.com/pixlise/core/v4/core/logger"
+	"github.com/pixlise/core/v4/core/utils"
 )
 
 // Given an SDF path and an output path, this generates RSI files for each scan mentioned in the SDF.
 // Returns the file names generated and an error if any
-func ConvertSDFtoRSIs(sdfPath string, outPath string) ([]string, []int64, error) {
+func ConvertSDFtoRSIs(sdfPath string, outPath string, logger logger.ILogger) ([]string, []int64, error) {
 	files := []string{}
 	rtts := []int64{}
 
@@ -36,14 +39,19 @@ func ConvertSDFtoRSIs(sdfPath string, outPath string) ([]string, []int64, error)
 			} else if ref.Value != "end" {
 				return files, rtts, fmt.Errorf("End not found for science RTT: %v", rtt)
 			} else {
-				nameRSI := fmt.Sprintf("RSI-%v.csv", rtt)
-				nameHK := fmt.Sprintf("HK-%v.csv", rtt)
-				err = sdfToRSI(sdfPath, rtt, startLine, ref.Line, path.Join(outPath, nameRSI), path.Join(outPath, nameHK))
-				if err != nil {
-					return files, rtts, fmt.Errorf("Failed to generate files %v, %v: %v", nameRSI, nameHK, err)
+				// If we've already seen an "end" for this rtt, ignore
+				if utils.ItemInSlice(rtt, rtts) {
+					logger.Infof("ConvertSDFtoRSIs \"%v\" [%v]: Already detected end of RTT %v - skipping...", sdfPath, ref.Line, rtt)
+				} else {
+					nameRSI := fmt.Sprintf("RSI-%v.csv", rtt)
+					nameHK := fmt.Sprintf("HK-%v.csv", rtt)
+					err = sdfToRSI(sdfPath, rtt, startLine, ref.Line, path.Join(outPath, nameRSI), path.Join(outPath, nameHK))
+					if err != nil {
+						return files, rtts, fmt.Errorf("Failed to generate files %v, %v: %v", nameRSI, nameHK, err)
+					}
+					files = append(files, nameRSI, nameHK)
+					rtts = append(rtts, rtt)
 				}
-				files = append(files, nameRSI, nameHK)
-				rtts = append(rtts, rtt)
 			}
 		}
 	}

api/dataimport/sdfToRSI/sdfToRSI_test.go

@@ -7,6 +7,8 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+
+	"github.com/pixlise/core/v4/core/logger"
 )
 
 func Example_sdfToRSI_ConvertSDFtoRSI() {
@@ -16,7 +18,7 @@ func Example_sdfToRSI_ConvertSDFtoRSI() {
 	wd, err := os.Getwd()
 	fmt.Printf("Getwd: %v\n", err == nil)
 	p := filepath.Join(wd, "output")
-	files, rtts, err := ConvertSDFtoRSIs("./test-data/sdf_raw.txt", p)
+	files, rtts, err := ConvertSDFtoRSIs("./test-data/sdf_raw.txt", p, &logger.StdOutLogger{})
 	fmt.Printf("%v, %v: %v\n", files, rtts, err)
 
 	// Output:
@@ -30,7 +32,7 @@ func Example_sdfToRSI_ConvertSDFtoRSI_EndingPrematurely() {
 	wd, err := os.Getwd()
 	fmt.Printf("Getwd: %v\n", err == nil)
 	p := filepath.Join(wd, "output")
-	files, rtts, err := ConvertSDFtoRSIs("./test-data/sdf_raw_premature_end.txt", p)
+	files, rtts, err := ConvertSDFtoRSIs("./test-data/sdf_raw_premature_end.txt", p, &logger.StdOutLogger{})
 	fmt.Printf("%v, %v: %v\n", files, rtts, err)
 
 	// Output: