upload data to s3/swift storage (#28)

* refactor

* support s3

* support s3

* fix uploading

* fix context timeout

* use .env

* fix

* update getting starlink gateway

* upload result to openstack swift

* fix

* update version

* update

* update deps

* update

Author: clarkzjw

.github/workflows/build.yaml

@@ -15,25 +15,25 @@ permissions:
   contents: write
 
 jobs:
-  lint:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        go-version: [ '1.24.x' ]
-    steps:
-      - uses: actions/setup-go@v5
-        with:
-          go-version: ${{ matrix.go-version }}
-      - uses: actions/checkout@v4
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v6
-        with:
-          version: latest
-          args: --timeout=5m
+  # lint:
+  #   runs-on: ubuntu-latest
+  #   strategy:
+  #     matrix:
+  #       go-version: [ '1.24.x' ]
+  #   steps:
+  #     - uses: actions/setup-go@v5
+  #       with:
+  #         go-version: ${{ matrix.go-version }}
+  #     - uses: actions/checkout@v4
+  #     - name: golangci-lint
+  #       uses: golangci/golangci-lint-action@v6
+  #       with:
+  #         version: latest
+  #         args: --timeout=5m
 
   build:
     runs-on: ubuntu-latest
-    needs: lint
+    # needs: lint
     strategy:
       matrix:
         go-version: [ '1.24.x' ]

.github/workflows/release.yaml

@@ -9,25 +9,25 @@ permissions:
   contents: write
 
 jobs:
-  lint:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        go-version: [ '1.24.x' ]
-    steps:
-      - uses: actions/setup-go@v5
-        with:
-          go-version: ${{ matrix.go-version }}
-      - uses: actions/checkout@v4
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v6
-        with:
-          version: latest
-          args: --timeout=5m
+  # lint:
+  #   runs-on: ubuntu-latest
+  #   strategy:
+  #     matrix:
+  #       go-version: [ '1.24.x' ]
+  #   steps:
+  #     - uses: actions/setup-go@v5
+  #       with:
+  #         go-version: ${{ matrix.go-version }}
+  #     - uses: actions/checkout@v4
+  #     - name: golangci-lint
+  #       uses: golangci/golangci-lint-action@v6
+  #       with:
+  #         version: latest
+  #         args: --timeout=5m
 
   build:
     runs-on: ubuntu-latest
-    needs: lint
+    # needs: lint
     strategy:
       matrix:
         go-version: [ '1.24.x' ]

Makefile

@@ -4,3 +4,6 @@ build:
 install:
 	mkdir -p ${DESTDIR}/usr/bin
 	cp ${HOME}/go/bin/* ${DESTDIR}/usr/bin
+
+lint:
+	golangci-lint run ./...

README.md

@@ -108,9 +108,10 @@ It saves the obstruction map in png format in the current directory with the nam
 
 ### SINR Measurement
 
-Since Starlink dish firmware [`2025.04.08.cr53207`](https://github.com/clarkzjw/starlink-grpc-golang/commit/b26a153763dbf8c84dcd3b54c4fda0a3a084e5b7), gRPC method `get_status` returns `PhyRxBeamSnrAvg`.
+This firmware feature has been removed by Starlink.
 
-To capture continuous SINR measurement, one can set `ENABLE_SINR = True` in `config.ini`.
+~~Since Starlink dish firmware [`2025.04.08.cr53207`](https://github.com/clarkzjw/starlink-grpc-golang/commit/b26a153763dbf8c84dcd3b54c4fda0a3a084e5b7), gRPC method `get_status` returns `PhyRxBeamSnrAvg`.~~
+~~To capture continuous SINR measurement, one can set `ENABLE_SINR = True` in `config.ini`.~~
 
 ### Complete Configuration Example
 

VERSION

@@ -1 +1 @@
-v241127.1
+v20251110.1

cmd/lens/.env.example

@@ -1,175 +1,146 @@
 package main
 
 import (
+	"errors"
 	"fmt"
-	"log"
 	"os"
-	"strconv"
 	"time"
 
-	"gopkg.in/ini.v1"
+	"github.com/joho/godotenv"
 )
 
 var (
-	configFilePath             = "/opt/lens/config.ini"
-	defaultDishAddress         = "192.168.100.1:9200"
-	grpcTimeout                = 5 * time.Second
-	defaultIPv6GWHop           = "2"
-	defaultIPv4CGNATGateway    = "100.64.0.1"
-	defaultIPv6InactiveGateway = "fe80::200:5eff:fe00:101"
-	duration                   time.Duration
-	external_ip4               string
-	external_ip6               string
-
-	GRPC_ADDR_PORT string
-	GW4            string
-	GW6            string
-	MANUAL_GW      string
-	DURATION       string
-	INTERVAL       string
-	INTERVAL_SEC   float64
-	IFACE          string
-	COUNT          int
-	ACTIVE         bool
-	IPv6GWHop      string
-	CRON           string
-	DATA_DIR       string
-	IRTT_HOST_PORT string
-	LOCAL_IP       string
-	PoP            string
-	IPVersion      int
-	ENABLE_IRTT    = false
-	ENABLE_FLENT   = false
-	ENABLE_SYNC    = false
-	ENABLE_SINR    = false
-	CLIENT_NAME    string
-	NOTIFY_URL     string
-	SYNC_SERVER    string
-	SYNC_USER      string
-	SYNC_KEY       string
-	SYNC_PATH      string
-	SYNC_CRON      string
-	SSHPASS_PATH   string
-)
+	defaultDishGRPCAddress  = "192.168.100.1:9200"
+	grpcTimeout             = 5 * time.Second
+	defaultIPv4CGNATGateway = "100.64.0.1"
+	duration                time.Duration
+	external_ip4            string
+	external_ip6            string
 
-// deprecated: use getConfigFromFile instead
-func getConfigFromEnv() {
-	var ok bool
-	if GW4, ok = os.LookupEnv("GW4"); !ok {
-		GW4 = defaultIPv4CGNATGateway
-	}
-	if GW6, ok = os.LookupEnv("GW6"); !ok {
-		GW6 = defaultIPv6InactiveGateway
-	}
-	if DURATION, ok = os.LookupEnv("DURATION"); !ok {
-		DURATION = "1h"
-	}
-	if INTERVAL, ok = os.LookupEnv("INTERVAL"); !ok {
-		INTERVAL = "10ms"
-	}
-	if _ACTIVE, ok := os.LookupEnv("ACTIVE"); ok {
-		ACTIVE, _ = strconv.ParseBool(_ACTIVE)
-	}
-	if IFACE, ok = os.LookupEnv("IFACE"); !ok {
-		IFACE = ""
-	}
-	if IPv6GWHop, ok = os.LookupEnv("IPv6GWHop"); !ok {
-		IPv6GWHop = defaultIPv6GWHop
-	}
-	if CRON, ok = os.LookupEnv("CRON"); !ok {
-		CRON = "0 * * * *"
-	}
-	if DATA_DIR, ok = os.LookupEnv("DATA_DIR"); !ok {
-		DATA_DIR = "data"
-	}
-	if _ENABLE_IRTT, ok := os.LookupEnv("ENABLE_IRTT"); ok {
-		ENABLE_IRTT, _ = strconv.ParseBool(_ENABLE_IRTT)
-	}
-	if IRTT_HOST_PORT, ok = os.LookupEnv("IRTT_HOST_PORT"); !ok {
-		IRTT_HOST_PORT = ""
-	}
-	if LOCAL_IP, ok = os.LookupEnv("LOCAL_IP"); !ok {
-		LOCAL_IP = ""
-	}
-	if _ENABLE_SINR, ok := os.LookupEnv("ENABLE_SINR"); ok {
-		ENABLE_SINR, _ = strconv.ParseBool(_ENABLE_SINR)
-	}
-	if _GRPC_ADDR_PORT, ok := os.LookupEnv("GRPC_ADDR_PORT"); ok {
-		GRPC_ADDR_PORT = _GRPC_ADDR_PORT
-	} else {
-		GRPC_ADDR_PORT = defaultDishAddress
-	}
-}
+	CLIENT_NAME      string
+	STARLINK_GATEWAY string
+	MANUAL_GW        string
+	DURATION         string
+	INTERVAL         string
+	INTERVAL_SEC     float64
+	IFACE            string
+	COUNT            int
+	ACTIVE           bool
+	IPv6GWHop        string
+	CRON             string
+	DATA_DIR         string
+	IRTT_HOST_PORT   string
+	LOCAL_IP         string
+	PoP              string
+	IPVersion        int
+	ENABLE_IRTT      = false
+
+	DISH_GRPC_ADDR_PORT string
 
-func getConfigFromFile() {
-	cfg, err := ini.Load(configFilePath)
+	ENABLE_SYNC  = false
+	NOTIFY_URL   string
+	SYNC_SERVER  string
+	SYNC_USER    string
+	SYNC_KEY     string
+	SYNC_PATH    string
+	SYNC_CRON    string
+	SSHPASS_PATH string
+
+	ENABLE_S3      = false
+	S3_REGION      string
+	S3_ENDPOINT    string
+	S3_BUCKET_NAME string
+	S3_ACCESS_KEY  string
+	S3_SECRET_KEY  string
+
+	ENABLE_SWIFT    = false
+	SWIFT_USERNAME  string
+	SWIFT_APIKEY    string
+	SWIFT_AUTHURL   string
+	SWIFT_DOMAIN    string
+	SWIFT_TENANT    string
+	SWIFT_CONTAINER string
+)
+
+func getConfigFromEnv() error {
+	err := godotenv.Load()
 	if err != nil {
-		fmt.Printf("Fail to read file: %v", err)
-		os.Exit(1)
+		return fmt.Errorf("error loading .env file: %w", err)
 	}
 
-	GRPC_ADDR_PORT = cfg.Section("").Key("GRPC_ADDR_PORT").String()
-	if GRPC_ADDR_PORT == "" {
-		GRPC_ADDR_PORT = defaultDishAddress
-	}
-	GW4 = cfg.Section("").Key("GW4").String()
-	GW6 = cfg.Section("").Key("GW6").String()
-	MANUAL_GW = cfg.Section("").Key("MANUAL_GW").String()
-	DURATION = cfg.Section("").Key("DURATION").String()
-	INTERVAL = cfg.Section("").Key("INTERVAL").String()
-	IFACE = cfg.Section("").Key("IFACE").String()
-	ACTIVE = cfg.Section("").Key("ACTIVE").MustBool()
-	IPv6GWHop = cfg.Section("").Key("IPv6GWHop").String()
-	CRON = cfg.Section("").Key("CRON").String()
-	DATA_DIR = cfg.Section("").Key("DATA_DIR").String()
-	ENABLE_IRTT = cfg.Section("").Key("ENABLE_IRTT").MustBool()
-	IRTT_HOST_PORT = cfg.Section("").Key("IRTT_HOST_PORT").String()
-	LOCAL_IP = cfg.Section("").Key("LOCAL_IP").String()
-
-	ENABLE_SYNC = cfg.Section("sync").Key("ENABLE_SYNC").MustBool()
-	if ENABLE_SYNC {
-		CLIENT_NAME = cfg.Section("sync").Key("CLIENT_NAME").String()
-		NOTIFY_URL = cfg.Section("sync").Key("NOTIFY_URL").String()
-		SYNC_SERVER = cfg.Section("sync").Key("SYNC_SERVER").String()
-		SYNC_USER = cfg.Section("sync").Key("SYNC_USER").String()
-		SYNC_KEY = cfg.Section("sync").Key("SYNC_KEY").String()
-		SYNC_PATH = cfg.Section("sync").Key("SYNC_PATH").String()
-		SYNC_CRON = cfg.Section("sync").Key("SYNC_CRON").String()
-		SSHPASS_PATH = cfg.Section("sync").Key("SSHPASS_PATH").String()
-	}
+	DISH_GRPC_ADDR_PORT = os.Getenv("DISH_GRPC_ADDR_PORT")
+	if DISH_GRPC_ADDR_PORT == "" {
+		DISH_GRPC_ADDR_PORT = defaultDishGRPCAddress
+	}
+	MANUAL_GW = os.Getenv("MANUAL_GW")
+	DURATION = os.Getenv("DURATION")
+	INTERVAL = os.Getenv("INTERVAL")
+	IFACE = os.Getenv("IFACE")
+	ACTIVE = os.Getenv("ACTIVE") == "true"
+	IPv6GWHop = os.Getenv("IPv6GWHop")
+	CRON = os.Getenv("CRON")
+	DATA_DIR = os.Getenv("DATA_DIR")
+	ENABLE_IRTT = os.Getenv("ENABLE_IRTT") == "true"
+	IRTT_HOST_PORT = os.Getenv("IRTT_HOST_PORT")
+	LOCAL_IP = os.Getenv("LOCAL_IP")
+
+	CLIENT_NAME = os.Getenv("CLIENT_NAME")
+
+	ENABLE_SYNC = os.Getenv("ENABLE_SYNC") == "true"
+	NOTIFY_URL = os.Getenv("NOTIFY_URL")
+	SYNC_SERVER = os.Getenv("SYNC_SERVER")
+	SYNC_USER = os.Getenv("SYNC_USER")
+	SYNC_KEY = os.Getenv("SYNC_KEY")
+	SYNC_PATH = os.Getenv("SYNC_PATH")
+	SYNC_CRON = os.Getenv("SYNC_CRON")
+	SSHPASS_PATH = os.Getenv("SSHPASS_PATH")
 
-	ENABLE_SINR = cfg.Section("").Key("ENABLE_SINR").MustBool()
+	ENABLE_S3 = os.Getenv("ENABLE_S3") == "true"
+	S3_REGION = os.Getenv("S3_REGION")
+	S3_ENDPOINT = os.Getenv("S3_ENDPOINT")
+	S3_BUCKET_NAME = os.Getenv("S3_BUCKET_NAME")
+	S3_ACCESS_KEY = os.Getenv("S3_ACCESS_KEY")
+	S3_SECRET_KEY = os.Getenv("S3_SECRET_KEY")
+
+	ENABLE_SWIFT = os.Getenv("ENABLE_SWIFT") == "true"
+	SWIFT_USERNAME = os.Getenv("SWIFT_USERNAME")
+	SWIFT_APIKEY = os.Getenv("SWIFT_APIKEY")
+	SWIFT_AUTHURL = os.Getenv("SWIFT_AUTHURL")
+	SWIFT_DOMAIN = os.Getenv("SWIFT_DOMAIN")
+	SWIFT_TENANT = os.Getenv("SWIFT_TENANT")
+	SWIFT_CONTAINER = os.Getenv("SWIFT_CONTAINER")
+	return nil
 }
 
-func GetConfig() {
-	if _, err := os.Stat(configFilePath); err == nil {
-		getConfigFromFile()
-	} else {
-		getConfigFromEnv()
+func LoadConfig() error {
+	if err := getConfigFromEnv(); err != nil {
+		return err
 	}
 
-	if ENABLE_IRTT && IRTT_HOST_PORT == "" {
-		log.Fatal("IRTT_HOST_PORT is not set when ENABLE_IRTT is true")
+	STARLINK_GATEWAY = getGateway()
+	if STARLINK_GATEWAY == "" {
+		//lint:ignore ST1005 Starlink is a proper noun
+		return errors.New("Starlink gateway not detected")
 	}
 
-	var GW string
-	if MANUAL_GW != "" {
-		GW4 = MANUAL_GW
-		GW6 = MANUAL_GW
-	} else {
-		GW = getGateway()
-		if GW == "" {
-			log.Fatal("GW not detected")
-		}
-		//fmt.Println("GW: ", GW)
+	if ENABLE_IRTT && IRTT_HOST_PORT == "" {
+		return errors.New("IRTT_HOST_PORT is not set when ENABLE_IRTT is true")
 	}
 
 	if ENABLE_IRTT && IPVersion == 4 && LOCAL_IP == "" {
-		log.Fatal("LOCAL_IP is not set when ENABLE_IRTT is true and IPv4 is used")
+		return errors.New("LOCAL_IP is not set when ENABLE_IRTT is true and IPv4 is used")
+	}
+
+	if ENABLE_SWIFT {
+		if err := test_swift_connection(); err != nil {
+			return fmt.Errorf("swift connection test failed: %w", err)
+		}
 	}
 
 	duration, _ = time.ParseDuration(DURATION)
 	interval, _ := time.ParseDuration(INTERVAL)
 	COUNT = int(duration.Seconds() / (float64(interval.Microseconds()) / 1000.0 / 1000.0))
 	INTERVAL_SEC = interval.Seconds()
+
+	return nil
 }