Merge pull request #928 from CortexFoundation/dev

Boot memory detect & update

Author: ucwong

cmd/cortex/main.go

@@ -31,12 +31,9 @@ import (
 	"github.com/CortexFoundation/CortexTheseus/node"
 	_ "github.com/CortexFoundation/statik"
 	"github.com/arsham/figurine/figurine"
-	gopsutil "github.com/shirou/gopsutil/mem"
 	cli "gopkg.in/urfave/cli.v1"
-	"math"
 	"math/big"
 	"os"
-	godebug "runtime/debug"
 	"sort"
 	"strconv"
 	"strings"
@@ -302,28 +299,6 @@ func prepare(ctx *cli.Context) {
 		log.Info("Bumping default cache on mainnet", "provided", ctx.GlobalInt(utils.CacheFlag.Name), "updated", 4096)
 		ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(4096))
 	}
-	// Cap the cache allowance and tune the garbage collector
-	mem, err := gopsutil.VirtualMemory()
-	// Workaround until OpenBSD support lands into gosigar
-	// Check https://github.com/elastic/gosigar#supported-platforms
-	if err == nil {
-		if 32<<(^uintptr(0)>>63) == 32 && mem.Total > 2*1024*1024*1024 {
-			log.Warn("Lowering memory allowance on 32bit arch", "available", mem.Total/1024/1024, "addressable", 2*1024)
-			mem.Total = 2 * 1024 * 1024 * 1024
-		}
-		allowance := int(mem.Total / 1024 / 1024 / 3)
-		if cache := ctx.GlobalInt(utils.CacheFlag.Name); cache > allowance {
-			log.Warn("Sanitizing cache to Go's GC limits", "provided", cache, "updated", allowance)
-			ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(allowance))
-		}
-	}
-	// Ensure Go's GC ignores the database cache for trigger percentage
-	cache := ctx.GlobalInt(utils.CacheFlag.Name)
-	gogc := math.Max(20, math.Min(100, 100/(float64(cache)/1024)))
-
-	log.Info("Sanitizing Go's GC trigger", "percent", int(gogc), "cache", cache)
-	godebug.SetGCPercent(int(gogc))
-
 	// Start metrics export if enabled
 	utils.SetupMetrics(ctx)
 

cmd/utils/flags.go

@@ -24,9 +24,11 @@ import (
 	"io/ioutil"
 	"os"
 
+	"math"
 	"math/big"
 	"path/filepath"
 	"runtime"
+	godebug "runtime/debug"
 	"strconv"
 	"strings"
 	"time"
@@ -1377,6 +1379,27 @@ func SetCortexConfig(ctx *cli.Context, stack *node.Node, cfg *ctxc.Config) {
 	setGPO(ctx, &cfg.GPO)
 	setTxPool(ctx, &cfg.TxPool)
 	setWhitelist(ctx, cfg)
+	// Cap the cache allowance and tune the garbage collector
+	mem, err := gopsutil.VirtualMemory()
+	// Workaround until OpenBSD support lands into gosigar
+	// Check https://github.com/elastic/gosigar#supported-platforms
+	if err == nil {
+		if 32<<(^uintptr(0)>>63) == 32 && mem.Total > 2*1024*1024*1024 {
+			log.Warn("Lowering memory allowance on 32bit arch", "available", mem.Total/1024/1024, "addressable", 2*1024)
+			mem.Total = 2 * 1024 * 1024 * 1024
+		}
+		allowance := int(mem.Total / 1024 / 1024 / 3)
+		if cache := ctx.GlobalInt(CacheFlag.Name); cache > allowance {
+			log.Warn("Sanitizing cache to Go's GC limits", "provided", cache, "updated", allowance)
+			ctx.GlobalSet(CacheFlag.Name, strconv.Itoa(allowance))
+		}
+	}
+	// Ensure Go's GC ignores the database cache for trigger percentage
+	cache := ctx.GlobalInt(CacheFlag.Name)
+	gogc := math.Max(20, math.Min(100, 100/(float64(cache)/1024)))
+
+	log.Info("Sanitizing Go's GC trigger", "percent", int(gogc), "cache", cache)
+	godebug.SetGCPercent(int(gogc))
 
 	if ctx.GlobalIsSet(SyncModeFlag.Name) {
 		cfg.SyncMode = *GlobalTextMarshaler(ctx, SyncModeFlag.Name).(*downloader.SyncMode)
@@ -1524,8 +1547,9 @@ func SetCortexConfig(ctx *cli.Context, stack *node.Node, cfg *ctxc.Config) {
 	} else {
 		panic(fmt.Sprintf("invalid device: %s", cfg.InferDeviceType))
 	}
+
 	cfg.InferDeviceId = ctx.GlobalInt(InferDeviceIdFlag.Name)
-	mem, err := gopsutil.VirtualMemory()
+	mem, err = gopsutil.VirtualMemory()
 	if err == nil {
 		if 32<<(^uintptr(0)>>63) == 32 && mem.Total > 2*1024*1024*1024 {
 			log.Warn("Lowering memory allowance on 32bit arch", "available", mem.Total/1024/1024, "addressable", 2*1024)
@@ -1537,6 +1561,7 @@ func SetCortexConfig(ctx *cli.Context, stack *node.Node, cfg *ctxc.Config) {
 			ctx.GlobalSet(InferMemoryFlag.Name, strconv.Itoa(allowance))
 		}
 	}
+
 	cfg.InferMemoryUsage = int64(ctx.GlobalInt(InferMemoryFlag.Name))
 	cfg.InferMemoryUsage = cfg.InferMemoryUsage << 20
 	//log.Warn("C MEMORY FOR CVM", "cache", cfg.InferMemoryUsage)