Merge pull request #879 from CortexFoundation/dev

Miner & trie state & storage & network

Author: ucwong

accounts/scwallet/securechannel.go

@@ -20,14 +20,14 @@ import (
 	"bytes"
 	"crypto/aes"
 	"crypto/cipher"
+	"crypto/elliptic"
 	"crypto/rand"
 	"crypto/sha256"
 	"crypto/sha512"
 	"fmt"
 
 	"github.com/CortexFoundation/CortexTheseus/crypto"
 	pcsc "github.com/gballet/go-libpcsclite"
-	"github.com/wsddn/go-ecdh"
 	"golang.org/x/crypto/pbkdf2"
 	"golang.org/x/text/unicode/norm"
 )
@@ -63,26 +63,19 @@ type SecureChannelSession struct {
 // NewSecureChannelSession creates a new secure channel for the given card and public key.
 func NewSecureChannelSession(card *pcsc.Card, keyData []byte) (*SecureChannelSession, error) {
 	// Generate an ECDSA keypair for ourselves
-	gen := ecdh.NewEllipticECDH(crypto.S256())
-	private, public, err := gen.GenerateKey(rand.Reader)
+	key, err := crypto.GenerateKey()
 	if err != nil {
 		return nil, err
 	}
-
-	cardPublic, ok := gen.Unmarshal(keyData)
-	if !ok {
-		return nil, fmt.Errorf("could not unmarshal public key from card")
-	}
-
-	secret, err := gen.GenerateSharedSecret(private, cardPublic)
+	cardPublic, err := crypto.UnmarshalPubkey(keyData)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("could not unmarshal public key from card: %v", err)
 	}
-
+	secret, _ := key.Curve.ScalarMult(cardPublic.X, cardPublic.Y, key.D.Bytes())
 	return &SecureChannelSession{
 		card:      card,
-		secret:    secret,
-		publicKey: gen.Marshal(public),
+		secret:    secret.Bytes(),
+		publicKey: elliptic.Marshal(crypto.S256(), key.PublicKey.X, key.PublicKey.Y),
 	}, nil
 }
 

cmd/bootnode/main.go

@@ -28,7 +28,6 @@ import (
 	"github.com/CortexFoundation/CortexTheseus/crypto"
 	"github.com/CortexFoundation/CortexTheseus/log"
 	"github.com/CortexFoundation/CortexTheseus/p2p/discover"
-	"github.com/CortexFoundation/CortexTheseus/p2p/discv5"
 	"github.com/CortexFoundation/CortexTheseus/p2p/enode"
 	"github.com/CortexFoundation/CortexTheseus/p2p/nat"
 	"github.com/CortexFoundation/CortexTheseus/p2p/netutil"
@@ -128,25 +127,16 @@ func main() {
 
 	db, _ := enode.OpenDB("")
 	ln := enode.NewLocalNode(db, nodeKey)
-
-	var unhandled chan discover.ReadPacket
-	var sconn *sharedUDPConn
-	if *runv5 {
-		unhandled = make(chan discover.ReadPacket, 100)
-		sconn = &sharedUDPConn{conn, unhandled}
-	}
 	cfg := discover.Config{
 		PrivateKey:  nodeKey,
 		NetRestrict: restrictList,
-		Unhandled:   unhandled,
-	}
-
-	if _, err := discover.ListenUDP(conn, ln, cfg); err != nil {
-		utils.Fatalf("%v", err)
 	}
-
 	if *runv5 {
-		if _, err = discv5.ListenUDP(nodeKey, sconn, "", restrictList); err != nil {
+		if _, err := discover.ListenV5(conn, ln, cfg); err != nil {
+			utils.Fatalf("%v", err)
+		}
+	} else {
+		if _, err := discover.ListenUDP(conn, ln, cfg); err != nil {
 			utils.Fatalf("%v", err)
 		}
 	}

cmd/cortex/chaincmd.go

@@ -38,7 +38,6 @@ import (
 	"github.com/CortexFoundation/CortexTheseus/event"
 	"github.com/CortexFoundation/CortexTheseus/log"
 	"github.com/CortexFoundation/CortexTheseus/trie"
-	//"github.com/syndtr/goleveldb/leveldb/util"
 	"gopkg.in/urfave/cli.v1"
 )
 

cmd/cortex/main.go

@@ -61,6 +61,8 @@ var (
 		utils.BootnodesV4Flag,
 		// utils.BootnodesV5Flag,
 		utils.DataDirFlag,
+		utils.AncientFlag,
+		utils.MinFreeDiskSpaceFlag,
 		utils.KeyStoreDirFlag,
 		utils.ExternalSignerFlag,
 		// utils.NoUSBFlag,
@@ -337,7 +339,7 @@ func startNode(ctx *cli.Context, stack *node.Node) {
 	debug.Memsize.Add("node", stack)
 
 	// Start up the node itself
-	utils.StartNode(stack)
+	utils.StartNode(ctx, stack)
 	var unlocks []string
 	inputs := strings.Split(ctx.GlobalString(utils.UnlockedAccountFlag.Name), ",")
 	for _, input := range inputs {

cmd/cortex/usage.go

@@ -69,6 +69,8 @@ var AppHelpFlagGroups = []flagGroup{
 		Flags: []cli.Flag{
 			configFileFlag,
 			utils.DataDirFlag,
+			utils.AncientFlag,
+			utils.MinFreeDiskSpaceFlag,
 			utils.KeyStoreDirFlag,
 			// utils.NoUSBFlag,
 			utils.NetworkIdFlag,

cmd/utils/cmd.go

@@ -26,17 +26,20 @@ import (
 	"runtime"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/CortexFoundation/CortexTheseus/common"
 	"github.com/CortexFoundation/CortexTheseus/core"
 	"github.com/CortexFoundation/CortexTheseus/core/rawdb"
 	"github.com/CortexFoundation/CortexTheseus/core/types"
 	"github.com/CortexFoundation/CortexTheseus/crypto"
+	"github.com/CortexFoundation/CortexTheseus/ctxc"
 	"github.com/CortexFoundation/CortexTheseus/ctxcdb"
 	"github.com/CortexFoundation/CortexTheseus/internal/debug"
 	"github.com/CortexFoundation/CortexTheseus/log"
 	"github.com/CortexFoundation/CortexTheseus/node"
 	"github.com/CortexFoundation/CortexTheseus/rlp"
+	"gopkg.in/urfave/cli.v1"
 )
 
 const (
@@ -63,14 +66,23 @@ func Fatalf(format string, args ...interface{}) {
 	os.Exit(1)
 }
 
-func StartNode(stack *node.Node) {
+func StartNode(ctx *cli.Context, stack *node.Node) {
 	if err := stack.Start(); err != nil {
 		Fatalf("Error starting protocol stack: %v", err)
 	}
 	go func() {
 		sigc := make(chan os.Signal, 1)
 		signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM)
 		defer signal.Stop(sigc)
+		minFreeDiskSpace := ctxc.DefaultConfig.TrieDirtyCache
+		if ctx.GlobalIsSet(MinFreeDiskSpaceFlag.Name) {
+			minFreeDiskSpace = ctx.GlobalInt(MinFreeDiskSpaceFlag.Name)
+		} else if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
+			minFreeDiskSpace = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
+		}
+		if minFreeDiskSpace > 0 {
+			go monitorFreeDiskSpace(sigc, stack.InstanceDir(), uint64(minFreeDiskSpace)*1024*1024)
+		}
 		<-sigc
 		log.Info("Got interrupt, shutting down...")
 		go stack.Stop()
@@ -85,6 +97,24 @@ func StartNode(stack *node.Node) {
 	}()
 }
 
+func monitorFreeDiskSpace(sigc chan os.Signal, path string, freeDiskSpaceCritical uint64) {
+	for {
+		freeSpace, err := getFreeDiskSpace(path)
+		if err != nil {
+			log.Warn("Failed to get free disk space", "path", path, "err", err)
+			break
+		}
+		if freeSpace < freeDiskSpaceCritical {
+			log.Error("Low disk space. Gracefully shutting down Geth to prevent database corruption.", "available", common.StorageSize(freeSpace))
+			sigc <- syscall.SIGTERM
+			break
+		} else if freeSpace < 2*freeDiskSpaceCritical {
+			log.Warn("Disk space is running low. Geth will shutdown if disk space runs below critical level.", "available", common.StorageSize(freeSpace), "critical_level", common.StorageSize(freeDiskSpaceCritical))
+		}
+		time.Sleep(60 * time.Second)
+	}
+}
+
 func ImportChain(chain *core.BlockChain, fn string) error {
 	// Watch for Ctrl-C while the import is running.
 	// If a signal is received, the import will stop at the next batch.

p2p/discv5/metrics.go renamed to cmd/utils/diskusage.go

@@ -1,4 +1,4 @@
-// Copyright 2018 The CortexTheseus Authors
+// Copyright 2021 The CortexTheseus Authors
 // This file is part of the CortexTheseus library.
 //
 // The CortexTheseus library is free software: you can redistribute it and/or modify
@@ -14,11 +14,22 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the CortexTheseus library. If not, see <http://www.gnu.org/licenses/>.
 
-package discv5
+// +build !windows
 
-import "github.com/CortexFoundation/CortexTheseus/metrics"
+package utils
 
-var (
-	ingressTrafficMeter = metrics.NewRegisteredMeter("discv5/InboundTraffic", nil)
-	egressTrafficMeter  = metrics.NewRegisteredMeter("discv5/OutboundTraffic", nil)
+import (
+	"fmt"
+
+	"golang.org/x/sys/unix"
 )
+
+func getFreeDiskSpace(path string) (uint64, error) {
+	var stat unix.Statfs_t
+	if err := unix.Statfs(path, &stat); err != nil {
+		return 0, fmt.Errorf("failed to call Statfs: %v", err)
+	}
+
+	// Available blocks * size per block = available space in bytes
+	return stat.Bavail * uint64(stat.Bsize), nil
+}

p2p/discv5/sim_testmain_test.go renamed to cmd/utils/diskusage_windows.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The CortexTheseus Authors
+// Copyright 2021 The CortexTheseus Authors
 // This file is part of the CortexTheseus library.
 //
 // The CortexTheseus library is free software: you can redistribute it and/or modify
@@ -14,30 +14,25 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the CortexTheseus library. If not, see <http://www.gnu.org/licenses/>.
 
-// +build go1.4,nacl,faketime_simulation
-
-package discv5
+package utils
 
 import (
-	"os"
-	"runtime"
-	"testing"
-	"unsafe"
+	"fmt"
+
+	"golang.org/x/sys/windows"
 )
 
-// Enable fake time mode in the runtime, like on the go playground.
-// There is a slight chance that this won't work because some go code
-// might have executed before the variable is set.
+func getFreeDiskSpace(path string) (uint64, error) {
 
-//go:linkname faketime runtime.faketime
-var faketime = 1
+	cwd, err := windows.UTF16PtrFromString(path)
+	if err != nil {
+		return 0, fmt.Errorf("failed to call UTF16PtrFromString: %v", err)
+	}
 
-func TestMain(m *testing.M) {
-	// We need to use unsafe somehow in order to get access to go:linkname.
-	_ = unsafe.Sizeof(0)
+	var freeBytesAvailableToCaller, totalNumberOfBytes, totalNumberOfFreeBytes uint64
+	if err := windows.GetDiskFreeSpaceEx(cwd, &freeBytesAvailableToCaller, &totalNumberOfBytes, &totalNumberOfFreeBytes); err != nil {
+		return 0, fmt.Errorf("failed to call GetDiskFreeSpaceEx: %v", err)
+	}
 
-	// Run the actual test. runWithPlaygroundTime ensures that the only test
-	// that runs is the one calling it.
-	runtime.GOMAXPROCS(8)
-	os.Exit(m.Run())
+	return freeBytesAvailableToCaller, nil
 }

cmd/utils/flags.go

@@ -54,7 +54,6 @@ import (
 	"github.com/CortexFoundation/CortexTheseus/metrics/influxdb"
 	"github.com/CortexFoundation/CortexTheseus/node"
 	"github.com/CortexFoundation/CortexTheseus/p2p"
-	"github.com/CortexFoundation/CortexTheseus/p2p/discv5"
 	"github.com/CortexFoundation/CortexTheseus/p2p/enode"
 	"github.com/CortexFoundation/CortexTheseus/p2p/nat"
 	"github.com/CortexFoundation/CortexTheseus/p2p/netutil"
@@ -134,14 +133,18 @@ var (
 		Usage: "Data directory for the databases and keystore",
 		Value: DirectoryString{node.DefaultDataDir()},
 	}
-	KeyStoreDirFlag = DirectoryFlag{
-		Name:  "keystore",
-		Usage: "Directory for the keystore (default = inside the datadir)",
-	}
 	AncientFlag = DirectoryFlag{
 		Name:  "datadir.ancient",
 		Usage: "Data directory for ancient chain segments (default = inside chaindata)",
 	}
+	MinFreeDiskSpaceFlag = DirectoryFlag{
+		Name:  "datadir.minfreedisk",
+		Usage: "Minimum free disk space in MB, once reached triggers auto shut down (default = --cache.gc converted to MB, 0 = disabled)",
+	}
+	KeyStoreDirFlag = DirectoryFlag{
+		Name:  "keystore",
+		Usage: "Directory for the keystore (default = inside the datadir)",
+	}
 	// NoUSBFlag = cli.BoolFlag{
 	// 	Name:  "nousb",
 	// 	Usage: "Disables monitoring for and managing USB hardware wallets",
@@ -901,26 +904,26 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) {
 // setBootstrapNodesV5 creates a list of bootstrap nodes from the command line
 // flags, reverting to pre-configured ones if none have been specified.
 func setBootstrapNodesV5(ctx *cli.Context, cfg *p2p.Config) {
-	urls := params.MainnetBootnodes
+	urls := params.V5Bootnodes
 	switch {
 	case ctx.GlobalIsSet(BootnodesFlag.Name) || ctx.GlobalIsSet(BootnodesV5Flag.Name):
 		if ctx.GlobalIsSet(BootnodesV5Flag.Name) {
 			urls = splitAndTrim(ctx.GlobalString(BootnodesV5Flag.Name))
 		} else {
 			urls = splitAndTrim(ctx.GlobalString(BootnodesFlag.Name))
 		}
-	case ctx.GlobalBool(BernardFlag.Name):
-		urls = params.BernardBootnodes
-	case ctx.GlobalBool(DoloresFlag.Name):
-		urls = params.BernardBootnodes
+	//case ctx.GlobalBool(BernardFlag.Name):
+	//	urls = params.BernardBootnodes
+	//case ctx.GlobalBool(DoloresFlag.Name):
+	//	urls = params.BernardBootnodes
 	case cfg.BootstrapNodesV5 != nil:
 		return // already set, don't apply defaults.
 	}
 
-	cfg.BootstrapNodesV5 = make([]*discv5.Node, 0, len(urls))
+	cfg.BootstrapNodesV5 = make([]*enode.Node, 0, len(urls))
 	for _, url := range urls {
 		if url != "" {
-			node, err := discv5.ParseNode(url)
+			node, err := enode.Parse(enode.ValidSchemes, url)
 			if err != nil {
 				log.Error("Bootstrap URL invalid", "enode", url, "err", err)
 				continue
@@ -1370,7 +1373,6 @@ func SetCortexConfig(ctx *cli.Context, stack *node.Node, cfg *ctxc.Config) {
 	if ctx.GlobalIsSet(NetworkIdFlag.Name) {
 		cfg.NetworkId = ctx.GlobalUint64(NetworkIdFlag.Name)
 	}
-
 	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheDatabaseFlag.Name) {
 		cfg.DatabaseCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100
 	}

common/mclock/mclock.go

@@ -20,15 +20,19 @@ package mclock
 import (
 	"time"
 
-	"github.com/aristanetworks/goarista/monotime"
+	_ "unsafe" // for go:linkname
 )
 
+//go:noescape
+//go:linkname nanotime runtime.nanotime
+func nanotime() int64
+
 // AbsTime represents absolute monotonic time.
-type AbsTime time.Duration
+type AbsTime int64
 
 // Now returns the current absolute monotonic time.
 func Now() AbsTime {
-	return AbsTime(monotime.Now())
+	return AbsTime(nanotime())
 }
 
 // Add returns t + d as absolute time.
@@ -76,11 +80,7 @@ type System struct {
 
 // Now returns the current monotonic time.
 func (c System) Now() AbsTime {
-	//return AbsTime(monotime.Now())
-	return AbsTime(monotime.Now()) + c.offset
-}
-func (c *System) SetAbsTime(now AbsTime) {
-	c.offset = now - AbsTime(monotime.Now())
+	return Now()
 }
 
 // Sleep blocks for the given duration.