feat(backup)_: poc for local user data backups

Fixes status-im/status-desktop#18107

Creates a proof of concept of the local user data backups.
When a back up is done, if the feature flag is active, a local file will be created with (almost) all the same backup data.
Then, that file can be passed to the import function and it will import that data.
A test case was added to test the flow using contacts and it works.

Author: jrainville

api/defaults.go

@@ -420,6 +420,10 @@ func DefaultNodeConfig(installationID, keyUID string, request *requests.CreateAc
 		overrideApiConfig(nodeConfig, request.APIConfig)
 	}
 
+	nodeConfig.BackupConfig = params.BackupConfig{
+		DataDir: filepath.Join(nodeConfig.RootDataDir, params.BackupsRelativePath),
+	}
+
 	for _, opt := range opts {
 		if err := opt(nodeConfig); err != nil {
 			return nil, err

params/config.go

@@ -395,6 +395,8 @@ type NodeConfig struct {
 
 	TorrentConfig TorrentConfig
 
+	BackupConfig BackupConfig
+
 	// RegisterTopics a list of specific topics where the peer wants to be
 	// discoverable.
 	RegisterTopics []discv5.Topic `json:"RegisterTopics"`
@@ -604,6 +606,12 @@ type TorrentConfig struct {
 	TorrentDir string
 }
 
+// BackupConfig provides configuration for the local backups
+type BackupConfig struct {
+	// DataDir is the file system folder Status should use for storing local backups.
+	DataDir string
+}
+
 // Validate validates the ShhextConfig struct and returns an error if inconsistent values are found
 func (c *ShhextConfig) Validate(validate *validator.Validate) error {
 	if err := validate.Struct(c); err != nil {
@@ -744,6 +752,10 @@ func (c *NodeConfig) UpdateWithDefaults() error {
 		}
 	}
 
+	c.BackupConfig = BackupConfig{
+		DataDir: filepath.Join(c.RootDataDir, BackupsRelativePath),
+	}
+
 	return c.setDefaultPushNotificationsServers()
 }
 

params/defaults.go

@@ -8,6 +8,7 @@ const (
 
 	ArchivesRelativePath        = "data/archivedata"
 	TorrentTorrentsRelativePath = "data/torrents"
+	BackupsRelativePath         = "backups"
 
 	// SendTransactionMethodName https://docs.walletconnect.com/advanced/rpc-reference/ethereum-rpc#eth_sendtransaction
 	SendTransactionMethodName = "eth_sendTransaction"

protocol/common/feature_flags.go

@@ -35,4 +35,7 @@ type FeatureFlags struct {
 
 	// EnableNewsFeed indicates whether we should enable the News Feed polling (this is not the user setting)
 	EnableNewsFeed bool
+
+	// EnableLocalBackup indicates whether we should also back up user data locally
+	EnableLocalBackup bool
 }

protocol/messenger.go

@@ -3525,6 +3525,13 @@ func (m *Messenger) saveDataAndPrepareResponse(messageState *ReceivedMessageStat
 		if ok {
 			contactsToSave = append(contactsToSave, contact)
 			messageState.Response.AddContact(contact)
+
+			_, ok := m.allContacts.Load(id)
+			if !ok {
+				// If the contact is not in the allContacts map, it means it's a new contact
+				// and we need to add it to the allContacts map.
+				m.allContacts.Store(id, contact)
+			}
 		}
 		return true
 	})

protocol/messenger_backup.go

@@ -2,6 +2,8 @@ package protocol
 
 import (
 	"context"
+	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/golang/protobuf/proto"
@@ -13,7 +15,9 @@ import (
 	"github.com/status-im/status-go/multiaccounts/settings"
 	"github.com/status-im/status-go/protocol/common"
 	"github.com/status-im/status-go/protocol/communities"
+	"github.com/status-im/status-go/protocol/encryption"
 	"github.com/status-im/status-go/protocol/protobuf"
+	v1protocol "github.com/status-im/status-go/protocol/v1"
 )
 
 const (
@@ -136,7 +140,7 @@ func (m *Messenger) BackupData(ctx context.Context) (uint64, error) {
 			},
 			ProfileDetails: &protobuf.FetchingBackedUpDataDetails{
 				DataNumber:  uint32(0),
-				TotalNumber: uint32(len(profileToBackup)),
+				TotalNumber: uint32(1), // Profile is always one
 			},
 			SettingsDetails: &protobuf.FetchingBackedUpDataDetails{
 				DataNumber:  uint32(0),
@@ -153,11 +157,14 @@ func (m *Messenger) BackupData(ctx context.Context) (uint64, error) {
 		}
 	}
 
+	fullBackup := &protobuf.Backup{}
+
 	// Update contacts messages encode and dispatch
 	for i, d := range contactsToBackup {
 		pb := backupDetailsOnly()
 		pb.ContactsDetails.DataNumber = uint32(i + 1)
 		pb.Contacts = d.Contacts
+		fullBackup.Contacts = append(fullBackup.Contacts, d.Contacts...)
 		err = m.encodeAndDispatchBackupMessage(ctx, pb, chat.ID)
 		if err != nil {
 			return 0, err
@@ -169,28 +176,29 @@ func (m *Messenger) BackupData(ctx context.Context) (uint64, error) {
 		pb := backupDetailsOnly()
 		pb.CommunitiesDetails.DataNumber = uint32(i + 1)
 		pb.Communities = d.Communities
+		fullBackup.Communities = append(fullBackup.Communities, d.Communities...)
 		err = m.encodeAndDispatchBackupMessage(ctx, pb, chat.ID)
 		if err != nil {
 			return 0, err
 		}
 	}
 
 	// Update profile messages encode and dispatch
-	for i, d := range profileToBackup {
-		pb := backupDetailsOnly()
-		pb.ProfileDetails.DataNumber = uint32(i + 1)
-		pb.Profile = d.Profile
-		err = m.encodeAndDispatchBackupMessage(ctx, pb, chat.ID)
-		if err != nil {
-			return 0, err
-		}
+	pb := backupDetailsOnly()
+	pb.ProfileDetails.DataNumber = uint32(1)
+	pb.Profile = profileToBackup.Profile
+	fullBackup.Profile = profileToBackup.Profile
+	err = m.encodeAndDispatchBackupMessage(ctx, pb, chat.ID)
+	if err != nil {
+		return 0, err
 	}
 
 	// Update chats encode and dispatch
 	for i, d := range chatsToBackup {
 		pb := backupDetailsOnly()
 		pb.ChatsDetails.DataNumber = uint32(i + 1)
 		pb.Chats = d.Chats
+		fullBackup.Chats = append(fullBackup.Chats, d.Chats...)
 		err = m.encodeAndDispatchBackupMessage(ctx, pb, chat.ID)
 		if err != nil {
 			return 0, err
@@ -202,6 +210,8 @@ func (m *Messenger) BackupData(ctx context.Context) (uint64, error) {
 		pb := backupDetailsOnly()
 		pb.SettingsDetails.DataNumber = uint32(i + 1)
 		pb.Setting = d
+		// TODO find a way to get all settings
+		// fullBackup.Setting = append(fullBackup.Setting, d)
 		err = m.encodeAndDispatchBackupMessage(ctx, pb, chat.ID)
 		if err != nil {
 			return 0, err
@@ -213,6 +223,8 @@ func (m *Messenger) BackupData(ctx context.Context) (uint64, error) {
 		pb := backupDetailsOnly()
 		pb.KeypairDetails.DataNumber = uint32(i + 1)
 		pb.Keypair = d.Keypair
+		// TODO find a way to get all settings
+		// fullBackup.Keypair = append(fullBackup.Keypair, d.Keypair)
 		err = m.encodeAndDispatchBackupMessage(ctx, pb, chat.ID)
 		if err != nil {
 			return 0, err
@@ -224,12 +236,49 @@ func (m *Messenger) BackupData(ctx context.Context) (uint64, error) {
 		pb := backupDetailsOnly()
 		pb.WatchOnlyAccountDetails.DataNumber = uint32(i + 1)
 		pb.WatchOnlyAccount = d.WatchOnlyAccount
+		// TODO find a way to get all settings
+		// fullBackup.WatchOnlyAccount = append(fullBackup.WatchOnlyAccount, d.Keypair)
 		err = m.encodeAndDispatchBackupMessage(ctx, pb, chat.ID)
 		if err != nil {
 			return 0, err
 		}
 	}
 
+	if m.config.featureFlags.EnableLocalBackup {
+		// TODO put file in a constant
+		path := filepath.Join(m.config.backupConfig.DataDir, "user_data.bkp")
+
+		if err := os.MkdirAll(m.config.backupConfig.DataDir, 0700); err != nil {
+			return 0, err
+		}
+
+		file, err := os.Create(path)
+		if err != nil {
+			return 0, err
+		}
+		defer file.Close()
+
+		mashalledMessage, err := proto.Marshal(fullBackup)
+		if err != nil {
+			return 0, err
+		}
+
+		messageSpec, err := m.encryptor.BuildDHMessage(m.identity, &m.identity.PublicKey, mashalledMessage)
+		if err != nil {
+			return 0, err
+		}
+
+		encryptedMessage, err := proto.Marshal(messageSpec.Message)
+		if err != nil {
+			return 0, err
+		}
+		err = os.WriteFile(path, encryptedMessage, 0600)
+		if err != nil {
+			m.logger.Error("failed to write backup message to file", zap.Error(err), zap.String("path", path))
+			return 0, err
+		}
+	}
+
 	chat.LastClockValue = clock
 	err = m.saveChat(chat)
 	if err != nil {
@@ -248,6 +297,62 @@ func (m *Messenger) BackupData(ctx context.Context) (uint64, error) {
 	return clockInSeconds, nil
 }
 
+func (m *Messenger) importLocalBackupFile(filePath string) (*MessengerResponse, error) {
+	if !m.config.featureFlags.EnableLocalBackup {
+		return nil, nil
+	}
+
+	// Make sure the backup file exists
+	content, err := os.ReadFile(filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	// Decrypt the backup file
+	// Unmarshal the content to get the message spec
+	var messageSpec encryption.ProtocolMessage
+	err = proto.Unmarshal(content, &messageSpec)
+	if err != nil {
+		return nil, err
+	}
+
+	// Decrypt the payload
+	var defaultMessageID = []byte("default")
+	decryptedPayload1, err := m.encryptor.HandleMessage(m.identity, &m.identity.PublicKey, &messageSpec, defaultMessageID)
+	if err != nil {
+		return nil, err
+	}
+
+	// Unmarshal the decrypted payload to get the backup message
+	var backupMessage protobuf.Backup
+	err = proto.Unmarshal(decryptedPayload1.DecryptedMessage, &backupMessage)
+	if err != nil {
+		return nil, err
+	}
+
+	// Handle the backup
+	state := ReceivedMessageState{
+		Response: &MessengerResponse{},
+		AllChats: &chatMap{},
+		AllContacts: &contactMap{
+			me: m.selfContact,
+		},
+		Timesource:            m.getTimesource(),
+		ModifiedContacts:      &stringBoolMap{},
+		ModifiedInstallations: &stringBoolMap{},
+	}
+	err = m.HandleBackup(
+		&state,
+		&backupMessage,
+		&v1protocol.StatusMessage{},
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return m.saveDataAndPrepareResponse(&state)
+}
+
 func (m *Messenger) encodeAndDispatchBackupMessage(ctx context.Context, message *protobuf.Backup, chatID string) error {
 	encodedMessage, err := proto.Marshal(message)
 	if err != nil {
@@ -447,7 +552,7 @@ func (m *Messenger) buildSyncContactMessage(contact *Contact) *protobuf.SyncInst
 	}
 }
 
-func (m *Messenger) backupProfile(ctx context.Context, clock uint64) ([]*protobuf.Backup, error) {
+func (m *Messenger) backupProfile(ctx context.Context, clock uint64) (*protobuf.Backup, error) {
 	displayName, err := m.settings.DisplayName()
 	if err != nil {
 		return nil, err
@@ -515,9 +620,7 @@ func (m *Messenger) backupProfile(ctx context.Context, clock uint64) ([]*protobu
 		},
 	}
 
-	backupMessages := []*protobuf.Backup{backupMessage}
-
-	return backupMessages, nil
+	return backupMessage, nil
 }
 
 func (m *Messenger) backupKeypairs() ([]*protobuf.Backup, error) {

protocol/messenger_backup_test.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"os"
+	"path/filepath"
 	"reflect"
 	"testing"
 	"time"
@@ -13,6 +15,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/event"
 
+	"github.com/status-im/status-go/params"
 	v1protocol "github.com/status-im/status-go/protocol/v1"
 	"github.com/status-im/status-go/protocol/wakusync"
 	"github.com/status-im/status-go/services/accounts/accountsevent"
@@ -110,6 +113,76 @@ func (s *MessengerBackupSuite) TestBackupContacts() {
 	s.Require().Equal(clock, lastBackup)
 }
 
+func (s *MessengerBackupSuite) TestBackupContactsLocally() {
+	backupOptions := []Option{
+		WithLocalBackup(&params.BackupConfig{
+			DataDir: filepath.Join(s.tmpdir, params.BackupsRelativePath),
+		}),
+	}
+
+	// Create bob1
+	privateKey, err := crypto.GenerateKey()
+	s.Require().NoError(err)
+	bob1, err := newMessengerWithKey(s.shh, privateKey, s.logger, backupOptions)
+	s.Require().NoError(err)
+	defer TearDownMessenger(&s.Suite, bob1)
+
+	// Create bob2
+	bob2, err := newMessengerWithKey(s.shh, bob1.identity, s.logger, backupOptions)
+	s.Require().NoError(err)
+	defer TearDownMessenger(&s.Suite, bob2)
+
+	// Make sure there is no backup at first
+	backupFile := filepath.Join(bob1.config.backupConfig.DataDir, "user_data.bkp")
+	err = os.RemoveAll(backupFile)
+	s.Require().NoError(err)
+
+	// Create 2 contacts
+	contact1Key, err := crypto.GenerateKey()
+	s.Require().NoError(err)
+	contactID1 := types.EncodeHex(crypto.FromECDSAPub(&contact1Key.PublicKey))
+
+	_, err = bob1.AddContact(context.Background(), &requests.AddContact{ID: contactID1})
+	s.Require().NoError(err)
+
+	contact2Key, err := crypto.GenerateKey()
+	s.Require().NoError(err)
+	contactID2 := types.EncodeHex(crypto.FromECDSAPub(&contact2Key.PublicKey))
+
+	_, err = bob1.AddContact(context.Background(), &requests.AddContact{ID: contactID2})
+	s.Require().NoError(err)
+
+	s.Require().Len(bob1.Contacts(), 2)
+
+	actualContacts := bob1.Contacts()
+	if actualContacts[0].ID == contactID1 {
+		s.Require().Equal(actualContacts[0].ID, contactID1)
+		s.Require().Equal(actualContacts[1].ID, contactID2)
+	} else {
+		s.Require().Equal(actualContacts[0].ID, contactID2)
+		s.Require().Equal(actualContacts[1].ID, contactID1)
+	}
+
+	s.Require().Equal(ContactRequestStateSent, actualContacts[0].ContactRequestLocalState)
+	s.Require().Equal(ContactRequestStateSent, actualContacts[1].ContactRequestLocalState)
+	s.Require().True(actualContacts[0].added())
+	s.Require().True(actualContacts[1].added())
+
+	// Backup
+	_, err = bob1.BackupData(context.Background())
+	s.Require().NoError(err)
+
+	// Safety check
+	s.Require().Len(bob2.Contacts(), 0)
+
+	// Import the backup file and process it
+	response, err := bob2.importLocalBackupFile(backupFile)
+	s.Require().NoError(err)
+	s.Require().NotNil(response)
+	s.Require().Len(response.Contacts, 2)
+	s.Require().Len(bob2.Contacts(), 2)
+}
+
 func (s *MessengerBackupSuite) TestBackupProfile() {
 	const bob1DisplayName = "bobby"