Merge branch 'main' into release-lts-1

api/censuses.go

@@ -36,7 +36,7 @@ const (
 	MaxCensusAddBatchSize = 8192
 
 	censusIDsize          = 32
-	censusRetrieveTimeout = 5 * time.Minute
+	censusRetrieveTimeout = 10 * time.Minute
 )
 
 func (a *API) enableCensusHandlers() error {
@@ -170,6 +170,14 @@ func (a *API) enableCensusHandlers() error {
 	); err != nil {
 		return err
 	}
+	if err := a.Endpoint.RegisterMethod(
+		"/censuses/export/ipfs/list",
+		"GET",
+		apirest.MethodAccessTypeAdmin,
+		a.censusExportIPFSListDBHandler,
+	); err != nil {
+		return err
+	}
 	if err := a.Endpoint.RegisterMethod(
 		"/censuses/export",
 		"GET",
@@ -203,6 +211,9 @@ func (a *API) enableCensusHandlers() error {
 		return err
 	}
 
+	// Initialize the map to store the status of the async export to ipfs
+	censusIPFSExports = make(map[string]time.Time)
+
 	return nil
 }
 
@@ -972,6 +983,9 @@ func (a *API) censusListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext)
 	return ctx.Send(data, apirest.HTTPstatusOK)
 }
 
+// censusIPFSExports is a map of ipfs uri to the time when the export was requested
+var censusIPFSExports = map[string]time.Time{}
+
 // censusExportDBHandler
 //
 //	@Summary		Export census database
@@ -986,27 +1000,58 @@ func (a *API) censusListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext)
 func (a *API) censusExportDBHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error {
 	isIPFSExport := strings.HasSuffix(ctx.Request.URL.Path, "ipfs")
 	buf := bytes.Buffer{}
-	if err := a.censusdb.ExportCensusDB(&buf); err != nil {
-		return err
-	}
 	var data []byte
 	if isIPFSExport {
-		uri, err := a.storage.PublishReader(ctx.Request.Context(), &buf)
-		if err != nil {
-			return err
-		}
+		go func() {
+			log.Infow("exporting census database to ipfs async")
+			startTime := time.Now()
+			if err := a.censusdb.ExportCensusDB(&buf); err != nil {
+				log.Errorw(err, "could not export census database")
+				return
+			}
+			log.Infow("census database exported", "duration (s)", time.Since(startTime).Seconds())
+			startTime = time.Now()
+			uri, err := a.storage.PublishReader(context.Background(), &buf)
+			if err != nil {
+				log.Errorw(err, "could not publish census database to ipfs")
+				return
+			}
+			log.Infow("census database published to ipfs", "uri", uri, "duration (s)", time.Since(startTime).Seconds())
+			censusIPFSExports[uri] = time.Now()
+		}()
+		var err error
 		data, err = json.Marshal(map[string]string{
-			"uri": uri,
+			"message": "scheduled, check /censuses/export/ipfs/list",
 		})
 		if err != nil {
-			return err
+			log.Errorw(err, "could not marshal response")
 		}
 	} else {
+		if err := a.censusdb.ExportCensusDB(&buf); err != nil {
+			return err
+		}
 		data = buf.Bytes()
 	}
 	return ctx.Send(data, apirest.HTTPstatusOK)
 }
 
+// censusExportIPFSListDBHandler
+//
+//	@Summary		List export census database to IPFS
+//	@Description	List the IPFS URIs of the census database exports
+//	@Tags			Censuses
+//	@Accept			json
+//	@Produce		json
+//	@Success		200	{object}	object{valid=bool}
+//	@Router			/censuses/export/ipfs/list [get]
+func (a *API) censusExportIPFSListDBHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error {
+	data, err := json.Marshal(censusIPFSExports)
+	if err != nil {
+		return err
+	}
+	return ctx.Send(data, apirest.HTTPstatusOK)
+}
+
 // censusImportHandler
 //
 //	@Summary		Import census database

api/helpers.go

@@ -1,12 +1,16 @@
 package api
 
 import (
+	"bytes"
+	"encoding/base64"
 	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"math"
 	"math/big"
+	"reflect"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -71,9 +75,54 @@ func protoTxAsJSON(tx []byte) []byte {
 	if err != nil {
 		panic(err)
 	}
+	// protojson follows protobuf's json mapping behavior,
+	// which requires bytes to be encoded as base64:
+	// https://protobuf.dev/programming-guides/proto3/#json
+	//
+	// We want hex rather than base64 for consistency with our REST API
+	// protojson does not expose any option to configure its behavior,
+	// and as the Go types are code generated, we cannot use our HexBytes type.
+	//
+	// Do a bit of a hack: walk the protobuf value with reflection,
+	// find all []byte values, and search-and-replace their base64 encoding with hex
+	// in the protojson bytes. This can be slow if we have many byte slices,
+	// but in general the protobuf message will be small so there will be few.
+	bytesValues := collectBytesValues(nil, reflect.ValueOf(&ptx))
+	for _, bv := range bytesValues {
+		asBase64 := base64.StdEncoding.AppendEncode(nil, bv)
+		asHex := hex.AppendEncode(nil, bv)
+		asJSON = bytes.Replace(asJSON, asBase64, asHex, 1)
+	}
 	return asJSON
 }
 
+var typBytes = reflect.TypeFor[[]byte]()
+
+func collectBytesValues(result [][]byte, val reflect.Value) [][]byte {
+	typ := val.Type()
+	if typ == typBytes {
+		return append(result, val.Bytes())
+	}
+	switch typ.Kind() {
+	case reflect.Pointer, reflect.Interface:
+		if !val.IsNil() {
+			result = collectBytesValues(result, val.Elem())
+		}
+	case reflect.Struct:
+		for i := 0; i < val.NumField(); i++ {
+			if !typ.Field(i).IsExported() {
+				continue
+			}
+			result = collectBytesValues(result, val.Field(i))
+		}
+	case reflect.Slice, reflect.Array:
+		for i := 0; i < val.Len(); i++ {
+			result = collectBytesValues(result, val.Index(i))
+		}
+	}
+	return result
+}
+
 // isTransactionType checks if the given transaction is of the given type.
 // t is expected to be a pointer to a protobuf transaction message.
 func isTransactionType[T any](signedTxBytes []byte) (bool, error) {
@@ -158,13 +207,13 @@ func encodeEVMResultsArgs(electionId common.Hash, organizationId common.Address,
 
 // decryptVotePackage decrypts a vote package using the given private keys and indexes.
 func decryptVotePackage(vp []byte, privKeys []string, indexes []uint32) ([]byte, error) {
-	for i := len(indexes) - 1; i >= 0; i-- {
-		if indexes[i] >= uint32(len(privKeys)) {
-			return nil, fmt.Errorf("invalid key index %d", indexes[i])
+	for _, index := range slices.Backward(indexes) {
+		if index >= uint32(len(privKeys)) {
+			return nil, fmt.Errorf("invalid key index %d", index)
 		}
-		priv, err := nacl.DecodePrivate(privKeys[indexes[i]])
+		priv, err := nacl.DecodePrivate(privKeys[index])
 		if err != nil {
-			return nil, fmt.Errorf("cannot decode encryption key with index %d: (%s)", indexes[i], err)
+			return nil, fmt.Errorf("cannot decode encryption key with index %d: (%s)", index, err)
 		}
 		vp, err = priv.Decrypt(vp)
 		if err != nil {

api/helpers_test.go

@@ -156,7 +156,7 @@ func TestConvertKeysToCamel(t *testing.T) {
 }
 
 func TestProtoTxAsJSON(t *testing.T) {
-	wantJSON := strings.TrimSpace(`
+	inputJSON := strings.TrimSpace(`
 {
 	"setProcess": {
 		"txtype": "SET_PROCESS_CENSUS",
@@ -167,9 +167,21 @@ func TestProtoTxAsJSON(t *testing.T) {
 		"censusSize": "1000"
 	}
 }
+`)
+	wantJSON := strings.TrimSpace(`
+{
+	"setProcess": {
+		"txtype": "SET_PROCESS_CENSUS",
+		"nonce": 1,
+		"processId": "b31dff61814dab90d6c3a9b65d6c90830480e4b75ad32e747942020000000000",
+		"censusRoot": "cd453d05c4cb0429ee5ee1aefed016f553b8026b4ced5b0c352905bfa53c7e81",
+		"censusURI": "ipfs://bafybeicyfukarcryrvy5oe37ligmxwf55sbfiojori4t25wencma4ymxfa",
+		"censusSize": "1000"
+	}
+}
 `)
 	var ptx models.Tx
-	err := protojson.Unmarshal([]byte(wantJSON), &ptx)
+	err := protojson.Unmarshal([]byte(inputJSON), &ptx)
 	qt.Assert(t, err, qt.IsNil)
 
 	asProto, err := proto.Marshal(&ptx)

data/ipfs/ipfs.go

@@ -37,7 +37,7 @@ import (
 
 const (
 	// MaxFileSizeBytes is the maximum size of a file to be published to IPFS
-	MaxFileSizeBytes = 1024 * 1024 * 100 // 100 MB
+	MaxFileSizeBytes = 1024 * 1024 * 1024 // 1 GB
 	// RetrievedFileCacheSize is the maximum number of files to be cached in memory
 	RetrievedFileCacheSize = 128
 )

go.mod

@@ -42,7 +42,7 @@ require (
 	github.com/libp2p/go-libp2p-pubsub v0.11.0
 	github.com/libp2p/go-reuseport v0.4.0
 	github.com/manifoldco/promptui v0.9.0
-	github.com/mattn/go-sqlite3 v1.14.22
+	github.com/mattn/go-sqlite3 v1.14.23
 	github.com/multiformats/go-multiaddr v0.12.4
 	github.com/multiformats/go-multicodec v0.9.0
 	github.com/multiformats/go-multihash v0.2.3
@@ -56,7 +56,6 @@ require (
 	go.mongodb.org/mongo-driver v1.12.1
 	go.vocdoni.io/proto v1.15.10-0.20240903073233-86144b1e2165
 	golang.org/x/crypto v0.26.0
-	golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa
 	golang.org/x/net v0.28.0
 	google.golang.org/protobuf v1.34.2
 	lukechampine.com/blake3 v1.3.0
@@ -327,6 +326,7 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	go4.org v0.0.0-20230225012048-214862532bf5 // indirect
+	golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect
 	golang.org/x/mod v0.20.0 // indirect
 	golang.org/x/oauth2 v0.22.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect

go.sum

@@ -970,8 +970,8 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m
 github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
 github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
-github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
-github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/mattn/go-sqlite3 v1.14.23 h1:gbShiuAP1W5j9UOksQ06aiiqPMxYecovVGwmTxWtuw0=
+github.com/mattn/go-sqlite3 v1.14.23/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=

tree/arbo/proof.go

@@ -5,6 +5,7 @@ import (
 	"encoding/binary"
 	"fmt"
 	"math"
+	"slices"
 
 	"go.vocdoni.io/dvote/db"
 )
@@ -173,21 +174,15 @@ func CheckProof(hashFunc HashFunction, k, v, root, packedSiblings []byte) (bool,
 	}
 
 	path := getPath(len(siblings), keyPath)
-	for i := len(siblings) - 1; i >= 0; i-- {
+	for i, sibling := range slices.Backward(siblings) {
 		if path[i] {
-			key, _, err = newIntermediate(hashFunc, siblings[i], key)
-			if err != nil {
-				return false, err
-			}
+			key, _, err = newIntermediate(hashFunc, sibling, key)
 		} else {
-			key, _, err = newIntermediate(hashFunc, key, siblings[i])
-			if err != nil {
-				return false, err
-			}
+			key, _, err = newIntermediate(hashFunc, key, sibling)
+		}
+		if err != nil {
+			return false, err
 		}
 	}
-	if bytes.Equal(key, root) {
-		return true, nil
-	}
-	return false, nil
+	return bytes.Equal(key, root), nil
 }