From 551ddff57c57f797e49edeb6f192543f4a455d4d Mon Sep 17 00:00:00 2001 From: TimeEngineer Date: Fri, 31 Mar 2023 11:53:11 +0200 Subject: [PATCH] Oxidize SequesterCrypto --- .github/workflows/ci.yml | 8 ++ Cargo.lock | 38 ++++---- .../libparsec/crates/client_types/Cargo.toml | 2 +- .../crates/client_types/src/local_device.rs | 2 +- .../crates/client_types/src/local_manifest.rs | 2 +- .../libparsec/crates/protocol/Cargo.toml | 2 +- .../libparsec/crates/protocol/src/lib.rs | 2 +- .../crates/serialization_format/Cargo.toml | 2 +- .../src/old_parser/utils.rs | 2 + oxidation/libparsec/crates/types/Cargo.toml | 2 +- .../libparsec/crates/types/src/certif.rs | 2 +- .../libparsec/crates/types/src/invite.rs | 2 +- .../libparsec/crates/types/src/manifest.rs | 2 +- oxidation/libparsec/crates/types/src/pki.rs | 2 +- parsec/__init__.py | 16 ---- parsec/_parsec.pyi | 8 ++ parsec/api/data/certif.py | 7 +- parsec/backend/cli/sequester.py | 35 ++++--- parsec/backend/organization.py | 4 +- parsec/backend/postgresql/organization.py | 4 +- parsec/backend/postgresql/sequester.py | 3 +- parsec/core/cli/bootstrap_organization.py | 6 +- parsec/core/invite/organization.py | 3 +- parsec/sequester_crypto.py | 18 ---- parsec/sequester_export_reader.py | 26 +++--- parsec/serde/fields.py | 18 ++-- tests/backend/sequester/test_crypto.py | 91 ------------------- tests/backend/sequester/test_export.py | 12 +-- tests/common/sequester.py | 46 ++++------ tests/core/fs/test_sequester.py | 9 +- tests/core/test_bootstrap_organization.py | 13 +-- tests/schemas/api_data.json | 2 +- tests/test_cli.py | 11 +-- 33 files changed, 132 insertions(+), 270 deletions(-) delete mode 100644 parsec/sequester_crypto.py delete mode 100644 tests/backend/sequester/test_crypto.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fa2bbc068d..395b9a74c8c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -459,6 +459,14 @@ jobs: shell: bash run: cargo test --workspace --profile ${{ env.CARGO_PROFILE }} --no-run + # Building OpenSSL requires a perl interpreter. + # The default one does not provide windows-style filesystem + # paths so we have to switch to Strawberry. + - name: Use strawberry perl + if: startsWith(matrix.os, 'windows') + shell: bash + run: echo OPENSSL_SRC_PERL=C:/Strawberry/perl/bin/perl >> $GITHUB_ENV + - name: Test rust codebase if: steps.rust-changes.outputs.run == 'true' shell: bash diff --git a/Cargo.lock b/Cargo.lock index 7c43169d25f..078b16aca2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1479,6 +1479,7 @@ dependencies = [ "glob", "hex-literal", "libparsec_crypto", + "libparsec_serialization_format", "libparsec_types", "paste", "pretty_assertions", @@ -1490,7 +1491,6 @@ dependencies = [ "serde_bytes", "serde_test", "serde_with", - "serialization_format", "sha2 0.10.6", "tests_fixtures", "thiserror", @@ -1611,6 +1611,7 @@ dependencies = [ "glob", "hex-literal", "libparsec_crypto", + "libparsec_serialization_format", "libparsec_types", "paste", "rand 0.8.5", @@ -1619,11 +1620,26 @@ dependencies = [ "serde", "serde_json", "serde_with", - "serialization_format", "tests_fixtures", "thiserror", ] +[[package]] +name = "libparsec_serialization_format" +version = "0.0.0" +dependencies = [ + "anyhow", + "itertools", + "miniserde", + "pretty_assertions", + "proc-macro2", + "quote", + "rstest 0.15.0", + "serde", + "serde_json", + "syn", +] + [[package]] name = "libparsec_types" version = "0.0.0" @@ -1638,6 +1654,7 @@ dependencies = [ "lazy_static", "libparsec_crypto", "libparsec_platform_async", + "libparsec_serialization_format", "paste", "percent-encoding", "pretty_assertions", @@ -1651,7 +1668,6 @@ dependencies = [ "serde_json", "serde_test", "serde_with", - "serialization_format", "tests_fixtures", "thiserror", "unicode-normalization", @@ -2828,22 +2844,6 @@ dependencies = [ "syn", ] -[[package]] -name = "serialization_format" -version = "0.0.0" -dependencies = [ - "anyhow", - "itertools", - "miniserde", - "pretty_assertions", - "proc-macro2", - "quote", - "rstest 0.15.0", - "serde", - "serde_json", - "syn", -] - [[package]] name = "sha-1" version = "0.8.2" diff --git a/oxidation/libparsec/crates/client_types/Cargo.toml b/oxidation/libparsec/crates/client_types/Cargo.toml index a49b611e296..33983bf89b0 100644 --- a/oxidation/libparsec/crates/client_types/Cargo.toml +++ b/oxidation/libparsec/crates/client_types/Cargo.toml @@ -14,7 +14,7 @@ path = "tests/mod.rs" [dependencies] libparsec_crypto = { path = "../crypto" } libparsec_types = { path = "../types" } -serialization_format = { path = "../serialization_format" } +libparsec_serialization_format = { path = "../serialization_format" } flate2 = "1.0.24" serde = { version = "1.0.147", features = ["derive"] } diff --git a/oxidation/libparsec/crates/client_types/src/local_device.rs b/oxidation/libparsec/crates/client_types/src/local_device.rs index 7eec7170f96..0270ff95b40 100644 --- a/oxidation/libparsec/crates/client_types/src/local_device.rs +++ b/oxidation/libparsec/crates/client_types/src/local_device.rs @@ -2,10 +2,10 @@ use serde::{Deserialize, Serialize}; use serde_with::*; -use serialization_format::parsec_data; use sha2::Digest; use libparsec_crypto::prelude::*; +use libparsec_serialization_format::parsec_data; use libparsec_types::*; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] diff --git a/oxidation/libparsec/crates/client_types/src/local_manifest.rs b/oxidation/libparsec/crates/client_types/src/local_manifest.rs index 8c5435d3a50..a14881f9bd1 100644 --- a/oxidation/libparsec/crates/client_types/src/local_manifest.rs +++ b/oxidation/libparsec/crates/client_types/src/local_manifest.rs @@ -9,8 +9,8 @@ use std::{ }; use libparsec_crypto::{HashDigest, SecretKey}; +use libparsec_serialization_format::parsec_data; use libparsec_types::*; -use serialization_format::parsec_data; use crate as libparsec_client_types; diff --git a/oxidation/libparsec/crates/protocol/Cargo.toml b/oxidation/libparsec/crates/protocol/Cargo.toml index 082a905224f..ddaeb27be1f 100644 --- a/oxidation/libparsec/crates/protocol/Cargo.toml +++ b/oxidation/libparsec/crates/protocol/Cargo.toml @@ -14,7 +14,7 @@ path = "tests/mod.rs" [dependencies] libparsec_crypto = { path = "../crypto" } libparsec_types = { path = "../types" } -serialization_format = { path = "../serialization_format" } +libparsec_serialization_format = { path = "../serialization_format" } paste = "1.0.9" rand = "0.8.5" diff --git a/oxidation/libparsec/crates/protocol/src/lib.rs b/oxidation/libparsec/crates/protocol/src/lib.rs index 947f9c1a07b..6842381dd79 100644 --- a/oxidation/libparsec/crates/protocol/src/lib.rs +++ b/oxidation/libparsec/crates/protocol/src/lib.rs @@ -6,7 +6,7 @@ mod handshake; use serde::{Deserialize, Serialize}; use std::num::NonZeroU8; -use serialization_format::parsec_protocol; +use libparsec_serialization_format::parsec_protocol; pub use error::*; pub use handshake::*; diff --git a/oxidation/libparsec/crates/serialization_format/Cargo.toml b/oxidation/libparsec/crates/serialization_format/Cargo.toml index cdc547c62a6..1030e8eafbc 100644 --- a/oxidation/libparsec/crates/serialization_format/Cargo.toml +++ b/oxidation/libparsec/crates/serialization_format/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "serialization_format" +name = "libparsec_serialization_format" version = "0.0.0" edition = "2021" license = " BUSL-1.1" diff --git a/oxidation/libparsec/crates/serialization_format/src/old_parser/utils.rs b/oxidation/libparsec/crates/serialization_format/src/old_parser/utils.rs index 75546411b96..f8e6e7a25e3 100644 --- a/oxidation/libparsec/crates/serialization_format/src/old_parser/utils.rs +++ b/oxidation/libparsec/crates/serialization_format/src/old_parser/utils.rs @@ -35,6 +35,8 @@ pub(crate) fn _inspect_type(ty: &Type, types: &HashMap) -> Strin "PrivateKey" => "libparsec_crypto::PrivateKey", "SecretKey" => "libparsec_crypto::SecretKey", "HashDigest" => "libparsec_crypto::HashDigest", + "SequesterVerifyKeyDer" => "libparsec_crypto::SequesterVerifyKeyDer", + "SequesterPublicKeyDer" => "libparsec_crypto::SequesterPublicKeyDer", "DateTime" => "libparsec_types::DateTime", "BlockID" => "libparsec_types::BlockID", "DeviceID" => "libparsec_types::DeviceID", diff --git a/oxidation/libparsec/crates/types/Cargo.toml b/oxidation/libparsec/crates/types/Cargo.toml index e7aef7f7ad5..5fd0a59555f 100644 --- a/oxidation/libparsec/crates/types/Cargo.toml +++ b/oxidation/libparsec/crates/types/Cargo.toml @@ -12,7 +12,7 @@ path = "tests/mod.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -serialization_format = { path = "../serialization_format" } +libparsec_serialization_format = { path = "../serialization_format" } libparsec_crypto = { path = "../crypto" } libparsec_platform_async = { path = "../platform_async" } diff --git a/oxidation/libparsec/crates/types/src/certif.rs b/oxidation/libparsec/crates/types/src/certif.rs index 87090c53c3d..ed322b4fb3e 100644 --- a/oxidation/libparsec/crates/types/src/certif.rs +++ b/oxidation/libparsec/crates/types/src/certif.rs @@ -7,7 +7,7 @@ use serde_with::*; use std::io::{Read, Write}; use libparsec_crypto::{PublicKey, SigningKey, VerifyKey}; -use serialization_format::parsec_data; +use libparsec_serialization_format::parsec_data; use crate as libparsec_types; use crate::data_macros::impl_transparent_data_format_conversion; diff --git a/oxidation/libparsec/crates/types/src/invite.rs b/oxidation/libparsec/crates/types/src/invite.rs index c2acb1d8d8a..ed61bbb81d5 100644 --- a/oxidation/libparsec/crates/types/src/invite.rs +++ b/oxidation/libparsec/crates/types/src/invite.rs @@ -7,7 +7,7 @@ use serde_with::*; use std::str::FromStr; use libparsec_crypto::{PrivateKey, PublicKey, SecretKey, VerifyKey}; -use serialization_format::parsec_data; +use libparsec_serialization_format::parsec_data; use crate::{ self as libparsec_types, data_macros::impl_transparent_data_format_conversion, DeviceID, diff --git a/oxidation/libparsec/crates/types/src/manifest.rs b/oxidation/libparsec/crates/types/src/manifest.rs index aec1b105040..f96fc838602 100644 --- a/oxidation/libparsec/crates/types/src/manifest.rs +++ b/oxidation/libparsec/crates/types/src/manifest.rs @@ -12,7 +12,7 @@ use std::{ use unicode_normalization::UnicodeNormalization; use libparsec_crypto::{HashDigest, SecretKey, SigningKey, VerifyKey}; -use serialization_format::parsec_data; +use libparsec_serialization_format::parsec_data; use crate::{ self as libparsec_types, data_macros::impl_transparent_data_format_conversion, BlockID, diff --git a/oxidation/libparsec/crates/types/src/pki.rs b/oxidation/libparsec/crates/types/src/pki.rs index f368e3d84f1..66612f621dd 100644 --- a/oxidation/libparsec/crates/types/src/pki.rs +++ b/oxidation/libparsec/crates/types/src/pki.rs @@ -6,7 +6,7 @@ use flate2::{read::ZlibDecoder, write::ZlibEncoder, Compression}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use libparsec_crypto::{PublicKey, VerifyKey}; -use serialization_format::parsec_data; +use libparsec_serialization_format::parsec_data; use crate::{ self as libparsec_types, impl_transparent_data_format_conversion, DataError, DataResult, diff --git a/parsec/__init__.py b/parsec/__init__.py index 6da925f2a77..73bdf7b37e2 100644 --- a/parsec/__init__.py +++ b/parsec/__init__.py @@ -1,24 +1,8 @@ # Parsec Cloud (https://parsec.cloud) Copyright (c) AGPL-3.0 2016-present Scille SAS from __future__ import annotations -import os - from parsec._version import __version__ -# The oscrypto library relies on `ctypes.util.find_library`, -# which doesn't work for snap classic environments. -# Hence, we rely on env variables similar to `FUSE_LIBRARY_PATH` -# to configure oscrypto correctly if those variables are provided. -SSL_LIBRARY_PATH = os.environ.get("SSL_LIBRARY_PATH") -CRYPTO_LIBRARY_PATH = os.environ.get("CRYPTO_LIBRARY_PATH") -if SSL_LIBRARY_PATH and CRYPTO_LIBRARY_PATH: - import oscrypto - - oscrypto.use_openssl( - libcrypto_path=CRYPTO_LIBRARY_PATH, - libssl_path=SSL_LIBRARY_PATH, - ) - # The parsec.utils module includes a bit of patching, let's make sure it is imported __import__("parsec.utils") diff --git a/parsec/_parsec.pyi b/parsec/_parsec.pyi index d39d9b0482a..a00969dac7a 100644 --- a/parsec/_parsec.pyi +++ b/parsec/_parsec.pyi @@ -24,6 +24,10 @@ from parsec._parsec_pyi.crypto import ( PrivateKey, PublicKey, SecretKey, + SequesterPrivateKeyDer, + SequesterPublicKeyDer, + SequesterSigningKeyDer, + SequesterVerifyKeyDer, SigningKey, VerifyKey, generate_nonce, @@ -510,6 +514,10 @@ __all__ = [ "VerifyKey", "PrivateKey", "PublicKey", + "SequesterPrivateKeyDer", + "SequesterPublicKeyDer", + "SequesterSigningKeyDer", + "SequesterVerifyKeyDer", "generate_nonce", # Enumerate "ClientType", diff --git a/parsec/api/data/certif.py b/parsec/api/data/certif.py index 874d66f333b..aadc4e684b0 100644 --- a/parsec/api/data/certif.py +++ b/parsec/api/data/certif.py @@ -5,10 +5,9 @@ import attr -from parsec._parsec import DateTime +from parsec._parsec import DateTime, SequesterPublicKeyDer, SequesterVerifyKeyDer from parsec.api.data.base import BaseAPIData, BaseAPISignedData, BaseSignedDataSchema from parsec.api.protocol import SequesterServiceID, SequesterServiceIDField -from parsec.sequester_crypto import SequesterEncryptionKeyDer, SequesterVerifyKeyDer from parsec.serde import fields, post_load from parsec.serde.schema import BaseSchema @@ -41,7 +40,7 @@ class SCHEMA_CLS(BaseSchema): timestamp = fields.DateTime(required=True) service_id = SequesterServiceIDField(required=True) service_label = fields.String(required=True) - encryption_key_der = fields.SequesterEncryptionKeyDerField(required=True) + encryption_key_der = fields.SequesterPublicKeyDerField(required=True) @post_load def make_obj(self, data: Dict[str, Any]) -> "SequesterServiceCertificate": # type: ignore[misc] @@ -51,4 +50,4 @@ def make_obj(self, data: Dict[str, Any]) -> "SequesterServiceCertificate": # ty timestamp: DateTime service_id: SequesterServiceID service_label: str - encryption_key_der: SequesterEncryptionKeyDer + encryption_key_der: SequesterPublicKeyDer diff --git a/parsec/backend/cli/sequester.py b/parsec/backend/cli/sequester.py index cb36b96b6ec..5d3a2bc1f51 100644 --- a/parsec/backend/cli/sequester.py +++ b/parsec/backend/cli/sequester.py @@ -8,11 +8,15 @@ import attr import click -import oscrypto from async_generator import asynccontextmanager -from oscrypto.asymmetric import PrivateKey -from parsec._parsec import DateTime +from parsec._parsec import ( + DateTime, + SequesterPrivateKeyDer, + SequesterPublicKeyDer, + SequesterSigningKeyDer, + SequesterVerifyKeyDer, +) from parsec.api.data import DataError, SequesterServiceCertificate from parsec.api.protocol import HumanHandle, OrganizationID, RealmID, SequesterServiceID, UserID from parsec.backend.blockstore import blockstore_factory @@ -33,13 +37,8 @@ ) from parsec.backend.user import User from parsec.cli_utils import cli_exception_handler, debug_config_options, operation +from parsec.crypto import CryptoError from parsec.event_bus import EventBus -from parsec.sequester_crypto import ( - CryptoError, - SequesterEncryptionKeyDer, - SequesterVerifyKeyDer, - sequester_authority_sign, -) from parsec.sequester_export_reader import RealmExportProgress, extract_workspace from parsec.utils import open_service_nursery, trio_run @@ -50,11 +49,9 @@ def dump_sequester_service_certificate_pem( certificate_data: SequesterServiceCertificate, - authority_signing_key: PrivateKey, + authority_signing_key: SequesterSigningKeyDer, ) -> str: - certificate = sequester_authority_sign( - signing_key=authority_signing_key, data=certificate_data.dump() - ) + certificate = authority_signing_key.sign(certificate_data.dump()) return "\n".join( ( SEQUESTER_SERVICE_CERTIFICATE_PEM_HEADER, @@ -217,8 +214,8 @@ def generate_service_certificate( ) -> None: with cli_exception_handler(debug): # Load key files - service_key = SequesterEncryptionKeyDer(service_public_key.read_bytes()) - authority_key = oscrypto.asymmetric.load_private_key(authority_private_key.read_bytes()) + service_key = SequesterPublicKeyDer.load_pem(service_public_key.read_text()) + authority_key = SequesterSigningKeyDer.load_pem(authority_private_key.read_text()) # Generate data schema service_id = SequesterServiceID.new() @@ -427,8 +424,8 @@ def create_service( "Webhook sequester service requires webhook_url argument" ) # Load key files - service_key = SequesterEncryptionKeyDer(service_public_key.read_bytes()) - authority_key = oscrypto.asymmetric.load_private_key(authority_private_key.read_bytes()) + service_key = SequesterPublicKeyDer.load_pem(service_public_key.read_text()) + authority_key = SequesterSigningKeyDer.load_pem(authority_private_key.read_text()) # Generate data schema service_id = SequesterServiceID.new() now = DateTime.now() @@ -438,7 +435,7 @@ def create_service( service_label=service_label, encryption_key_der=service_key, ) - certificate = sequester_authority_sign(signing_key=authority_key, data=certif_data.dump()) + certificate = authority_key.sign(certif_data.dump()) sequester_service: BaseSequesterService if cooked_service_type == SequesterServiceType.STORAGE: @@ -797,7 +794,7 @@ def extract_realm_export( # Finally a command that is not async ! # This is because here we do only a single thing at a time and sqlite3 provide # a synchronous api anyway - decryption_key = oscrypto.asymmetric.load_private_key(service_decryption_key.read_bytes()) + decryption_key = SequesterPrivateKeyDer.load_pem(service_decryption_key.read_text()) # Convert filter_date from click.Datetime to parsec.Datetime date: DateTime diff --git a/parsec/backend/organization.py b/parsec/backend/organization.py index fe38827b736..cba805c313e 100644 --- a/parsec/backend/organization.py +++ b/parsec/backend/organization.py @@ -18,7 +18,9 @@ OrganizationStatsRepNotFound, OrganizationStatsRepOk, OrganizationStatsReq, + SequesterVerifyKeyDer, UsersPerProfileDetailItem, + VerifyKey, ) from parsec.api.data import ( DataError, @@ -41,8 +43,6 @@ from parsec.backend.user import Device, User from parsec.backend.utils import Unset, UnsetType, api, api_typed_msg_adapter, catch_protocol_errors from parsec.backend.webhooks import WebhooksComponent -from parsec.crypto import VerifyKey -from parsec.sequester_crypto import SequesterVerifyKeyDer from parsec.utils import timestamps_in_the_ballpark diff --git a/parsec/backend/postgresql/organization.py b/parsec/backend/postgresql/organization.py index 3380707fd1f..8bf8535054f 100644 --- a/parsec/backend/postgresql/organization.py +++ b/parsec/backend/postgresql/organization.py @@ -7,7 +7,7 @@ import triopg from triopg import UniqueViolationError -from parsec._parsec import DateTime +from parsec._parsec import DateTime, SequesterVerifyKeyDer, VerifyKey from parsec.api.protocol import OrganizationID, UserProfile from parsec.backend.events import BackendEvent from parsec.backend.organization import ( @@ -28,8 +28,6 @@ from parsec.backend.postgresql.utils import Q, q_organization_internal_id from parsec.backend.user import Device, User, UserError from parsec.backend.utils import Unset, UnsetType -from parsec.crypto import VerifyKey -from parsec.sequester_crypto import SequesterVerifyKeyDer _q_insert_organization = Q( diff --git a/parsec/backend/postgresql/sequester.py b/parsec/backend/postgresql/sequester.py index b2fb88d482a..becb1165ace 100644 --- a/parsec/backend/postgresql/sequester.py +++ b/parsec/backend/postgresql/sequester.py @@ -6,7 +6,7 @@ import triopg -from parsec._parsec import DateTime +from parsec._parsec import DateTime, SequesterVerifyKeyDer from parsec.api.data import DataError, SequesterServiceCertificate from parsec.api.protocol import OrganizationID, RealmID, SequesterServiceID, VlobID from parsec.backend.organization import SequesterAuthority @@ -29,7 +29,6 @@ WebhookSequesterService, ) from parsec.crypto import CryptoError -from parsec.sequester_crypto import SequesterVerifyKeyDer # Sequester authority never gets modified past organization bootstrap, hence no need diff --git a/parsec/core/cli/bootstrap_organization.py b/parsec/core/cli/bootstrap_organization.py index 90aebe5fff7..d89161879cf 100644 --- a/parsec/core/cli/bootstrap_organization.py +++ b/parsec/core/cli/bootstrap_organization.py @@ -7,15 +7,13 @@ import click -from parsec._parsec import LocalDevice -from parsec.api.protocol import DeviceLabel, HumanHandle +from parsec._parsec import DeviceLabel, HumanHandle, LocalDevice, SequesterVerifyKeyDer from parsec.cli_utils import async_confirm, async_prompt, cli_exception_handler, spinner from parsec.core.cli.utils import cli_command_base_options, core_config_options, save_device_options from parsec.core.config import CoreConfig from parsec.core.fs.storage.user_storage import user_storage_non_speculative_init from parsec.core.invite import bootstrap_organization as do_bootstrap_organization from parsec.core.types import BackendOrganizationBootstrapAddr -from parsec.sequester_crypto import SequesterVerifyKeyDer from parsec.utils import trio_run @@ -63,7 +61,7 @@ async def _bootstrap_organization( ) if not answer: raise SystemExit("Bootstrap aborted") - sequester_vrf_key = SequesterVerifyKeyDer(sequester_verify_key.read_bytes()) + sequester_vrf_key = SequesterVerifyKeyDer.load_pem(sequester_verify_key.read_text()) human_label: str = human_label or await async_prompt("User fullname") human_email: str = human_email or await async_prompt("User email") diff --git a/parsec/core/invite/organization.py b/parsec/core/invite/organization.py index 9cc21d56304..df97a8a622f 100644 --- a/parsec/core/invite/organization.py +++ b/parsec/core/invite/organization.py @@ -1,6 +1,7 @@ # Parsec Cloud (https://parsec.cloud) Copyright (c) AGPL-3.0 2016-present Scille SAS from __future__ import annotations +from parsec._parsec import SequesterVerifyKeyDer, SigningKey, VerifyKey from parsec.api.data import DeviceCertificate, SequesterAuthorityCertificate, UserCertificate from parsec.api.protocol import DeviceLabel, HumanHandle, UserProfile from parsec.core.backend_connection import apiv1_backend_anonymous_cmds_factory @@ -14,8 +15,6 @@ ) from parsec.core.local_device import generate_new_device from parsec.core.types import BackendOrganizationAddr, BackendOrganizationBootstrapAddr, LocalDevice -from parsec.crypto import SigningKey, VerifyKey -from parsec.sequester_crypto import SequesterVerifyKeyDer def _check_rep(rep: dict[str, object], step_name: str) -> None: diff --git a/parsec/sequester_crypto.py b/parsec/sequester_crypto.py deleted file mode 100644 index 814cbeb9a9d..00000000000 --- a/parsec/sequester_crypto.py +++ /dev/null @@ -1,18 +0,0 @@ -# Parsec Cloud (https://parsec.cloud) Copyright (c) AGPL-3.0 2016-present Scille SAS -from __future__ import annotations - -from parsec._parsec import ( - CryptoError, - SequesterEncryptionKeyDer, - SequesterVerifyKeyDer, - sequester_authority_sign, - sequester_service_decrypt, -) - -__all__ = ( - "CryptoError", - "SequesterVerifyKeyDer", - "SequesterEncryptionKeyDer", - "sequester_authority_sign", - "sequester_service_decrypt", -) diff --git a/parsec/sequester_export_reader.py b/parsec/sequester_export_reader.py index a0b5e029e04..b7992e0313d 100644 --- a/parsec/sequester_export_reader.py +++ b/parsec/sequester_export_reader.py @@ -8,25 +8,26 @@ from pathlib import Path, PurePath from typing import Dict, Iterator, List, Mapping, Tuple -from oscrypto.asymmetric import PrivateKey - -from parsec._parsec import DateTime -from parsec.api.data import ( +from parsec._parsec import ( DataError, + DateTime, DeviceCertificate, + DeviceID, EntryID, EntryName, FileManifest, FolderManifest, + RealmID, RealmRoleCertificate, RevokedUserCertificate, + SequesterPrivateKeyDer, UserCertificate, + VerifyKey, WorkspaceManifest, + manifest_verify_and_load, ) -from parsec.api.data.manifest import AnyRemoteManifest, manifest_verify_and_load -from parsec.api.protocol import DeviceID, RealmID -from parsec.crypto import CryptoError, VerifyKey -from parsec.sequester_crypto import sequester_service_decrypt +from parsec.api.data.manifest import AnyRemoteManifest +from parsec.crypto import CryptoError REALM_EXPORT_DB_MAGIC_NUMBER = 87947 REALM_EXPORT_DB_VERSION = 1 # Only supported version so far @@ -175,7 +176,7 @@ def load_role_certificates( @dataclass class WorkspaceExport: db: RealmExportDb - decryption_key: PrivateKey + decryption_key: SequesterPrivateKeyDer devices_form_internal_id: Dict[int, Tuple[DeviceID, VerifyKey]] filter_on_date: DateTime @@ -203,9 +204,8 @@ def load_manifest(self, manifest_id: EntryID) -> AnyRemoteManifest: raise InconsistentWorkspaceError(f"Missing device certificate for `{author}`") timestamp = DateTime.from_timestamp(raw_timestamp / 1000000) - decrypted_blob = sequester_service_decrypt( - decryption_key=self.decryption_key, data=blob - ) + decrypted_blob = self.decryption_key.decrypt(blob) + manifest = manifest_verify_and_load( signed=decrypted_blob, author_verify_key=author_verify_key, @@ -374,7 +374,7 @@ def extract_workspace( def extract_workspace( - output: Path, export_db: Path, decryption_key: PrivateKey, filter_on_date: DateTime + output: Path, export_db: Path, decryption_key: SequesterPrivateKeyDer, filter_on_date: DateTime ) -> Iterator[Tuple[PurePath | None, RealmExportProgress, str]]: with RealmExportDb.open(export_db) as db: out_certificates: list[Tuple[int, DeviceCertificate]] = [] diff --git a/parsec/serde/fields.py b/parsec/serde/fields.py index 57bd18e1863..273bdc7f804 100644 --- a/parsec/serde/fields.py +++ b/parsec/serde/fields.py @@ -26,10 +26,10 @@ from parsec._parsec import PrivateKey as _PrivateKey from parsec._parsec import PublicKey as _PublicKey from parsec._parsec import SecretKey as _SecretKey +from parsec._parsec import SequesterPublicKeyDer as _SequesterPublicKeyDer +from parsec._parsec import SequesterVerifyKeyDer as _SequesterVerifyKeyDer from parsec._parsec import SigningKey as _SigningKey from parsec._parsec import VerifyKey as _VerifyKey -from parsec.sequester_crypto import SequesterEncryptionKeyDer as _SequesterEncryptionKeyDer -from parsec.sequester_crypto import SequesterVerifyKeyDer as _SequesterVerifyKeyDer from parsec.types import FrozenDict as _FrozenDict __all__ = ( @@ -495,27 +495,25 @@ def _deserialize( SequesterVerifyKeyDer = SequesterVerifyKeyDerField -class SequesterEncryptionKeyDerField(Field[_SequesterEncryptionKeyDer]): - def _serialize( - self, value: _SequesterEncryptionKeyDer | None, attr: str, obj: Any - ) -> bytes | None: +class SequesterPublicKeyDerField(Field[_SequesterPublicKeyDer]): + def _serialize(self, value: _SequesterPublicKeyDer | None, attr: str, obj: Any) -> bytes | None: if value is None: return None - assert isinstance(value, _SequesterEncryptionKeyDer) + assert isinstance(value, _SequesterPublicKeyDer) return value.dump() def _deserialize( self, value: object, attr: str, data: dict[str, object] - ) -> _SequesterEncryptionKeyDer: + ) -> _SequesterPublicKeyDer: if not isinstance(value, bytes): raise ValidationError("Expecting bytes") try: - return _SequesterEncryptionKeyDer(value) + return _SequesterPublicKeyDer(value) except Exception as exc: raise ValidationError(str(exc)) from exc -SequesterEncryptionKeyDer = SequesterEncryptionKeyDerField +SequesterPublicKeyDer = SequesterPublicKeyDerField class PkiEnrollmentSubmitPayloadField(Field[_PkiEnrollmentSubmitPayload]): diff --git a/tests/backend/sequester/test_crypto.py b/tests/backend/sequester/test_crypto.py deleted file mode 100644 index d01abaa2a1c..00000000000 --- a/tests/backend/sequester/test_crypto.py +++ /dev/null @@ -1,91 +0,0 @@ -# Parsec Cloud (https://parsec.cloud) Copyright (c) AGPL-3.0 2016-present Scille SAS -from __future__ import annotations - -import oscrypto.asymmetric -import pytest - -from parsec.sequester_crypto import ( - SequesterEncryptionKeyDer, - SequesterVerifyKeyDer, - sequester_authority_sign, - sequester_service_decrypt, -) - - -def test_only_rsa_is_supported(): - # Unsupported format for service encryption key (only RSA is currently supported) - unsupported_public_key, _ = oscrypto.asymmetric.generate_pair("dsa", bit_size=1024) - with pytest.raises(ValueError): - SequesterEncryptionKeyDer(unsupported_public_key), - with pytest.raises(ValueError): - SequesterVerifyKeyDer(unsupported_public_key), - - -def test_encryption_output_has_key_size(monkeypatch): - key_bit_size = 1024 - key_byte_size = key_bit_size // 8 - patched_called = 0 - - def patched_rsa_oaep_encrypt(key, data): - nonlocal patched_called - patched_called += 1 - assert key.byte_size == key_byte_size - assert len(data) == 32 # Data here is a Salsa20 key - # Output is 64bytes so much smaller than the 128bytes key - return b"\x01" * 32 + data - - def patched_rsa_oaep_decrypt(key, encrypted): - nonlocal patched_called - patched_called += 1 - assert key.byte_size == key_byte_size - assert len(encrypted) == key_byte_size - assert encrypted[:64] == b"\x00" * 64 # Null bytes added to have the correct size - assert encrypted[64:96] == b"\x01" * 32 - return encrypted[96:] - - monkeypatch.setattr("oscrypto.asymmetric.rsa_oaep_encrypt", patched_rsa_oaep_encrypt) - monkeypatch.setattr("oscrypto.asymmetric.rsa_oaep_decrypt", patched_rsa_oaep_decrypt) - - public_key, private_key = oscrypto.asymmetric.generate_pair("rsa", bit_size=key_bit_size) - encryption_key = SequesterEncryptionKeyDer(public_key) - - encrypted = encryption_key.encrypt(b"foo") - decrypted = sequester_service_decrypt(private_key, encrypted) - assert decrypted == b"foo" - - assert patched_called == 2 # Make sure the mock has been used - - -def test_signature_output_has_key_size(monkeypatch): - key_bit_size = 1024 - key_byte_size = key_bit_size // 8 - patched_called = 0 - - def patched_rsa_pss_sign(key, data, hash_algorithm): - nonlocal patched_called - patched_called += 1 - assert hash_algorithm == "sha256" - assert key.byte_size == key_byte_size - # Output is 64bytes so much smaller than the 128bytes key - return b"\x01" * 64 - - def patched_rsa_pss_verify(key, signed, data, hash_algorithm): - nonlocal patched_called - patched_called += 1 - assert hash_algorithm == "sha256" - assert key.byte_size == key_byte_size - assert len(signed) == key_byte_size - assert signed[:64] == b"\x00" * 64 # Null bytes added to have the correct size - assert signed[64:] == b"\x01" * 64 - - monkeypatch.setattr("oscrypto.asymmetric.rsa_pss_sign", patched_rsa_pss_sign) - monkeypatch.setattr("oscrypto.asymmetric.rsa_pss_verify", patched_rsa_pss_verify) - - public_key, private_key = oscrypto.asymmetric.generate_pair("rsa", bit_size=key_bit_size) - verify_key = SequesterVerifyKeyDer(public_key) - - signed = sequester_authority_sign(private_key, b"foo") - verified = verify_key.verify(signed) - assert verified == b"foo" - - assert patched_called == 2 # Make sure the mock has been used diff --git a/tests/backend/sequester/test_export.py b/tests/backend/sequester/test_export.py index 171e99930ce..87f72d6c088 100644 --- a/tests/backend/sequester/test_export.py +++ b/tests/backend/sequester/test_export.py @@ -3,10 +3,9 @@ import sqlite3 -import oscrypto.asymmetric import pytest -from parsec._parsec import DateTime +from parsec._parsec import DateTime, HashDigest, SecretKey, SequesterPrivateKeyDer from parsec.api.data import ( BlockAccess, DeviceCertificate, @@ -35,8 +34,6 @@ RealmExporterOutputDbError, ) from parsec.backend.realm import RealmGrantedRole -from parsec.crypto import HashDigest, SecretKey -from parsec.sequester_crypto import SequesterEncryptionKeyDer from parsec.sequester_export_reader import extract_workspace from tests.common import OrganizationFullData, customize_fixtures, sequester_service_factory @@ -436,10 +433,9 @@ def _sqlite_timestamp(year: int, month: int, day: int) -> int: async def test_export_reader_full_run(tmp_path, coolorg: OrganizationFullData, alice, bob, adam): output_db_path = tmp_path / "export.sqlite" realm1 = RealmID.new() - service_encryption_key, service_decryption_key = oscrypto.asymmetric.generate_pair( - "rsa", bit_size=1024 - ) - service_encryption_key = SequesterEncryptionKeyDer(service_encryption_key) + # Don't use such a small key size in real world, this is only for test ! + # (RSA key generation gets ~10x slower between 1024 and 4096) + service_decryption_key, service_encryption_key = SequesterPrivateKeyDer.generate_pair(1024) # Generate the export db by hand here con = sqlite3.connect(output_db_path) diff --git a/tests/common/sequester.py b/tests/common/sequester.py index dc29a41f2ff..5bcc4021245 100644 --- a/tests/common/sequester.py +++ b/tests/common/sequester.py @@ -2,11 +2,15 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Optional -import oscrypto.asymmetric - -from parsec._parsec import DateTime +from parsec._parsec import ( + DateTime, + SequesterPrivateKeyDer, + SequesterPublicKeyDer, + SequesterSigningKeyDer, + SequesterVerifyKeyDer, + SigningKey, +) from parsec.api.data import SequesterAuthorityCertificate, SequesterServiceCertificate from parsec.api.protocol import SequesterServiceID from parsec.backend.sequester import ( @@ -15,35 +19,27 @@ StorageSequesterService, WebhookSequesterService, ) -from parsec.crypto import SigningKey -from parsec.sequester_crypto import ( - SequesterEncryptionKeyDer, - SequesterVerifyKeyDer, - sequester_authority_sign, -) @dataclass class SequesterAuthorityFullData: certif: bytes certif_data: SequesterAuthorityCertificate - signing_key: oscrypto.asymmetric.PrivateKey - verify_key: oscrypto.asymmetric.PublicKey + signing_key: SequesterSigningKeyDer + verify_key: SequesterVerifyKeyDer def sequester_authority_factory( - organization_root_signing_key: SigningKey, timestamp: Optional[DateTime] = None + organization_root_signing_key: SigningKey, timestamp: DateTime | None = None ) -> SequesterAuthorityFullData: timestamp = timestamp or DateTime.now() # Don't use such a small key size in real world, this is only for test ! # (RSA key generation gets ~10x slower between 1024 and 4096) - verify_key, signing_key = oscrypto.asymmetric.generate_pair("rsa", bit_size=1024) + signing_key, verify_key = SequesterSigningKeyDer.generate_pair(1024) certif = SequesterAuthorityCertificate( author=None, timestamp=timestamp, - verify_key_der=SequesterVerifyKeyDer( - oscrypto.asymmetric.dump_public_key(verify_key, encoding="der") - ), + verify_key_der=verify_key, ) return SequesterAuthorityFullData( certif=certif.dump_and_sign(organization_root_signing_key), @@ -57,8 +53,8 @@ def sequester_authority_factory( class SequesterServiceFullData: certif: bytes certif_data: SequesterServiceCertificate - decryption_key: oscrypto.asymmetric.PrivateKey - encryption_key: oscrypto.asymmetric.PublicKey + decryption_key: SequesterPrivateKeyDer + encryption_key: SequesterPublicKeyDer backend_service: BaseSequesterService @property @@ -69,23 +65,21 @@ def service_id(self) -> SequesterServiceID: def sequester_service_factory( label: str, authority: SequesterAuthorityFullData, - timestamp: Optional[DateTime] = None, + timestamp: DateTime | None = None, service_type: SequesterServiceType = SequesterServiceType.STORAGE, - webhook_url: Optional[str] = None, + webhook_url: str | None = None, ) -> SequesterServiceFullData: timestamp = timestamp or DateTime.now() # Don't use such a small key size in real world, this is only for test ! # (RSA key generation gets ~10x slower between 1024 and 4096) - encryption_key, decryption_key = oscrypto.asymmetric.generate_pair("rsa", bit_size=1024) + decryption_key, encryption_key = SequesterPrivateKeyDer.generate_pair(1024) certif_data = SequesterServiceCertificate( service_id=SequesterServiceID.new(), timestamp=timestamp, service_label=label, - encryption_key_der=SequesterEncryptionKeyDer( - oscrypto.asymmetric.dump_public_key(encryption_key, encoding="der") - ), + encryption_key_der=encryption_key, ) - certif = sequester_authority_sign(signing_key=authority.signing_key, data=certif_data.dump()) + certif = authority.signing_key.sign(certif_data.dump()) if service_type == SequesterServiceType.STORAGE: assert webhook_url is None backend_service = StorageSequesterService( diff --git a/tests/core/fs/test_sequester.py b/tests/core/fs/test_sequester.py index c0f4ebadd4c..53d07bc23ff 100644 --- a/tests/core/fs/test_sequester.py +++ b/tests/core/fs/test_sequester.py @@ -12,7 +12,6 @@ from parsec.core.core_events import CoreEvent from parsec.core.fs.exceptions import FSServerUploadTemporarilyUnavailableError from parsec.core.fs.path import FsPath -from parsec.sequester_crypto import sequester_service_decrypt from tests.common import customize_fixtures, sequester_service_factory @@ -49,9 +48,7 @@ async def _assert_sequester_dump(service, local_device, expected_versions): # Advanced check: make sure each item contain the valid data for _, version, sequestered_blob in realm_dump: - clear_blob_from_sequester = sequester_service_decrypt( - service.decryption_key, sequestered_blob - ) + clear_blob_from_sequester = service.decryption_key.decrypt(sequestered_blob) _, blob, _, _, _ = await backend.vlob.read( organization_id=coolorg.organization_id, author=local_device.device_id, @@ -121,9 +118,7 @@ async def _assert_sequester_dump(service, workspace, expected_items): # Advanced check: make sure each item contain the valid data for vlob_id, version, sequestered_blob in realm_dump: - clear_blob_from_sequester = sequester_service_decrypt( - service.decryption_key, sequestered_blob - ) + clear_blob_from_sequester = service.decryption_key.decrypt(sequestered_blob) _, blob, _, _, _ = await backend.vlob.read( organization_id=coolorg.organization_id, author=alice.device_id, diff --git a/tests/core/test_bootstrap_organization.py b/tests/core/test_bootstrap_organization.py index c4fa1ff0e71..1cc0d1f47df 100644 --- a/tests/core/test_bootstrap_organization.py +++ b/tests/core/test_bootstrap_organization.py @@ -1,16 +1,15 @@ # Parsec Cloud (https://parsec.cloud) Copyright (c) AGPL-3.0 2016-present Scille SAS from __future__ import annotations -import oscrypto import pytest from quart import Response +from parsec._parsec import SequesterSigningKeyDer from parsec.api.data import EntryName from parsec.api.protocol import DeviceLabel, HumanHandle, OrganizationID, UserProfile from parsec.core.fs.storage.user_storage import user_storage_non_speculative_init from parsec.core.invite import InviteAlreadyUsedError, InviteNotFoundError, bootstrap_organization from parsec.core.types import BackendOrganizationBootstrapAddr -from parsec.sequester_crypto import SequesterVerifyKeyDer, sequester_authority_sign from parsec.serde import packb @@ -115,16 +114,18 @@ async def test_bootstrap_sequester_verify_key(running_backend, backend): human_handle = HumanHandle(email="zack@example.com", label="Zack") device_label = DeviceLabel("PC1") - verify_key, signing_key = oscrypto.asymmetric.generate_pair("rsa", bit_size=1024) + # Don't use such a small key size in real world, this is only for test ! + # (RSA key generation gets ~10x slower between 1024 and 4096) + signing_key, verify_key = SequesterSigningKeyDer.generate_pair(1024) + ref_data = b"SomeData" - ref_data_sign = sequester_authority_sign(signing_key, ref_data) - der_verify_key = SequesterVerifyKeyDer(verify_key) + ref_data_sign = signing_key.sign(ref_data) await bootstrap_organization( organization_addr, human_handle=human_handle, device_label=device_label, - sequester_authority_verify_key=der_verify_key, + sequester_authority_verify_key=verify_key, ) organization = await backend.organization.get(org_id) diff --git a/tests/schemas/api_data.json b/tests/schemas/api_data.json index 11b68c7c322..deee7c93c92 100644 --- a/tests/schemas/api_data.json +++ b/tests/schemas/api_data.json @@ -31,7 +31,7 @@ "encryption_key_der": { "allow_none": false, "required": true, - "type": "SequesterEncryptionKeyDerField" + "type": "SequesterPublicKeyDerField" }, "service_id": { "allow_none": false, diff --git a/tests/test_cli.py b/tests/test_cli.py index 5791f9af04a..cab2de06d18 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -8,7 +8,6 @@ from uuid import UUID import click -import oscrypto.asymmetric import trio try: @@ -1067,13 +1066,9 @@ def _setup_sequester_key_paths(tmp_path, coolorg): service = sequester_service_factory("Test Service", coolorg.sequester_authority) service_key = service.encryption_key authority_key = coolorg.sequester_authority.signing_key - service_key_path.write_bytes(oscrypto.asymmetric.dump_public_key(service_key)) - authority_key_path.write_bytes( - oscrypto.asymmetric.dump_private_key(authority_key, passphrase=None) - ) - authority_pubkey_path.write_bytes( - oscrypto.asymmetric.dump_public_key(coolorg.sequester_authority.verify_key) - ) + service_key_path.write_text(service_key.dump_pem()) + authority_key_path.write_text(authority_key.dump_pem()) + authority_pubkey_path.write_text(coolorg.sequester_authority.verify_key.dump_pem()) return authority_key_path, authority_pubkey_path, service_key_path