From d926c1e3ea441a9af38649acffa057a992b8985c Mon Sep 17 00:00:00 2001 From: Ren Amamiya Date: Wed, 16 Sep 2026 17:35:33 +0700 Subject: [PATCH] update concord backend --- Cargo.lock | 1 + PLAN.md | 52 +++-- crates/concord/Cargo.toml | 3 +- crates/concord/src/control.rs | 300 ++++++++++++++++++++++++++ crates/concord/src/edition.rs | 392 ++++++++++++++++++++++++++++++++++ crates/concord/src/lib.rs | 36 +++- crates/concord/src/store.rs | 147 ++++++++++++- crates/concord/src/stream.rs | 29 ++- 8 files changed, 929 insertions(+), 31 deletions(-) create mode 100644 crates/concord/src/control.rs create mode 100644 crates/concord/src/edition.rs diff --git a/Cargo.lock b/Cargo.lock index 6e03af56..de364b01 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1313,6 +1313,7 @@ dependencies = [ "nostr-memory", "nostr-sdk", "rand 0.10.2", + "serde", "serde_json", "sha2 0.10.9", "smol", diff --git a/PLAN.md b/PLAN.md index e40af3c2..ec6529b6 100644 --- a/PLAN.md +++ b/PLAN.md @@ -93,7 +93,7 @@ crates/concord/ Dependencies: `common`, `state`, `person`, `device`, `settings`, `gpui`, `nostr` (for `nip44` features), `nostr-sdk`, `hkdf`, `sha2`, `data-encoding`, `rand`, `serde`, `serde_json`, `anyhow`, `flume`, `smallvec`, `itertools`, `futures`, `log`, `instant`. Everything under `cfg(not(target_arch = "wasm32"))` follows the `chat`/`state` split so the crate still builds for web. -Declare only what a milestone actually uses. As of M1 the crate depends on `nostr`, `nostr-sdk`, `hkdf`, `sha2`, `data-encoding`, `rand`, `anyhow` (plus `nostr-memory`, `serde_json`, `smol` for tests). `rand` is pinned to the `0.10.2` instance `nostr` already builds and shares its `getrandom`, which the `web` crate already enables `wasm_js` on — so no new package and no new wasm obligation. +Declare only what a milestone actually uses. As of M2 the crate depends on `nostr`, `nostr-sdk`, `hkdf`, `sha2`, `data-encoding`, `rand`, `serde`, `serde_json`, `anyhow` (plus `nostr-memory` and `smol` for tests). `rand` is pinned to the `0.10.2` instance `nostr` already builds and shares its `getrandom`, which the `web` crate already enables `wasm_js` on — so no new package and no new wasm obligation. `serde`/`serde_json` were already in the graph via `nostr`; promoting `serde_json` from dev to main for the metadata content types added no package either, only the `concord → serde` edge. ## 5. Core types @@ -252,22 +252,33 @@ Design points that are easy to get wrong: ### 8.1 Editions and authority (`edition.rs`, `control.rs`) ```rust -pub const EDITION_LABEL: &[u8] = b"vector-community/v1/edition"; // frozen, cross-client +pub const EDITION_LABEL: &[u8] = b"vector-community/v1/edition"; // frozen, cross-client (27 bytes) +// sha256( u64be(len(label)) ‖ label ‖ entity[32] ‖ u64be(version) +// ‖ flag[1] ‖ prev[32] ‖ u64be(len(content)) ‖ content ) +// `prev` is always 33 bytes: 0x01 ‖ hash, or 0x00 ‖ zeroes when absent. +// The hash commits to no actor: identity enters only via the rumor id. pub fn edition_hash(entity: &[u8; 32], version: u64, prev: Option<&[u8; 32]>, content: &[u8]) -> [u8; 32]; -pub struct ParsedEdition { author: PublicKey, vsk: String, entity: [u8; 32], version: u64, - prev: Option<[u8; 32]>, content: String, self_hash: [u8; 32] }; +pub struct ParsedEdition { author: PublicKey, subkind: String, entity: [u8; 32], version: u64, + prev: Option<[u8; 32]>, citation: Option, + content: String, self_hash: [u8; 32], rumor_id: EventId }; pub fn parse_edition(rumor: &UnsignedEvent) -> Result; pub struct FoldResult { pub head: Option, pub gap: bool, pub anchored: bool } pub fn fold(editions: &[EditionMeta], floor: u64, floor_hash: Option<&[u8; 32]>) -> FoldResult; -pub fn bootstrap_head(editions: &[EditionMeta], floor: u64) -> Option; +pub fn bootstrap_head(editions: &[EditionMeta]) -> Option; // highest, contiguity ignored ``` -- Tag grammar: `["vsk", sub]`, `["eid", hex32]`, `["ev", decimal]`, `["ep", hex32]`, `["vac", eid, version, hash]`. Duplicates of any of the five reject the edition; `ev` must pass a decimal check before parsing. -- Tie-break at equal version is the lower **inner rumor id**, never `created_at`. -- `gap` is a safety signal: a tracking client (already holds the floor) fails closed for that entity and refetches; a bootstrapping client (floor 0) may accept the highest authority-verified head, which is what makes compaction re-wrapping work. -- Entity coordinates are `vsk 0` → `community_id`, `1` → `role_id`, `2` → `channel_id`, `3` → `grant_locator`, `4` → `banlist_locator`, `8` → `invite_links_locator`, `11` → `pins_locator`. All derive from `community_id` only, so a refounding re-wraps heads verbatim. +- Tag grammar: `["vsk", sub]`, `["eid", hex32]`, `["ev", decimal]`, `["ep", hex32]`, `["vac", eid, version, hash]`. Duplicates of any of the five reject the edition; `ev` must pass a decimal check before parsing. A version of `0` parses and then reads as a gap — the rule lives in the fold, not the parser. +- **Versions start at 1, not 0** (CORD-04 §1: "climbs from 1"). Genesis is `(version 1, prev None)` for both entities. +- **The edition hash is not the signature.** The actor's Schnorr signature covers the kind-20014 plaintext seal; `edition_hash` is a separate SHA-256 used only for chaining (`ep`, `vac`). `content` is the rumor's content string byte-verbatim, never re-serialized, which is what lets compaction re-wrap a head and preserve its hash. +- The domain label is `vector-community/v1/edition`, not a `concord/…` label. Inconsistent with Appendix A.6, frozen anyway — do not "fix" it. +- Tie-break at equal version is the lower **inner rumor id** (the kind-3308 rumor), never the outer wrap id and never `created_at`. Only one of the two implementations that must agree applies to a wrap, so the inner id is the only stable choice. +- `gap` is a safety signal: a tracking client (already holds the floor) fails closed for that entity and refetches; a bootstrapping client (floor 0) may accept the highest authority-verified head, which is what makes compaction re-wrapping work. `bootstrap_head` therefore takes no floor: it *is* the floor-zero path. +- **Owner anchoring is not in the fold.** `fold` is a pure function of chain shape; authority is a pre-filter the caller applies to the candidate set before folding. `community_id` proves the owner, and `is_authorized` short-circuits `owner == actor`, so the owner needs no Grant entity at all. +- Entity coordinates are `vsk 0` → `community_id`, `1` → `role_id`, `2` → `channel_id`, `3` → `grant_locator`, `4` → `banlist_locator`, `8` → `invite_links_locator`, `11` → `pins_locator`. `5` is reserved, `6`/`9` belong to the 33301 invite marker, `7` is retired. All derive from `community_id` only, so a refounding re-wraps heads verbatim. +- The Control Plane is **plaintext-seal only**. A 20013-encrypted control edition is rejected, because compaction re-wraps a signed plaintext seal byte-verbatim into the new epoch; accepting an encrypted one would let a later compaction fork the chain. +- **Genesis is exactly two owner-signed editions** — community metadata (`vsk 0`, `eid = community_id`) and one public `#general` channel (`vsk 2`, fresh random `channel_id`) — at epoch 0, version 1, no `ep`, no `vac`. No default roles, no scaffolding, and no Grant for the owner. Secrets minted: `owner_salt`, `community_root`, `control_root` (deliberately not derived from `community_id`). ```rust pub const P_MANAGE_ROLES: u64 = 1 << 0; // …bit table from CORD-04 §3, frozen; retired bits are burned @@ -432,20 +443,17 @@ pub struct CommunityState { pub community_root: [u8; 32], pub root_epoch: Epoch, pub control_root: Option<[u8; 32]>, // present iff the holder is staff - pub control_pks: BTreeMap, - pub channels: Vec, // id, key, epoch, name, private - pub epoch_keys: Vec<([u8; 32], Epoch, [u8; 32])>, // (scope, epoch, key) — the history backfill index + pub control_pks: BTreeMap, // epoch → the plane's signer address + pub channels: Vec, // id, name, private, epoch pub relays: Vec, - pub heads: BTreeMap<[u8; 32], (u64, [u8; 32], EventId)>, // entity → (version, self_hash, inner id) - pub guestbook: Vec, - pub observed: BTreeMap, - pub banned: BTreeSet, - pub dissolved: bool, + pub heads: Vec, // entity, version, self_hash, inner id pub added_at_ms: u64, } ``` -Writes are debounced (a fold head changes on every edition); reads load once at init. This layer lands in M2, once `Community` exists to hold it. +Landed in M2 with exactly the fields genesis can populate: `save_state`/`load_state` and `CommunityState::from_genesis`. Three fields the plan sketched are deliberately absent until something can fill them — `epoch_keys` (needs rekeys, M7), and `guestbook`/`observed`/`banned`/`dissolved` (need the guestbook, M5). `control_pks` keyed by `u64` rather than `Epoch` and `heads` as a `Vec` rather than a `BTreeMap<[u8; 32], _>`, because serde_json cannot use a byte-array map key. + +Writes are debounced (a fold head changes on every edition); reads load once at init. **Decision, stated for the record:** this document stores raw community keys unencrypted in a plaintext local database. That matches the existing posture — `chat` already caches decrypted message rumors in the same LMDB. If that posture ever changes, the state document is the one to wrap with NIP-44-to-self, since it is the only local artifact holding keys. @@ -569,7 +577,7 @@ pub fn pin(&self, id: EventId, cx: &App) -> Task>; // vsk 11, ## 11. Integration with existing crates -1. **`crates/chat/src/lib.rs` — required fix, applied in M2.** `handle_notifications` currently treats *every* kind 1059 event as a NIP-59 gift wrap for the current user and pushes an unwrap failure into the trash. Concord wraps are kind 1059 with an ephemeral `p` tag, so they would flood the trash and leak error toasts. Route by `subscription_id` from `RelayMessage::Event` against `sub_id1`/`sub_id2`, and drop the `if rumor.tags.is_empty()` recipient heuristic. M1 did not need it because nothing subscribes yet. +1. **`crates/chat/src/lib.rs` — required fix, moved from M2 to the milestone that first subscribes.** `handle_notifications` currently treats *every* kind 1059 event as a NIP-59 gift wrap for the current user and pushes an unwrap failure into the trash. Concord wraps are kind 1059 with an ephemeral `p` tag, so they would flood the trash and leak error toasts. Route by `subscription_id` from `RelayMessage::Event` against `sub_id1`/`sub_id2`, and drop the `if rumor.tags.is_empty()` recipient heuristic. **M2 did not apply it**: the crate has no subscription and no `ConcordRegistry` yet, so no concord wrap can reach that handler and the change would be untestable. It lands with the sync engine (§10), as does the `concord::init` wiring in `desktop` and `web`. 2. **`desktop/src/main.rs` and `web/src/lib.rs`** — add `concord::init(window, cx)` after `chat::init(window, cx)`. 3. **`Cargo.toml`** — add `hkdf = "0.12"` to `[workspace.dependencies]`; add the crate to `desktop` and `web` dependencies. No other workspace changes. 4. **No changes** to `state`, `person`, `device`, `settings`, `common`, or `ui`. @@ -598,7 +606,7 @@ Each of these has burned a real implementation, or is a documented cross-client | --- | --- | --- | | M0 | Crate skeleton, `derive.rs`, golden vectors, workspace wiring | ✅ `cargo test -p concord` pins every derivation; all labels match Appendix A.6 | | M1 | `stream.rs` + `store.rs` | ✅ seal/wrap/open round-trips for both seal forms; hostile wraps rejected in the documented order; the local cache reads back with the group key gone | -| M2 | `edition.rs` + `control.rs` genesis | a community is created and published; its two genesis wraps open at a second client sharing the keys; edition hash matches the cross-client vector | +| M2 | `edition.rs` + `control.rs` genesis + `store.rs` state document | ✅ `cargo test -p concord` (7 tests): `edition_hash` reproduces the cross-client vector `2daf42e6…`, and a community minted by one holder has both genesis wraps open for a second holder holding only the invite keys, folding to version 1 | | M3 | Control fold + roster + metadata/channels | fold tests for chains, gaps, downgrade refusal, fork tiebreak, compaction dangle; metadata and channel edits visible to a second client | | M4 | Chat plane | send/receive/edit/delete/reaction across two identities; history pages backwards across an epoch boundary (relay `backfill` lands here); binding checks reject a foreign channel/epoch | | M5 | Guestbook + member list + moderation | join/leave/kick/ban converge to the same memberlist on both clients; every authority rule from §8.1 has a negative test | @@ -610,6 +618,10 @@ Ordering is deliberately dependency-first: each milestone is usable on its own, M1 closed with `cargo test -p concord` (5 tests), `cargo clippy -p concord --all-targets`, and `cargo fmt -p concord --check` all clean. `rand` was added to the workspace pinned to the same `0.10.2` instance `nostr` already builds, so `Cargo.lock` gained no package. +M2 closed the same way at 7 tests, with `serde` added to the crate's dependencies (`serde_json` promoted from dev to main for the metadata content types) — `Cargo.lock` gained no package again, only the `concord → serde` edge. + +**M2's "created and published" is verified offline**: "published" is the two wraps existing and being openable by the invite keys, not a relay round-trip. There is no registry to publish through until §10, and a relay test would be testing the SDK, not the protocol. + ## 14. Open questions and risks 1. **Community List kind.** CORD-02 §8 specifies `13302`, replaceable. Vector has retired it in favour of fragmented `33302`, because a replaceable kind holds one event per pubkey and so cannot shard past the NIP-44 size cap. We implement `13302` per spec, enforce the 50-membership cap and pre-publish size check, and treat `33302` as an interop follow-up. Confirm with Armada before writing the multi-device code. diff --git a/crates/concord/Cargo.toml b/crates/concord/Cargo.toml index cb663be4..29d21505 100644 --- a/crates/concord/Cargo.toml +++ b/crates/concord/Cargo.toml @@ -12,9 +12,10 @@ hkdf.workspace = true sha2.workspace = true data-encoding.workspace = true rand.workspace = true +serde.workspace = true +serde_json.workspace = true anyhow.workspace = true [dev-dependencies] nostr-memory.workspace = true -serde_json.workspace = true smol.workspace = true diff --git a/crates/concord/src/control.rs b/crates/concord/src/control.rs new file mode 100644 index 00000000..70daa75e --- /dev/null +++ b/crates/concord/src/control.rs @@ -0,0 +1,300 @@ +use anyhow::{Result, bail}; +use nostr_sdk::prelude::{Event, Keys, PublicKey, Timestamp, UnsignedEvent}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +use crate::derive::{ + community_id_of, control_group_key, control_signer_group_key, verify_community_id, +}; +use crate::edition::{EditionFields, ParsedEdition, build_edition, parse_edition, vsk}; +use crate::stream::{KIND_WRAP, SealForm, build_seal, open_wrap_at, wrap_seal_with}; +use crate::{ChannelId, CommunityId, Epoch, GroupKey, random_32}; + +pub const MAX_NAME_BYTES: usize = 64; +pub const MAX_DESCRIPTION_BYTES: usize = 10_000; +pub const MAX_RELAYS: usize = 5; + +pub const GENERAL_CHANNEL: &str = "general"; +pub const ROOT_EPOCH: Epoch = Epoch(0); + +/// The first edition every entity starts at. Genuinely 1, not 0: a version of +/// 0 is what the fold treats as a gap. +const GENESIS_VERSION: u64 = 1; + +type Extra = Map; + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct ImageRef { + pub url: String, + pub key: String, + pub nonce: String, + pub hash: String, + #[serde(flatten)] + pub extra: Extra, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct CommunityMetadata { + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub relays: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub icon: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub banner: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub custom: Option, + #[serde(flatten)] + pub extra: Extra, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct ChannelMetadata { + pub name: String, + pub private: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub voice: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub deleted: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub custom: Option, + #[serde(flatten)] + pub extra: Extra, +} + +/// A community's permanent identity. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CommunityIdentity { + pub community_id: CommunityId, + pub owner: PublicKey, + pub owner_salt: [u8; 32], +} + +impl CommunityIdentity { + pub fn verify(&self) -> bool { + verify_community_id(&self.community_id, &self.owner.to_bytes(), &self.owner_salt) + } +} + +/// Everything a creation mints: +/// +/// - Identity +/// - Roots an invite will carry +/// - First channel +/// - Two genesis wraps to publish +#[derive(Debug, Clone)] +pub struct CommunityGenesis { + pub identity: CommunityIdentity, + pub community_root: [u8; 32], + pub control_root: [u8; 32], + pub channel_id: ChannelId, + pub wraps: Vec, +} + +/// Mints a community and signs its two genesis editions. +pub fn genesis( + owner: &Keys, + metadata: &CommunityMetadata, + at_secs: u64, +) -> Result { + let mut metadata = metadata.clone(); + + if metadata.name.len() > MAX_NAME_BYTES { + bail!("community name exceeds {MAX_NAME_BYTES} bytes"); + } + + if metadata + .description + .as_ref() + .is_some_and(|description| description.len() > MAX_DESCRIPTION_BYTES) + { + bail!("community description exceeds {MAX_DESCRIPTION_BYTES} bytes"); + } + + metadata.relays.truncate(MAX_RELAYS); + + let owner_salt = random_32()?; + let identity = CommunityIdentity { + community_id: community_id_of(&owner.public_key().to_bytes(), &owner_salt), + owner: owner.public_key(), + owner_salt, + }; + + let community_root = random_32()?; + let control_root = random_32()?; + let channel_id = ChannelId::from_bytes(random_32()?); + + let read = control_group_key(&community_root, &identity.community_id, ROOT_EPOCH)?; + let signer = control_signer_group_key(&control_root, &identity.community_id, ROOT_EPOCH)?; + + let metadata_content = serde_json::to_string(&metadata)?; + let channel_content = serde_json::to_string(&ChannelMetadata { + name: GENERAL_CHANNEL.to_owned(), + private: false, + ..ChannelMetadata::default() + })?; + + let editions = [ + build_edition(EditionFields { + author: identity.owner, + subkind: vsk::COMMUNITY_METADATA, + entity: *identity.community_id.as_bytes(), + version: GENESIS_VERSION, + prev: None, + citation: None, + content: &metadata_content, + at_secs, + }), + build_edition(EditionFields { + author: identity.owner, + subkind: vsk::CHANNEL_METADATA, + entity: *channel_id.as_bytes(), + version: GENESIS_VERSION, + prev: None, + citation: None, + content: &channel_content, + at_secs, + }), + ]; + + let mut wraps = Vec::with_capacity(editions.len()); + + for edition in &editions { + wraps.push(seal_edition(edition, owner, &read, &signer, at_secs)?); + } + + Ok(CommunityGenesis { + identity, + community_root, + control_root, + channel_id, + wraps, + }) +} + +/// Opens a Control Plane wrap from its reading key alone. +pub fn open_edition( + wrap: &Event, + read: &GroupKey, + address: &PublicKey, + verify_wrap_signature: bool, +) -> Result { + let opened = open_wrap_at(wrap, address, read.conversation(), verify_wrap_signature)?; + + if opened.seal_form != SealForm::Plaintext { + bail!("control editions require a plaintext seal"); + } + + Ok(parse_edition(&opened.rumor)?) +} + +fn seal_edition( + edition: &UnsignedEvent, + owner: &Keys, + read: &GroupKey, + signer: &GroupKey, + at_secs: u64, +) -> Result { + let seal = build_seal(edition, SealForm::Plaintext, read, owner)?; + + let (wrap, _) = wrap_seal_with( + &seal, + read.conversation(), + signer.keys(), + KIND_WRAP, + Timestamp::from_secs(at_secs), + &[], + )?; + + Ok(wrap) +} + +#[cfg(test)] +mod tests { + use nostr_memory::MemoryDatabase; + + use super::*; + use crate::edition::{EditionMeta, fold}; + use crate::store::{CommunityState, load_state, save_state}; + + #[test] + fn genesis_reopens_for_a_second_holder() { + let owner = Keys::generate(); + let metadata = CommunityMetadata { + name: "coop".to_owned(), + relays: vec!["wss://relay.example".to_owned()], + ..CommunityMetadata::default() + }; + let at_secs = 1_700_000_000; + + let minted = genesis(&owner, &metadata, at_secs).expect("mints"); + assert!(minted.identity.verify(), "identity is self-certifying"); + + // The second client holds only what an invite hands over: the roots, + // the community id and the owner salt. + let read = control_group_key( + &minted.community_root, + &minted.identity.community_id, + ROOT_EPOCH, + ) + .expect("derives"); + let address = control_signer_group_key( + &minted.control_root, + &minted.identity.community_id, + ROOT_EPOCH, + ) + .expect("derives") + .pk(); + + let mut editions = Vec::new(); + for wrap in &minted.wraps { + editions.push(open_edition(wrap, &read, &address, true).expect("opens")); + } + + assert_eq!(editions.len(), 2); + + let community = &editions[0]; + assert_eq!(community.subkind, vsk::COMMUNITY_METADATA); + assert_eq!(community.entity, *minted.identity.community_id.as_bytes()); + assert_eq!(community.author, owner.public_key()); + assert_eq!((community.version, community.prev), (1, None)); + assert_eq!( + serde_json::from_str::(&community.content) + .expect("parses") + .name, + "coop" + ); + + let channel = &editions[1]; + assert_eq!(channel.subkind, vsk::CHANNEL_METADATA); + assert_eq!(channel.entity, *minted.channel_id.as_bytes()); + + for edition in &editions { + let folded = fold(&[EditionMeta::from(edition)], 0, None); + assert_eq!(folded.head, Some(0)); + assert!( + folded.anchored && !folded.gap, + "genesis anchors at its floor" + ); + } + + let state = + CommunityState::from_genesis(&minted, &editions, at_secs * 1_000).expect("projects"); + + smol::block_on(async { + let database = MemoryDatabase::unbounded(); + save_state(&database, &state).await.expect("saves"); + let loaded = load_state(&database, &minted.identity.community_id) + .await + .expect("loads") + .expect("present"); + + assert_eq!(loaded.community_root, minted.community_root); + assert_eq!(loaded.control_root, Some(minted.control_root)); + assert_eq!(loaded.channels.len(), 1); + assert_eq!(loaded.heads.len(), 2); + }); + } +} diff --git a/crates/concord/src/edition.rs b/crates/concord/src/edition.rs new file mode 100644 index 00000000..a758b6cd --- /dev/null +++ b/crates/concord/src/edition.rs @@ -0,0 +1,392 @@ +use std::collections::BTreeMap; +use std::fmt; + +use data_encoding::HEXLOWER; +use nostr_sdk::prelude::{EventId, PublicKey, Tag, UnsignedEvent}; +use sha2::{Digest, Sha256}; + +use crate::decode_hex_32; +use crate::stream::build_rumor_secs; + +pub const KIND_CONTROL: u16 = 3308; + +const EDITION_LABEL: &[u8] = b"vector-community/v1/edition"; + +/// Entity types an edition can address (CORD-02 Appendix B). +pub mod vsk { + pub const COMMUNITY_METADATA: &str = "0"; + pub const ROLE: &str = "1"; + pub const CHANNEL_METADATA: &str = "2"; + pub const GRANT: &str = "3"; + pub const BANLIST: &str = "4"; + pub const INVITE_LIVE: &str = "6"; + pub const INVITE_LINKS: &str = "8"; + pub const INVITE_REVOKED: &str = "9"; + pub const DISSOLVED: &str = "10"; + pub const PINS: &str = "11"; +} + +const TAG_SUBKIND: &str = "vsk"; +const TAG_ENTITY: &str = "eid"; +const TAG_VERSION: &str = "ev"; +const TAG_PREV: &str = "ep"; +const TAG_CITATION: &str = "vac"; + +#[derive(Debug)] +pub enum EditionError { + BadKind(u16), + BadField(&'static str), + Duplicate(&'static str), + Missing(&'static str), +} + +impl fmt::Display for EditionError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + EditionError::BadKind(kind) => write!(f, "not an edition kind: {kind}"), + EditionError::BadField(name) => write!(f, "malformed edition field: {name}"), + EditionError::Duplicate(name) => write!(f, "duplicate edition field: {name}"), + EditionError::Missing(name) => write!(f, "missing edition field: {name}"), + } + } +} + +impl std::error::Error for EditionError {} + +/// A `vac` citation: the Grant edition an actor claims rank under, pinned by +/// coordinate, version and hash. It is a sync floor, not the verdict. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AuthorityCitation { + pub entity: [u8; 32], + pub version: u64, + pub hash: [u8; 32], +} + +#[derive(Debug, Clone)] +pub struct ParsedEdition { + pub author: PublicKey, + pub subkind: String, + pub entity: [u8; 32], + pub version: u64, + pub prev: Option<[u8; 32]>, + pub citation: Option, + pub content: String, + pub self_hash: [u8; 32], + pub rumor_id: EventId, +} + +pub struct EditionFields<'a> { + pub author: PublicKey, + pub subkind: &'a str, + pub entity: [u8; 32], + pub version: u64, + pub prev: Option<[u8; 32]>, + pub citation: Option, + pub content: &'a str, + pub at_secs: u64, +} + +fn signing_bytes( + entity: &[u8; 32], + version: u64, + prev: Option<&[u8; 32]>, + content: &[u8], +) -> Vec { + let mut bytes = + Vec::with_capacity(8 + EDITION_LABEL.len() + 32 + 8 + 1 + 32 + 8 + content.len()); + + bytes.extend_from_slice(&(EDITION_LABEL.len() as u64).to_be_bytes()); + bytes.extend_from_slice(EDITION_LABEL); + bytes.extend_from_slice(entity); + bytes.extend_from_slice(&version.to_be_bytes()); + + match prev { + Some(prev) => { + bytes.push(1); + bytes.extend_from_slice(prev); + } + None => { + bytes.push(0); + bytes.extend_from_slice(&[0u8; 32]); + } + } + + bytes.extend_from_slice(&(content.len() as u64).to_be_bytes()); + bytes.extend_from_slice(content); + bytes +} + +pub fn edition_hash( + entity: &[u8; 32], + version: u64, + prev: Option<&[u8; 32]>, + content: &[u8], +) -> [u8; 32] { + Sha256::digest(signing_bytes(entity, version, prev, content)).into() +} + +pub fn build_edition(fields: EditionFields<'_>) -> UnsignedEvent { + let mut tags = vec![ + Tag::custom(TAG_SUBKIND, [fields.subkind]), + Tag::custom(TAG_ENTITY, [HEXLOWER.encode(&fields.entity)]), + Tag::custom(TAG_VERSION, [fields.version.to_string()]), + ]; + + if let Some(prev) = fields.prev { + tags.push(Tag::custom(TAG_PREV, [HEXLOWER.encode(&prev)])); + } + + if let Some(citation) = fields.citation { + tags.push(Tag::custom( + TAG_CITATION, + [ + HEXLOWER.encode(&citation.entity), + citation.version.to_string(), + HEXLOWER.encode(&citation.hash), + ], + )); + } + + build_rumor_secs( + KIND_CONTROL, + fields.author, + fields.content, + tags, + fields.at_secs, + ) +} + +pub fn parse_edition(rumor: &UnsignedEvent) -> Result { + let kind = rumor.kind.as_u16(); + + if kind != KIND_CONTROL { + return Err(EditionError::BadKind(kind)); + } + + let subkind = value(rumor, TAG_SUBKIND)? + .ok_or(EditionError::Missing(TAG_SUBKIND))? + .to_owned(); + + if canonical_decimal(&subkind).is_none() { + return Err(EditionError::BadField(TAG_SUBKIND)); + } + + let entity = hex32( + value(rumor, TAG_ENTITY)?.ok_or(EditionError::Missing(TAG_ENTITY))?, + TAG_ENTITY, + )?; + + let version = + canonical_decimal(value(rumor, TAG_VERSION)?.ok_or(EditionError::Missing(TAG_VERSION))?) + .ok_or(EditionError::BadField(TAG_VERSION))?; + + let prev = match value(rumor, TAG_PREV)? { + Some(raw) => Some(hex32(raw, TAG_PREV)?), + None => None, + }; + + let citation = match fields(rumor, TAG_CITATION)? { + Some(fields) if fields.len() == 4 => Some(AuthorityCitation { + entity: hex32(&fields[1], TAG_CITATION)?, + version: canonical_decimal(&fields[2]).ok_or(EditionError::BadField(TAG_CITATION))?, + hash: hex32(&fields[3], TAG_CITATION)?, + }), + Some(_) => return Err(EditionError::BadField(TAG_CITATION)), + None => None, + }; + + let self_hash = edition_hash(&entity, version, prev.as_ref(), rumor.content.as_bytes()); + + Ok(ParsedEdition { + author: rumor.pubkey, + subkind, + entity, + version, + prev, + citation, + content: rumor.content.clone(), + self_hash, + rumor_id: rumor.id.unwrap_or_else(|| rumor.compute_id()), + }) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct EditionMeta { + pub version: u64, + pub self_hash: [u8; 32], + pub prev: Option<[u8; 32]>, + pub tiebreak_id: EventId, +} + +impl From<&ParsedEdition> for EditionMeta { + fn from(edition: &ParsedEdition) -> Self { + Self { + version: edition.version, + self_hash: edition.self_hash, + prev: edition.prev, + tiebreak_id: edition.rumor_id, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct FoldResult { + pub head: Option, + pub gap: bool, + pub anchored: bool, +} + +/// The highest version whose chain is intact, given a held floor. +pub fn fold(editions: &[EditionMeta], floor: u64, floor_hash: Option<&[u8; 32]>) -> FoldResult { + let mut by_version: BTreeMap = BTreeMap::new(); + + for (index, edition) in editions.iter().enumerate() { + if edition.version < floor { + continue; + } + + match by_version.get(&edition.version) { + Some(¤t) if editions[current].tiebreak_id <= edition.tiebreak_id => {} + _ => { + by_version.insert(edition.version, index); + } + } + } + + let Some((&lowest_version, &lowest_index)) = by_version.first_key_value() else { + return FoldResult::default(); + }; + + let lowest = editions[lowest_index]; + + let anchored = if floor == 0 { + lowest_version == 1 && lowest.prev.is_none() + } else if lowest_version == floor { + floor_hash == Some(&lowest.self_hash) + } else if lowest_version == floor + 1 { + floor_hash.is_some() && lowest.prev.as_ref() == floor_hash + } else { + false + }; + + let mut head = Some(lowest_index); + let mut gap = !anchored; + let mut previous_version = lowest_version; + let mut previous_hash = lowest.self_hash; + + for (&version, &index) in by_version.range(lowest_version + 1..) { + let edition = editions[index]; + + if version == previous_version + 1 && edition.prev == Some(previous_hash) { + head = Some(index); + previous_version = version; + previous_hash = edition.self_hash; + } else { + gap = true; + break; + } + } + + FoldResult { + head, + gap, + anchored, + } +} + +/// The highest version overall, ignoring contiguity. +pub fn bootstrap_head(editions: &[EditionMeta]) -> Option { + editions + .iter() + .enumerate() + .reduce(|(best_index, best), (index, candidate)| { + let supersedes = candidate.version > best.version + || (candidate.version == best.version && candidate.tiebreak_id < best.tiebreak_id); + + if supersedes { + (index, candidate) + } else { + (best_index, best) + } + }) + .map(|(index, _)| index) +} + +fn canonical_decimal(raw: &str) -> Option { + if raw.is_empty() || !raw.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + + if raw.len() > 1 && raw.starts_with('0') { + return None; + } + + raw.parse().ok() +} + +fn hex32(raw: &str, name: &'static str) -> Result<[u8; 32], EditionError> { + decode_hex_32(raw).map_err(|_| EditionError::BadField(name)) +} + +fn fields<'a>( + rumor: &'a UnsignedEvent, + name: &'static str, +) -> Result, EditionError> { + let mut found: Option<&[String]> = None; + + for tag in rumor.tags.iter() { + let tag_fields = tag.as_slice(); + + if tag_fields.first().map(String::as_str) != Some(name) { + continue; + } + + if found.is_some() { + return Err(EditionError::Duplicate(name)); + } + + found = Some(tag_fields); + } + + Ok(found) +} + +fn value<'a>( + rumor: &'a UnsignedEvent, + name: &'static str, +) -> Result, EditionError> { + match fields(rumor, name)? { + Some(fields) if fields.len() == 2 => Ok(Some(fields[1].as_str())), + Some(_) => Err(EditionError::BadField(name)), + None => Ok(None), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn edition_hash_matches_the_cross_client_vector() { + let entity = [0x11u8; 32]; + + assert_eq!( + HEXLOWER.encode(&edition_hash(&entity, 1, None, b"hello")), + "2daf42e65a6bc259a4c99fac6df754a5d3d92310607cf13e2a1e8c94d42f6303" + ); + + // The golden vector only exercises the absent-prev encoding; pin the + // present-prev branch structurally so a swapped flag stays visible. + let bytes = signing_bytes(&entity, 1, Some(&entity), b"hello"); + assert_eq!( + bytes.len(), + 8 + EDITION_LABEL.len() + 32 + 8 + 1 + 32 + 8 + 5 + ); + assert_eq!(&bytes[8..8 + EDITION_LABEL.len()], EDITION_LABEL); + assert_eq!( + bytes[8 + EDITION_LABEL.len() + 32..][..8], + 1u64.to_be_bytes() + ); + assert_eq!(bytes[8 + EDITION_LABEL.len() + 32 + 8], 1); + } +} diff --git a/crates/concord/src/lib.rs b/crates/concord/src/lib.rs index 82b4110e..481493fa 100644 --- a/crates/concord/src/lib.rs +++ b/crates/concord/src/lib.rs @@ -1,4 +1,6 @@ +pub mod control; pub mod derive; +pub mod edition; pub mod store; pub mod stream; @@ -8,6 +10,9 @@ use std::str::FromStr; use anyhow::{Result, anyhow, bail}; use data_encoding::HEXLOWER; pub use derive::GroupKey; +use rand::TryRng as _; +use rand::rngs::SysRng; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; macro_rules! hex_id { ($(#[$meta:meta])* $name:ident) => { @@ -54,6 +59,19 @@ macro_rules! hex_id { Ok(Self(decode_hex_32(value)?)) } } + + impl Serialize for $name { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.to_hex()) + } + } + + impl<'de> Deserialize<'de> for $name { + fn deserialize>(deserializer: D) -> Result { + let value = String::deserialize(deserializer)?; + value.parse().map_err(serde::de::Error::custom) + } + } }; } @@ -72,7 +90,9 @@ hex_id! { /// A key-rotation counter attached to each Community key. /// /// It bumps only on a Rekey, a membership change where somebody is removed. -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Default)] +#[derive( + Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Default, Serialize, Deserialize, +)] pub struct Epoch(pub u64); impl From for Epoch { @@ -94,7 +114,7 @@ impl fmt::Display for Epoch { } /// Uppercase and other non-canonical spellings are rejected. -fn decode_hex_32(value: &str) -> Result<[u8; 32]> { +pub(crate) fn decode_hex_32(value: &str) -> Result<[u8; 32]> { let bytes = HEXLOWER .decode(value.as_bytes()) .map_err(|error| anyhow!("invalid hex: {error}"))?; @@ -110,3 +130,15 @@ fn decode_hex_32(value: &str) -> Result<[u8; 32]> { Ok(decoded) } + +pub(crate) fn fill_random(bytes: &mut [u8]) -> Result<()> { + SysRng + .try_fill_bytes(bytes) + .map_err(|error| anyhow!("os rng: {error}")) +} + +pub(crate) fn random_32() -> Result<[u8; 32]> { + let mut bytes = [0u8; 32]; + fill_random(&mut bytes)?; + Ok(bytes) +} diff --git a/crates/concord/src/store.rs b/crates/concord/src/store.rs index 85ec7dc6..95f0aeb1 100644 --- a/crates/concord/src/store.rs +++ b/crates/concord/src/store.rs @@ -3,9 +3,13 @@ use std::sync::LazyLock; use anyhow::{Result, anyhow}; use nostr_sdk::prelude::*; +use serde::{Deserialize, Serialize}; -use crate::ChannelId; +use crate::control::{ChannelMetadata, CommunityGenesis, CommunityMetadata, ROOT_EPOCH}; +use crate::derive::control_signer_group_key; +use crate::edition::{ParsedEdition, vsk}; use crate::stream::OpenedStream; +use crate::{ChannelId, CommunityId, Epoch}; static LOCAL_KEYS: LazyLock = LazyLock::new(Keys::generate); @@ -14,6 +18,7 @@ const MARK_TAG: SingleLetterTag = SingleLetterTag::LOWERCASE_T; const MARK_VALUE: &str = "concord"; const WRAP_TAG: &str = "e"; const KIND_TAG: &str = "k"; +const STATE_PREFIX: &str = "concord/"; pub async fn cache_rumor( database: &dyn NostrDatabase, @@ -84,6 +89,146 @@ pub async fn query_rumors( Ok(rumors) } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct EntityHead { + pub entity: [u8; 32], + pub version: u64, + pub self_hash: [u8; 32], + pub rumor_id: EventId, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ChannelKeyRef { + pub id: ChannelId, + pub name: String, + pub private: bool, + pub epoch: Epoch, +} + +/// One local document per community, keyed by `concord/`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct CommunityState { + pub id: CommunityId, + pub owner: PublicKey, + pub owner_salt: [u8; 32], + pub community_root: [u8; 32], + pub root_epoch: Epoch, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub control_root: Option<[u8; 32]>, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub control_pks: BTreeMap, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub channels: Vec, + pub relays: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub heads: Vec, + pub added_at_ms: u64, +} + +impl CommunityState { + pub fn from_genesis( + genesis: &CommunityGenesis, + editions: &[ParsedEdition], + added_at_ms: u64, + ) -> Result { + let mut channels = Vec::new(); + let mut heads = Vec::with_capacity(editions.len()); + let mut relays = Vec::new(); + + for edition in editions { + heads.push(EntityHead { + entity: edition.entity, + version: edition.version, + self_hash: edition.self_hash, + rumor_id: edition.rumor_id, + }); + + match edition.subkind.as_str() { + vsk::COMMUNITY_METADATA => { + let metadata: CommunityMetadata = serde_json::from_str(&edition.content)?; + relays.extend( + metadata + .relays + .iter() + .filter_map(|relay| RelayUrl::parse(relay).ok()), + ); + } + vsk::CHANNEL_METADATA => { + let metadata: ChannelMetadata = serde_json::from_str(&edition.content)?; + channels.push(ChannelKeyRef { + id: ChannelId::from_bytes(edition.entity), + name: metadata.name, + private: metadata.private, + epoch: ROOT_EPOCH, + }); + } + _ => {} + } + } + + let control_pks = BTreeMap::from([( + ROOT_EPOCH.0, + control_signer_group_key( + &genesis.control_root, + &genesis.identity.community_id, + ROOT_EPOCH, + )? + .pk(), + )]); + + Ok(Self { + id: genesis.identity.community_id, + owner: genesis.identity.owner, + owner_salt: genesis.identity.owner_salt, + community_root: genesis.community_root, + root_epoch: ROOT_EPOCH, + control_root: Some(genesis.control_root), + control_pks, + channels, + relays, + heads, + added_at_ms, + }) + } + + pub fn identifier(&self) -> String { + state_identifier(&self.id) + } +} + +fn state_identifier(id: &CommunityId) -> String { + format!("{STATE_PREFIX}{}", id.to_hex()) +} + +pub async fn save_state(database: &D, state: &CommunityState) -> Result<()> +where + D: NostrDatabase, +{ + let event = EventBuilder::new(Kind::ApplicationSpecificData, serde_json::to_string(state)?) + .tags([Tag::identifier(state.identifier())]) + .finalize_async(&*LOCAL_KEYS) + .await?; + + database.save_event(&event).await?; + + Ok(()) +} + +pub async fn load_state(database: &D, id: &CommunityId) -> Result> +where + D: NostrDatabase, +{ + let filter = Filter::new() + .kind(Kind::ApplicationSpecificData) + .identifier(state_identifier(id)) + .limit(1); + + match database.query(filter).await?.into_iter().next() { + Some(event) => Ok(Some(serde_json::from_str(&event.content)?)), + None => Ok(None), + } +} + #[cfg(test)] mod tests { use nostr_memory::MemoryDatabase; diff --git a/crates/concord/src/stream.rs b/crates/concord/src/stream.rs index 040f77e4..2a04120d 100644 --- a/crates/concord/src/stream.rs +++ b/crates/concord/src/stream.rs @@ -6,8 +6,6 @@ use nostr_sdk::prelude::{ Event, EventBuilder, EventId, FinalizeEvent, Keys, Kind, PublicKey, Tag, Timestamp, UnsignedEvent, }; -use rand::TryRng as _; -use rand::rngs::SysRng; use crate::derive::GroupKey; use crate::{ChannelId, Epoch}; @@ -206,6 +204,25 @@ pub fn wrap_seal( wrap_kind: u16, at: Timestamp, extra: &[Tag], +) -> Result<(Event, Keys), StreamError> { + wrap_seal_with( + seal, + group.conversation(), + group.keys(), + wrap_kind, + at, + extra, + ) +} + +/// Signs with `signer` while encrypting under `conversation`. +pub fn wrap_seal_with( + seal: &Event, + conversation: &ConversationKey, + signer: &Keys, + wrap_kind: u16, + at: Timestamp, + extra: &[Tag], ) -> Result<(Event, Keys), StreamError> { if wrap_kind != KIND_WRAP && wrap_kind != KIND_WRAP_EPHEMERAL { return Err(StreamError::BadWrapKind(wrap_kind)); @@ -214,7 +231,7 @@ pub fn wrap_seal( let json = seal.as_json(); check_plaintext_cap(json.len())?; - let content = BASE64.encode(&encrypt(group.conversation(), json.as_bytes())?); + let content = BASE64.encode(&encrypt(conversation, json.as_bytes())?); let ephemeral = Keys::generate(); let mut tags = vec![Tag::public_key(ephemeral.public_key())]; @@ -223,7 +240,7 @@ pub fn wrap_seal( let wrap = EventBuilder::new(Kind::Custom(wrap_kind), content) .tags(tags) .custom_created_at(at) - .finalize(group.keys()) + .finalize(signer) .map_err(|error| StreamError::Sign(error.to_string()))?; Ok((wrap, ephemeral)) @@ -335,9 +352,7 @@ pub fn check_channel_binding( fn encrypt(conversation: &ConversationKey, plaintext: &[u8]) -> Result, StreamError> { let mut nonce = [0u8; 32]; - SysRng - .try_fill_bytes(&mut nonce) - .map_err(|error| StreamError::Encrypt(error.to_string()))?; + crate::fill_random(&mut nonce).map_err(|error| StreamError::Encrypt(error.to_string()))?; encrypt_to_bytes_with_nonce(conversation, plaintext, nonce) .map_err(|error| StreamError::Encrypt(error.to_string()))