reya/coop
1
0
Fork 0
Code Issues Pull Requests 1 Releases 3 Activity

update

Browse Source
This commit is contained in:
reya
2026-09-22 14:50:15 +07:00
parent 8914685c3d
commit 04fb70e657
7 changed files with 926 additions and 256 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/community/src/cache.rs
+38
View File
@@ -82,6 +82,44 @@ pub async fn purge_expired(client: &Client, channel: &ChannelId, now: Timestamp)
Ok(purged)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Observed {
pub author: PublicKey,
pub at_ms: u64,
}
/// The cached rumors of `channel`, keyed by the wrap they were opened from.
pub async fn wrapper_index(
client: &Client,
channel: &ChannelId,
) -> Result<BTreeMap<EventId, Observed>> {
let filter = Filter::new()
.kind(Kind::ApplicationSpecificData)
.custom_tag(MARK_TAG, MARK_VALUE)
.custom_tag(CHANNEL_TAG, channel.to_hex());
let mut index = BTreeMap::new();
for event in client.database().query(filter).await? {
let (Some(wrapper_id), Some(author)) = (
event.tags.event_ids().next(),
event.tags.public_keys().next(),
) else {
continue;
};
index.insert(
wrapper_id,
Observed {
author,
at_ms: event.created_at.as_secs().saturating_mul(1000),
},
);
}
Ok(index)
}
/// Cached rumors for `channel`, newest first, deduplicated by rumor id.
pub async fn query_rumors(
client: &Client,
crates/community/src/community.rs
+31 -10
View File
@@ -13,7 +13,7 @@ use nostr_sdk::prelude::*;
use state::NostrRegistry;
use crate::cache;
use crate::history::{self, Window, WrapPage};
use crate::history::{self, PageRegistry, Window, WrapPage};
use crate::sync::{self, Snapshot};
/// Wraps one relay returns for one page request.
@@ -102,7 +102,6 @@ pub struct Community {
state: CommunityState,
control: ControlFold,
members: BTreeSet<PublicKey>,
/// The channel the sidebar and panel show, once the user has picked one
active: Option<ChannelId>,
icon: Option<PathBuf>,
icon_ref: Option<ImageRef>,
@@ -113,12 +112,13 @@ pub struct Community {
icon_task: Option<Task<Result<()>>>,
banner_task: Option<Task<Result<()>>>,
rounds: HashMap<ChannelId, Round>,
pages: PageRegistry,
}
impl EventEmitter<CommunityEvent> for Community {}
impl Community {
pub fn new(state: CommunityState) -> Self {
pub fn new(state: CommunityState, pages: PageRegistry) -> Self {
Self {
state,
control: ControlFold::default(),
@@ -133,6 +133,7 @@ impl Community {
icon_task: None,
banner_task: None,
rounds: HashMap::new(),
pages,
}
}
@@ -262,6 +263,7 @@ impl Community {
let client = NostrRegistry::global(cx).read(cx).client();
let held = self.held_keys(&channel);
let relays = self.state.relays.clone();
let pages = self.pages.clone();
let saved = self
.state
@@ -276,7 +278,7 @@ impl Community {
}
let round = cx.background_spawn(async move {
sync_round(&client, &channel, &held, &relays, saved, intent).await
sync_round(&client, &pages, &channel, &held, &relays, saved, intent).await
});
cx.spawn(async move |this, cx| {
@@ -605,6 +607,7 @@ type RoundOutcome = (Progress, ChannelCursor);
/// Read a channel's history from the community's relays, in three passes.
async fn sync_round(
client: &Client,
pages: &PageRegistry,
channel: &ChannelId,
held: &[(Epoch, [u8; 32])],
relays: &[RelayUrl],
@@ -614,14 +617,25 @@ async fn sync_round(
let mut progress = Progress::default();
let mut round = ChannelCursor::default();
// Expired rows drop at fold time otherwise, never from disk.
let purged = cache::purge_expired(client, channel, Timestamp::now()).await?;
if purged > 0 {
log::debug!(
"community: purged {purged} expired rumor(s) from {}",
channel.to_hex()
);
}
let newest = match intent {
Intent::CatchUp => {
let page = history::page(
client,
pages,
channel,
held,
relays,
Window::newest(),
Window::opening(saved),
1,
PAGE_WRAPS,
)
@@ -636,6 +650,7 @@ async fn sync_round(
(Intent::CatchUp, Some(oldest), Some(saved_newest)) if oldest > saved_newest => {
let page = history::page(
client,
pages,
channel,
held,
relays,
@@ -659,27 +674,33 @@ async fn sync_round(
Intent::Older { .. } => saved.oldest_ms,
};
let pages = match intent {
let budget = match intent {
Intent::CatchUp => CATCH_UP_PAGES,
Intent::Older { pages } => pages,
};
let older = match resume {
Some(until) => {
// A channel already swept to the bottom has nothing older to ask for.
let older = match (saved.exhausted, resume) {
(true, _) => WrapPage {
exhausted: true,
..WrapPage::default()
},
(false, Some(until)) => {
let page = history::page(
client,
pages,
channel,
held,
relays,
Window::older_than(until),
pages,
budget,
PAGE_WRAPS,
)
.await?;
absorb(&mut progress, &page);
page
}
None => WrapPage::default(),
(false, None) => WrapPage::default(),
};
if intent == Intent::CatchUp {
crates/community/src/history.rs
+225 -79
View File
@@ -1,19 +1,24 @@
use std::collections::BTreeSet;
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::future::Future;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, MutexGuard};
use std::time::{Duration, Instant};
use anyhow::{Result, anyhow};
use anyhow::Result;
use concord::cord01::KIND_WRAP_EPHEMERAL;
use concord::cord03::{self, ChatRumor, plane_keys};
use concord::state::ChannelCursor;
use concord::{ChannelId, Epoch};
use futures::future::{Either, join_all, select};
use futures::future::{Either, select};
use nostr_sdk::prelude::*;
use crate::cache::cache_rumor;
use crate::sync::connect_relays;
/// How long one relay is given to answer one page of history.
const PAGE_TIMEOUT: Duration = Duration::from_secs(10);
/// How far below a cursor a warm window reaches back.
pub const CURSOR_OVERLAP_MS: u64 = 60_000;
/// The region of history to read.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
@@ -43,6 +48,87 @@ impl Window {
since_ms: Some(since_ms),
}
}
/// The window a channel is opened with.
pub fn opening(cursor: ChannelCursor) -> Self {
match cursor.newest_ms {
Some(newest_ms) => Self {
since_ms: Some(newest_ms.saturating_sub(CURSOR_OVERLAP_MS)),
until_ms: None,
},
None => Self::default(),
}
}
}
/// What a relay said about one page subscription.
#[derive(Debug, Clone)]
pub enum Settled {
Replayed,
Refused(String),
}
/// One relay's verdict on one page subscription.
#[derive(Debug, Clone)]
pub struct PageReport {
pub relay: RelayUrl,
pub outcome: Settled,
}
/// The page subscriptions in flight, by subscription id.
pub struct PageRegistry {
pages: Arc<Mutex<HashMap<SubscriptionId, flume::Sender<PageReport>>>>,
}
impl Default for PageRegistry {
fn default() -> Self {
Self {
pages: Arc::new(Mutex::new(HashMap::new())),
}
}
}
impl Clone for PageRegistry {
fn clone(&self) -> Self {
Self {
pages: Arc::clone(&self.pages),
}
}
}
impl PageRegistry {
pub fn register(&self, id: SubscriptionId, sender: flume::Sender<PageReport>) {
self.lock().insert(id, sender);
}
pub fn unregister(&self, id: &SubscriptionId) {
self.lock().remove(id);
}
/// Forget every page still in flight, because its round is gone.
pub fn clear(&self) {
self.lock().clear();
}
/// Hand a relay's verdict to the page that owns `id`, when it is still waiting.
pub fn deliver(&self, id: &SubscriptionId, relay: RelayUrl, outcome: Settled) {
let sender = self.lock().get(id).cloned();
let Some(sender) = sender else {
return;
};
if let Err(error) = sender.try_send(PageReport { relay, outcome }) {
log::debug!("community: a page report was not delivered: {error}");
}
}
fn lock(&self) -> MutexGuard<'_, HashMap<SubscriptionId, flume::Sender<PageReport>>> {
match self.pages.lock() {
Ok(pages) => pages,
Err(poisoned) => poisoned.into_inner(),
}
}
}
/// What one paged fetch saw.
@@ -58,8 +144,10 @@ pub struct WrapPage {
}
/// Walks a channel's history back over the community's own relays.
#[allow(clippy::too_many_arguments)]
pub async fn page(
client: &Client,
pages: &PageRegistry,
channel: &ChannelId,
held: &[(Epoch, [u8; 32])],
relays: &[RelayUrl],
@@ -77,6 +165,9 @@ pub async fn page(
});
}
// A REQ can only target a relay the pool already knows about.
connect_relays(client, relays).await;
let mut walk = Walk::new(relays, window);
let mut opened = Vec::new();
@@ -91,17 +182,19 @@ pub async fn page(
.map(|index| (index, walk.url(index).clone()))
.collect();
let answers = join_all(
asked
.iter()
.map(|(_, url)| ingest_page(client, url, &filter)),
)
.await;
let answers = ask_page(client, pages, &asked, &filter).await;
for ((index, url), answer) in asked.iter().zip(answers) {
if let Err(error) = answer {
log::warn!("community: relay {url} did not answer a history page: {error}");
walk.reject(*index);
for (index, url) in &asked {
match answers.get(url) {
Some(Settled::Replayed) => {}
Some(Settled::Refused(reason)) => {
log::warn!("community: relay {url} refused a history page: {reason}");
walk.reject(*index);
}
None => {
log::warn!("community: relay {url} did not finish a history page");
walk.reject(*index);
}
}
}
@@ -144,88 +237,99 @@ fn wrap_filter(authors: &[PublicKey], window: Window, limit: usize) -> Filter {
filter
}
/// Ask one relay for one page and wait until it has answered.
async fn ingest_page(client: &Client, url: &RelayUrl, filter: &Filter) -> Result<()> {
let relay = match client.relay(url).await? {
Some(relay) => relay,
None => {
client.add_relay(url).and_connect().await?;
client
.relay(url)
.await?
.ok_or_else(|| anyhow!("the relay was not added to the pool"))?
}
};
/// One page's verdicts, by relay.
type PageAnswers = BTreeMap<RelayUrl, Settled>;
let id = history_subscription();
let mut notifications = relay.notifications();
async fn ask_page(
client: &Client,
pages: &PageRegistry,
asked: &[(usize, RelayUrl)],
filter: &Filter,
) -> PageAnswers {
let mut answers = PageAnswers::new();
let distinct: BTreeSet<RelayUrl> = asked.iter().map(|(_, url)| url.clone()).collect();
let mut targets: Vec<(RelayUrl, Vec<Filter>)> = Vec::with_capacity(distinct.len());
// The SDK closes and unregisters the subscription itself once the relay says
// it is done, so nothing has to be unwound here.
relay
.subscribe(vec![filter.clone()])
.with_id(id.clone())
.close_on(
SubscribeAutoCloseOptions::default()
.exit_policy(ReqExitPolicy::ExitOnEOSE)
.timeout(Some(PAGE_TIMEOUT)),
)
.await?;
// One deadline for the whole page, so a relay that keeps talking cannot
// extend the wait past it.
let deadline = Instant::now() + PAGE_TIMEOUT;
loop {
let remaining = deadline.saturating_duration_since(Instant::now());
let Some(notification) = within(remaining, notifications.next()).await else {
return Err(anyhow!("the relay did not finish the page"));
};
let Some(notification) = notification else {
return Err(anyhow!("the relay's notification stream ended"));
};
let RelayNotification::Message { message } = notification else {
continue;
};
match *message {
RelayMessage::EndOfStoredEvents(ended) => {
if ended.as_ref() == &id {
return Ok(());
}
for url in &distinct {
match client.relay(url).await {
Ok(Some(_)) => targets.push((url.clone(), vec![filter.clone()])),
Ok(None) => {
log::warn!("community: relay {url} is not in the pool for a history page");
answers.insert(
url.clone(),
Settled::Refused("not in the relay pool".to_owned()),
);
}
RelayMessage::Closed {
subscription_id,
message,
} if subscription_id.as_ref() == &id && !auth_required(&message) => {
return Err(anyhow!("the relay closed the page: {message}"));
Err(error) => {
log::warn!("community: relay {url} could not be looked up: {error}");
answers.insert(url.clone(), Settled::Refused(error.to_string()));
}
_ => {}
}
}
if targets.is_empty() {
return answers;
}
let id = history_subscription();
let (sender, receiver) = flume::bounded(distinct.len());
pages.register(id.clone(), sender);
let options = SubscribeAutoCloseOptions::default()
.exit_policy(ReqExitPolicy::ExitOnEOSE)
.timeout(Some(PAGE_TIMEOUT));
match client
.subscribe(ReqTarget::manual(targets))
.with_id(id.clone())
.close_on(options)
.await
{
Ok(output) => {
for (url, reason) in output.failed {
answers.insert(url, Settled::Refused(reason));
}
let deadline = Instant::now() + PAGE_TIMEOUT;
while answers.len() < distinct.len() {
let remaining = deadline.saturating_duration_since(Instant::now());
let Some(report) = within(remaining, receiver.recv_async()).await else {
break;
};
match report {
Ok(report) => {
answers.insert(report.relay, report.outcome);
}
Err(error) => {
log::warn!("community: a history page's reports were lost: {error}");
break;
}
}
}
}
Err(error) => {
log::warn!("community: a history page REQ was refused: {error}");
}
}
pages.unregister(&id);
answers
}
/// Whether a CLOSED reason is the NIP-42 `auth-required` one, the only reason the
/// SDK recovers from on its own.
fn auth_required(reason: &str) -> bool {
pub(crate) fn auth_required(reason: &str) -> bool {
matches!(
MachineReadablePrefix::parse(reason),
Some(MachineReadablePrefix::AuthRequired)
)
}
/// A subscription id for one page of one relay, unique so a page's answer is
/// never confused with the community's standing subscription or another page's.
fn history_subscription() -> SubscriptionId {
static NEXT: AtomicU64 = AtomicU64::new(0);
SubscriptionId::new(format!(
"concord-history-{}",
NEXT.fetch_add(1, Ordering::Relaxed)
))
SubscriptionId::new(format!("history-{}", NEXT.fetch_add(1, Ordering::Relaxed)))
}
/// Await `future`, giving up after `limit`.
@@ -513,4 +617,46 @@ mod tests {
.finalize(&keys)
.expect("signs")
}
/// A cold channel asks wide; a warm one resumes at the overlap above its
/// cursor, and `older_than` never includes the boundary event itself.
#[test]
fn a_cold_window_is_open_and_a_warm_one_resumes_at_the_overlap() {
assert_eq!(Window::opening(ChannelCursor::default()), Window::default());
let warm = Window::opening(ChannelCursor {
newest_ms: Some(2_000_000),
oldest_ms: Some(1_000),
exhausted: false,
});
assert_eq!(
warm,
Window {
since_ms: Some(2_000_000 - CURSOR_OVERLAP_MS),
until_ms: None,
}
);
assert_eq!(
Window::older_than(1_000),
Window {
since_ms: None,
until_ms: Some(999),
}
);
}
/// A page whose round has moved on is simply a report nobody reads.
#[test]
fn a_page_that_moved_on_receives_nothing() {
let pages = PageRegistry::default();
let id = SubscriptionId::new("concord-history-9");
let (sender, receiver) = flume::bounded(1);
pages.register(id.clone(), sender);
pages.unregister(&id);
pages.deliver(&id, relay_url("history"), Settled::Replayed);
assert!(receiver.try_recv().is_err());
}
}
crates/community/src/lib.rs
+294 -35
View File
@@ -1,16 +1,19 @@
use std::collections::HashMap;
use std::collections::{BTreeSet, HashMap};
use std::time::{Duration, Instant};
use anyhow::Result;
use concord::cord01::KIND_WRAP;
pub use concord::cord02::CommunityMetadata;
pub use concord::cord03::{ChatMessage, ReplyRef};
use concord::state::CommunityState;
pub use concord::{ChannelId, CommunityId};
use futures::future::{Either, select};
use gpui::{App, AppContext, Context, Entity, EventEmitter, Global, Subscription, Task, Window};
use nostr_sdk::prelude::*;
use smallvec::{SmallVec, smallvec};
use state::NostrRegistry;
use crate::history::{PageRegistry, Settled, auth_required};
pub mod cache;
mod community;
pub mod history;
@@ -19,6 +22,9 @@ mod sync;
pub use community::*;
pub use sync::*;
/// How long a burst of relay notifications is collected before it is folded.
const PUMP_WINDOW: Duration = Duration::from_millis(200);
pub fn init(cx: &mut App) {
CommunityRegistry::set_global(cx.new(CommunityRegistry::new), cx);
}
@@ -33,6 +39,83 @@ enum Signal {
List,
}
/// Which standing subscription an id belongs to.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Route {
List,
Community(CommunityId),
}
fn route_of(id: &SubscriptionId) -> Option<Route> {
if sync::is_list_subscription(id) {
return Some(Route::List);
}
sync::community_of(id).map(Route::Community)
}
/// What a window of relay notifications saw, waiting to be folded once.
#[derive(Default)]
struct Batch {
list: bool,
communities: BTreeSet<CommunityId>,
}
/// Whether the pump should keep listening.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Flow {
Continue,
Stop,
}
/// Fold one notification into the window, or settle a page it belongs to.
fn route(notification: ClientNotification, pages: &PageRegistry, batch: &mut Batch) -> Flow {
match notification {
ClientNotification::Event {
subscription_id, ..
} => match route_of(&subscription_id) {
Some(Route::List) => batch.list = true,
// Only the database is told here; the fold reads it when the window closes.
Some(Route::Community(id)) => {
batch.communities.insert(id);
}
None => {}
},
ClientNotification::Message { relay_url, message } => match *message {
RelayMessage::EndOfStoredEvents(id) => {
pages.deliver(&id, relay_url, Settled::Replayed);
}
RelayMessage::Closed {
subscription_id,
message,
} if !auth_required(&message) => {
pages.deliver(
&subscription_id,
relay_url,
Settled::Refused(message.into_owned()),
);
}
_ => {}
},
ClientNotification::Shutdown => return Flow::Stop,
}
Flow::Continue
}
/// Hand the window's signals to the foreground consumer, one per community.
async fn flush(tx: &flume::Sender<Signal>, batch: &mut Batch) -> Result<()> {
for id in std::mem::take(&mut batch.communities) {
tx.send_async(Signal::Event(id)).await?;
}
if std::mem::take(&mut batch.list) {
tx.send_async(Signal::List).await?;
}
Ok(())
}
impl EventEmitter<CommunityEvent> for CommunityRegistry {}
pub struct CommunityRegistry {
@@ -44,6 +127,8 @@ pub struct CommunityRegistry {
observers: HashMap<CommunityId, Subscription>,
signal_tx: flume::Sender<Signal>,
signal_rx: flume::Receiver<Signal>,
/// The page subscriptions in flight, shared with the notification pump.
pages: PageRegistry,
tasks: SmallVec<[Task<Result<()>>; 2]>,
/// Notification listener task (cancelled on signer change)
notification_listener: Option<Task<Result<()>>>,
@@ -95,6 +180,7 @@ impl CommunityRegistry {
observers: HashMap::new(),
signal_tx: tx,
signal_rx: rx,
pages: PageRegistry::default(),
tasks: smallvec![],
notification_listener: None,
signal_consumer: None,
@@ -169,6 +255,7 @@ impl CommunityRegistry {
self.signal_consumer = None;
self.tasks.clear();
self.observers.clear();
self.pages.clear();
let nostr = NostrRegistry::global(cx);
let client = nostr.read(cx).client();
@@ -256,7 +343,7 @@ impl CommunityRegistry {
community
}
None => {
let community = cx.new(|_| Community::new(state));
let community = cx.new(|_| Community::new(state, self.pages.clone()));
self.observers.insert(
id,
@@ -334,7 +421,7 @@ impl CommunityRegistry {
};
let subscription = sync::subscription_id(&id);
let filter = sync::subscription_filter(&planes);
let filter = sync::live_filter(&planes, sync::live_window(&state));
let relays = key.relays().to_vec();
self.synced.insert(id, key);
@@ -360,34 +447,51 @@ impl CommunityRegistry {
let tx = self.signal_tx.clone();
let rx = self.signal_rx.clone();
let pages = self.pages.clone();
let executor = cx.background_executor().clone();
self.notification_listener = Some(cx.background_spawn(async move {
let mut notifications = client.notifications();
let mut batch = Batch::default();
while let Some(notification) = notifications.next().await {
let ClientNotification::Event {
subscription_id,
event,
..
} = notification
else {
continue;
};
if sync::is_list_subscription(&subscription_id) {
tx.send_async(Signal::List).await?;
continue;
'outer: loop {
match notifications.next().await {
Some(notification) => {
if route(notification, &pages, &mut batch) == Flow::Stop {
flush(&tx, &mut batch).await?;
break 'outer;
}
}
None => break 'outer,
}
if event.kind != Kind::from(KIND_WRAP) {
continue;
let deadline = Instant::now() + PUMP_WINDOW;
loop {
let now = Instant::now();
if now >= deadline {
break;
}
let timer = executor.timer(deadline - now);
let next = notifications.next();
futures::pin_mut!(timer);
futures::pin_mut!(next);
match select(next, timer).await {
Either::Left((Some(notification), _)) => {
if route(notification, &pages, &mut batch) == Flow::Stop {
flush(&tx, &mut batch).await?;
break 'outer;
}
}
Either::Left((None, _)) => break 'outer,
Either::Right(_) => break,
}
}
let Some(id) = sync::community_of(&subscription_id) else {
continue;
};
tx.send_async(Signal::Event(id)).await?;
flush(&tx, &mut batch).await?;
}
Ok(())
@@ -413,24 +517,34 @@ async fn subscribe(
) -> Result<()> {
client.unsubscribe(id).await?;
if relays.is_empty() {
log::warn!("community {id}: no relay to subscribe to");
return Ok(());
}
let mut targets: Vec<(RelayUrl, Vec<Filter>)> = Vec::with_capacity(relays.len());
for url in relays {
if let Err(error) = client.add_relay(url).and_connect().await {
log::warn!("community {id}: failed to add relay {url}: {error}");
}
match client.relay(url).await {
Ok(Some(_)) => targets.push((url.clone(), vec![filter.clone()])),
Ok(None) => log::warn!("community {id}: relay {url} is not in the pool"),
Err(error) => log::warn!("community {id}: relay {url} could not be looked up: {error}"),
}
}
let target = if relays.is_empty() {
ReqTarget::auto(vec![filter])
} else {
ReqTarget::manual(
relays
.iter()
.map(|url| (url.clone(), vec![filter.clone()]))
.collect::<Vec<_>>(),
)
};
if targets.is_empty() {
log::warn!("community {id}: no relay accepted the standing subscription");
return Ok(());
}
let output = client.subscribe(target).with_id(id.clone()).await?;
let output = client
.subscribe(ReqTarget::manual(targets))
.with_id(id.clone())
.await?;
if !output.failed.is_empty() {
log::warn!(
@@ -441,3 +555,148 @@ async fn subscribe(
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
fn relay() -> RelayUrl {
RelayUrl::parse("wss://relay.example").expect("a url")
}
fn event(subscription_id: SubscriptionId) -> ClientNotification {
let keys = Keys::generate();
let event = EventBuilder::new(Kind::TextNote, "hi")
.finalize(&keys)
.expect("signs");
ClientNotification::Event {
relay_url: relay(),
subscription_id,
event: Box::new(event),
}
}
fn message(message: RelayMessage<'static>) -> ClientNotification {
ClientNotification::Message {
relay_url: relay(),
message: Box::new(message),
}
}
#[test]
fn only_the_list_and_a_community_id_have_a_route() {
let id = CommunityId::from_bytes([0x42; 32]);
assert_eq!(route_of(&sync::list_subscription_id()), Some(Route::List));
assert_eq!(
route_of(&sync::subscription_id(&id)),
Some(Route::Community(id))
);
assert_eq!(route_of(&SubscriptionId::new("concord-history-3")), None);
assert_eq!(
route_of(&SubscriptionId::new("some/other/subscription")),
None
);
}
/// A burst within one window folds each community once, and the list once,
/// however many events arrived.
#[test]
fn a_burst_collapses_to_one_signal_per_community() {
let pages = PageRegistry::default();
let community = CommunityId::from_bytes([0x42; 32]);
let mut batch = Batch::default();
let plane = event(sync::subscription_id(&community));
for _ in 0..50 {
assert_eq!(route(plane.clone(), &pages, &mut batch), Flow::Continue);
}
route(event(sync::list_subscription_id()), &pages, &mut batch);
// A page's event is folded from the database later, not routed here.
route(
event(SubscriptionId::new("concord-history-7")),
&pages,
&mut batch,
);
assert!(batch.list);
assert_eq!(batch.communities, BTreeSet::from([community]));
}
#[test]
fn an_eose_settles_only_the_page_that_owns_the_id() {
let pages = PageRegistry::default();
let mine = SubscriptionId::new("concord-history-1");
let other = SubscriptionId::new("concord-history-2");
let (mine_tx, mine_rx) = flume::bounded(1);
let (other_tx, other_rx) = flume::bounded(1);
pages.register(mine.clone(), mine_tx);
pages.register(other, other_tx);
let mut batch = Batch::default();
let flow = route(message(RelayMessage::eose(mine)), &pages, &mut batch);
assert_eq!(flow, Flow::Continue);
assert!(matches!(
mine_rx.try_recv().expect("a report").outcome,
Settled::Replayed
));
assert!(other_rx.try_recv().is_err());
}
#[test]
fn a_refused_page_settles_its_relay_as_refused() {
let pages = PageRegistry::default();
let id = SubscriptionId::new("concord-history-1");
let (sender, receiver) = flume::bounded(1);
pages.register(id.clone(), sender);
let mut batch = Batch::default();
route(
message(RelayMessage::closed(id, "blocked: not allowed")),
&pages,
&mut batch,
);
match receiver.try_recv().expect("a report").outcome {
Settled::Refused(reason) => assert!(reason.contains("blocked")),
other => panic!("expected a refusal, got {other:?}"),
}
}
/// The SDK re-issues an `auth-required` REQ under the same id after AUTH, so
/// the page keeps waiting rather than writing the relay off.
#[test]
fn an_auth_required_close_settles_nothing() {
let pages = PageRegistry::default();
let id = SubscriptionId::new("concord-history-1");
let (sender, receiver) = flume::bounded(1);
pages.register(id.clone(), sender);
let mut batch = Batch::default();
route(
message(RelayMessage::closed(
id,
"auth-required: please authenticate",
)),
&pages,
&mut batch,
);
assert!(receiver.try_recv().is_err());
}
#[test]
fn a_shutdown_stops_the_pump() {
let pages = PageRegistry::default();
let mut batch = Batch::default();
assert_eq!(
route(ClientNotification::Shutdown, &pages, &mut batch),
Flow::Stop
);
}
}
crates/community/src/sync.rs
+120 -21
View File
@@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet};
use std::path::PathBuf;
use anyhow::{Context, Result};
use concord::cord01::KIND_WRAP;
use concord::cord01::KIND_WRAP_EPHEMERAL;
use concord::cord02::list::{CommunityList, KIND_COMMUNITY_LIST};
use concord::cord02::{self, ControlFold, ImageRef};
use concord::cord04::AuthorityCitation;
@@ -16,7 +16,11 @@ use gpui::AsyncApp;
use nostr_sdk::prelude::*;
use state::UniversalSigner;
use crate::cache;
use crate::cache::{self, Observed};
use crate::history::{CURSOR_OVERLAP_MS, Window};
/// How much of what a relay stores a cold subscription replays per relay.
const LIVE_REPLAY: usize = 500;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum PlaneKind {
@@ -63,11 +67,13 @@ pub fn planes(state: &CommunityState) -> Result<Vec<Plane>> {
});
for channel in &state.channels {
if channel.private {
continue;
}
let secret = match (channel.private, channel.key) {
(true, None) => continue,
(true, Some(key)) => key,
(false, _) => state.community_root,
};
let group = channel_group_key(&state.community_root, &channel.id, channel.epoch)?;
let group = channel_group_key(&secret, &channel.id, channel.epoch)?;
planes.push(Plane {
kind: PlaneKind::Channel(channel.id, channel.epoch),
address: group.pk(),
@@ -78,12 +84,43 @@ pub fn planes(state: &CommunityState) -> Result<Vec<Plane>> {
Ok(planes)
}
pub fn subscription_filter(planes: &[Plane]) -> Filter {
pub fn plane_filter(planes: &[Plane]) -> Filter {
Filter::new()
.kinds([Kind::from(KIND_WRAP)])
.kinds([Kind::GiftWrap, Kind::Custom(KIND_WRAP_EPHEMERAL)])
.authors(planes.iter().map(|plane| plane.address))
}
pub fn live_filter(planes: &[Plane], window: Window) -> Filter {
let mut filter = plane_filter(planes);
if let Some(until_ms) = window.until_ms {
filter = filter.until(Timestamp::from_secs(until_ms / 1000));
}
if let Some(since_ms) = window.since_ms {
filter = filter.since(Timestamp::from_secs(since_ms / 1000));
}
filter.limit(LIVE_REPLAY)
}
/// The window a community's standing subscription opens with.
pub fn live_window(state: &CommunityState) -> Window {
let floor = state
.channels
.iter()
.filter_map(|channel| state.cursors.get(&channel.id)?.newest_ms)
.min();
match floor {
Some(floor) => Window {
since_ms: Some(floor.saturating_sub(CURSOR_OVERLAP_MS)),
until_ms: None,
},
None => Window::default(),
}
}
/// The subscription id carrying a community's planes.
pub fn subscription_id(id: &CommunityId) -> SubscriptionId {
SubscriptionId::new(id.to_hex())
@@ -429,14 +466,20 @@ pub async fn fold(client: &Client, state: &CommunityState) -> Result<Option<Snap
return Ok(None);
}
let wraps = client
.database()
.query(subscription_filter(&planes))
.await?;
let wraps = client.database().query(plane_filter(&planes)).await?;
let mut editions = Vec::new();
let mut observed: BTreeMap<PublicKey, u64> = BTreeMap::new();
let mut guestbook_rumors = Vec::new();
let mut cached: BTreeMap<ChannelId, BTreeMap<EventId, Observed>> = BTreeMap::new();
for plane in &planes {
if let PlaneKind::Channel(channel, _) = plane.kind
&& !cached.contains_key(&channel)
{
cached.insert(channel, cache::wrapper_index(client, &channel).await?);
}
}
for wrap in &wraps {
let Some(plane) = planes.iter().find(|plane| plane.address == wrap.pubkey) else {
@@ -457,6 +500,11 @@ pub async fn fold(client: &Client, state: &CommunityState) -> Result<Option<Snap
}
}
PlaneKind::Channel(channel, epoch) => {
if let Some(row) = cached.get(&channel).and_then(|index| index.get(&wrap.id)) {
observe(&mut observed, row.author, row.at_ms);
continue;
}
if let Ok((opened, rumor)) =
concord::cord03::open(wrap, &plane.group, &channel, epoch)
{
@@ -496,6 +544,7 @@ pub async fn fold(client: &Client, state: &CommunityState) -> Result<Option<Snap
let now_ms = Timestamp::now().as_secs().saturating_mul(1000);
let coalesced = cord02::guestbook::coalesce(&guestbook_rumors, now_ms, None, can_kick);
let mut members = cord02::guestbook::complete_memberlist(
&coalesced,
&observed,
@@ -538,10 +587,11 @@ mod tests {
}
#[test]
fn planes_address_the_control_guestbook_and_only_public_channels() {
fn planes_address_the_control_guestbook_and_every_readable_channel() {
let owner = Keys::generate().public_key();
let control_pk = Keys::generate().public_key();
let general = ChannelId::from_bytes([0x9c; 32]);
let staff = ChannelId::from_bytes([0x9d; 32]);
let state = CommunityState {
id: CommunityId::from_bytes([0x42; 32]),
@@ -561,12 +611,19 @@ mod tests {
key: None,
},
concord::state::ChannelKeyRef {
id: ChannelId::from_bytes([0x9d; 32]),
id: staff,
name: "staff".to_owned(),
private: true,
epoch: Epoch(0),
key: Some([0x04; 32]),
},
concord::state::ChannelKeyRef {
id: ChannelId::from_bytes([0x9e; 32]),
name: "locked".to_owned(),
private: true,
epoch: Epoch(0),
key: None,
},
],
relays: vec![RelayUrl::parse("wss://relay.example").expect("a url")],
heads: Vec::new(),
@@ -578,7 +635,7 @@ mod tests {
let planes = planes(&state).expect("planes");
assert_eq!(planes.len(), 3);
assert_eq!(planes.len(), 4);
assert!(planes.iter().any(|plane| plane.address == control_pk));
assert!(
planes
@@ -590,11 +647,53 @@ mod tests {
.iter()
.any(|plane| matches!(plane.kind, PlaneKind::Channel(id, _) if id == general))
);
assert!(
planes
.iter()
.any(|plane| matches!(plane.kind, PlaneKind::Channel(id, _) if id == staff)),
"a private channel whose key is held is subscribed"
);
let filter = subscription_filter(&planes);
let filter = plane_filter(&planes);
let addresses: BTreeSet<PublicKey> = planes.iter().map(|plane| plane.address).collect();
assert_eq!(filter.authors, Some(addresses));
assert_eq!(filter.kinds, Some(BTreeSet::from([Kind::from(KIND_WRAP)])));
assert_eq!(
filter.kinds,
Some(BTreeSet::from([
Kind::GiftWrap,
Kind::Custom(KIND_WRAP_EPHEMERAL)
])),
"the standing subscription asks for both wrap kinds"
);
}
/// A cold subscription asks wide; a warm one resumes at the oldest held
/// cursor, minus the overlap, so no channel's new region is skipped.
#[test]
fn the_live_window_is_wide_cold_and_resumes_at_the_oldest_cursor_warm() {
let mut state = held(
CommunityId::from_bytes([0x42; 32]),
Keys::generate().public_key(),
);
let channel = state.channels[0].id;
assert_eq!(live_window(&state), Window::default());
state.cursors.insert(
channel,
concord::state::ChannelCursor {
newest_ms: Some(2_000_000),
oldest_ms: Some(1_000),
exhausted: false,
},
);
assert_eq!(
live_window(&state),
Window {
since_ms: Some(2_000_000 - CURSOR_OVERLAP_MS),
until_ms: None,
}
);
}
fn metadata(name: &str) -> cord02::CommunityMetadata {
@@ -661,16 +760,16 @@ mod tests {
.expect("loads");
assert_eq!(loaded, vec![created.clone()]);
// The subscription filter must address the genesis wraps, or the registry
// would listen to a plane nothing is ever published on.
// The plane filter must address the genesis wraps, or a fold would
// read a plane nothing is ever published on.
let planes = planes(&created).expect("planes");
let wraps = client
.database()
.query(subscription_filter(&planes))
.query(plane_filter(&planes))
.await
.expect("queries");
assert_eq!(wraps.len(), created.heads.len());
assert!(wraps.iter().all(|wrap| wrap.kind == Kind::from(KIND_WRAP)));
assert!(wraps.iter().all(|wrap| wrap.kind == Kind::GiftWrap));
let snapshot = fold(&client, &created)
.await