update backend
This commit is contained in:
@@ -0,0 +1,516 @@
|
||||
use std::collections::BTreeSet;
|
||||
use std::future::Future;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use anyhow::{Result, anyhow};
|
||||
use concord::cord01::KIND_WRAP_EPHEMERAL;
|
||||
use concord::cord03::{self, ChatRumor, plane_keys};
|
||||
use concord::{ChannelId, Epoch};
|
||||
use futures::future::{Either, join_all, select};
|
||||
use nostr_sdk::prelude::*;
|
||||
|
||||
use crate::cache::cache_rumor;
|
||||
|
||||
/// How long one relay is given to answer one page of history.
|
||||
const PAGE_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
/// The region of history to read.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct Window {
|
||||
pub until_ms: Option<u64>,
|
||||
pub since_ms: Option<u64>,
|
||||
}
|
||||
|
||||
impl Window {
|
||||
/// The newest wraps, with no older bound.
|
||||
pub fn newest() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// The wraps strictly older than `oldest_ms`.
|
||||
pub fn older_than(oldest_ms: u64) -> Self {
|
||||
Self {
|
||||
until_ms: Some(oldest_ms.saturating_sub(1)),
|
||||
since_ms: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The region between `since_ms` and `oldest_ms`, both inclusive.
|
||||
pub fn between(since_ms: u64, oldest_ms: u64) -> Self {
|
||||
Self {
|
||||
until_ms: Some(oldest_ms.saturating_sub(1)),
|
||||
since_ms: Some(since_ms),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What one paged fetch saw.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct WrapPage {
|
||||
pub opened: Vec<ChatRumor>,
|
||||
pub raw: usize,
|
||||
pub newest_ms: Option<u64>,
|
||||
pub oldest_ms: Option<u64>,
|
||||
pub exhausted: bool,
|
||||
pub failed: bool,
|
||||
pub errors: usize,
|
||||
}
|
||||
|
||||
/// Walks a channel's history back over the community's own relays.
|
||||
pub async fn page(
|
||||
client: &Client,
|
||||
channel: &ChannelId,
|
||||
held: &[(Epoch, [u8; 32])],
|
||||
relays: &[RelayUrl],
|
||||
window: Window,
|
||||
max_pages: usize,
|
||||
limit: usize,
|
||||
) -> Result<WrapPage> {
|
||||
let planes = plane_keys(held, channel)?;
|
||||
let authors: Vec<PublicKey> = planes.iter().map(|(_, group)| group.pk()).collect();
|
||||
|
||||
if authors.is_empty() || relays.is_empty() || limit == 0 {
|
||||
return Ok(WrapPage {
|
||||
failed: true,
|
||||
..WrapPage::default()
|
||||
});
|
||||
}
|
||||
|
||||
let mut walk = Walk::new(relays, window);
|
||||
let mut opened = Vec::new();
|
||||
|
||||
for _ in 0..max_pages {
|
||||
if walk.is_done() {
|
||||
break;
|
||||
}
|
||||
|
||||
let filter = wrap_filter(&authors, walk.region(), limit);
|
||||
let asked: Vec<(usize, RelayUrl)> = walk
|
||||
.live()
|
||||
.map(|index| (index, walk.url(index).clone()))
|
||||
.collect();
|
||||
|
||||
let answers = join_all(
|
||||
asked
|
||||
.iter()
|
||||
.map(|(_, url)| ingest_page(client, url, &filter)),
|
||||
)
|
||||
.await;
|
||||
|
||||
for ((index, url), answer) in asked.iter().zip(answers) {
|
||||
if let Err(error) = answer {
|
||||
log::warn!("community: relay {url} did not answer a history page: {error}");
|
||||
walk.reject(*index);
|
||||
}
|
||||
}
|
||||
|
||||
let answered = client.database().query(filter).await?;
|
||||
|
||||
for wrap in walk.accept(answered, limit) {
|
||||
let Some((epoch, group)) = planes.iter().find(|(_, group)| group.pk() == wrap.pubkey)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let Ok((stream, rumor)) = cord03::open(&wrap, group, channel, *epoch) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if cache_rumor(client, channel, &stream).await? {
|
||||
opened.push(rumor);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(walk.finish(opened))
|
||||
}
|
||||
|
||||
/// The one filter a page is asked for.
|
||||
fn wrap_filter(authors: &[PublicKey], window: Window, limit: usize) -> Filter {
|
||||
let mut filter = Filter::new()
|
||||
.kinds([Kind::GiftWrap, Kind::Custom(KIND_WRAP_EPHEMERAL)])
|
||||
.authors(authors.iter().copied())
|
||||
.limit(limit);
|
||||
|
||||
if let Some(until_ms) = window.until_ms {
|
||||
filter = filter.until(Timestamp::from_secs(until_ms / 1000));
|
||||
}
|
||||
|
||||
if let Some(since_ms) = window.since_ms {
|
||||
filter = filter.since(Timestamp::from_secs(since_ms / 1000));
|
||||
}
|
||||
|
||||
filter
|
||||
}
|
||||
|
||||
/// Ask one relay for one page and wait until it has answered.
|
||||
async fn ingest_page(client: &Client, url: &RelayUrl, filter: &Filter) -> Result<()> {
|
||||
let relay = match client.relay(url).await? {
|
||||
Some(relay) => relay,
|
||||
None => {
|
||||
client.add_relay(url).and_connect().await?;
|
||||
client
|
||||
.relay(url)
|
||||
.await?
|
||||
.ok_or_else(|| anyhow!("the relay was not added to the pool"))?
|
||||
}
|
||||
};
|
||||
|
||||
let id = history_subscription();
|
||||
let mut notifications = relay.notifications();
|
||||
|
||||
// The SDK closes and unregisters the subscription itself once the relay says
|
||||
// it is done, so nothing has to be unwound here.
|
||||
relay
|
||||
.subscribe(vec![filter.clone()])
|
||||
.with_id(id.clone())
|
||||
.close_on(
|
||||
SubscribeAutoCloseOptions::default()
|
||||
.exit_policy(ReqExitPolicy::ExitOnEOSE)
|
||||
.timeout(Some(PAGE_TIMEOUT)),
|
||||
)
|
||||
.await?;
|
||||
|
||||
// One deadline for the whole page, so a relay that keeps talking cannot
|
||||
// extend the wait past it.
|
||||
let deadline = Instant::now() + PAGE_TIMEOUT;
|
||||
|
||||
loop {
|
||||
let remaining = deadline.saturating_duration_since(Instant::now());
|
||||
|
||||
let Some(notification) = within(remaining, notifications.next()).await else {
|
||||
return Err(anyhow!("the relay did not finish the page"));
|
||||
};
|
||||
|
||||
let Some(notification) = notification else {
|
||||
return Err(anyhow!("the relay's notification stream ended"));
|
||||
};
|
||||
|
||||
let RelayNotification::Message { message } = notification else {
|
||||
continue;
|
||||
};
|
||||
|
||||
match *message {
|
||||
RelayMessage::EndOfStoredEvents(ended) => {
|
||||
if ended.as_ref() == &id {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
RelayMessage::Closed {
|
||||
subscription_id,
|
||||
message,
|
||||
} if subscription_id.as_ref() == &id && !auth_required(&message) => {
|
||||
return Err(anyhow!("the relay closed the page: {message}"));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a CLOSED reason is the NIP-42 `auth-required` one, the only reason the
|
||||
/// SDK recovers from on its own.
|
||||
fn auth_required(reason: &str) -> bool {
|
||||
matches!(
|
||||
MachineReadablePrefix::parse(reason),
|
||||
Some(MachineReadablePrefix::AuthRequired)
|
||||
)
|
||||
}
|
||||
|
||||
/// A subscription id for one page of one relay, unique so a page's answer is
|
||||
/// never confused with the community's standing subscription or another page's.
|
||||
fn history_subscription() -> SubscriptionId {
|
||||
static NEXT: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
SubscriptionId::new(format!(
|
||||
"concord-history-{}",
|
||||
NEXT.fetch_add(1, Ordering::Relaxed)
|
||||
))
|
||||
}
|
||||
|
||||
/// Await `future`, giving up after `limit`.
|
||||
async fn within<F>(limit: Duration, future: F) -> Option<F::Output>
|
||||
where
|
||||
F: Future,
|
||||
{
|
||||
let future = std::pin::pin!(future);
|
||||
let deadline = std::pin::pin!(smol::Timer::after(limit));
|
||||
|
||||
match select(future, deadline).await {
|
||||
Either::Left((output, _)) => Some(output),
|
||||
Either::Right(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// One pass over a channel's history, page by page.
|
||||
#[derive(Debug)]
|
||||
struct Walk {
|
||||
relays: Vec<Walker>,
|
||||
since_ms: Option<u64>,
|
||||
/// The inclusive upper bound of the next page.
|
||||
cursor: Option<u64>,
|
||||
seen: BTreeSet<EventId>,
|
||||
newest_ms: Option<u64>,
|
||||
oldest_ms: Option<u64>,
|
||||
raw: usize,
|
||||
errors: usize,
|
||||
/// A short page ended the walk.
|
||||
bottom: bool,
|
||||
}
|
||||
|
||||
/// One relay's standing in a walk.
|
||||
#[derive(Debug)]
|
||||
struct Walker {
|
||||
url: RelayUrl,
|
||||
dead: bool,
|
||||
}
|
||||
|
||||
impl Walk {
|
||||
fn new(relays: &[RelayUrl], window: Window) -> Self {
|
||||
Self {
|
||||
relays: relays
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|url| Walker { url, dead: false })
|
||||
.collect(),
|
||||
since_ms: window.since_ms,
|
||||
cursor: window.until_ms,
|
||||
seen: BTreeSet::new(),
|
||||
newest_ms: None,
|
||||
oldest_ms: None,
|
||||
raw: 0,
|
||||
errors: 0,
|
||||
bottom: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_done(&self) -> bool {
|
||||
self.bottom || self.relays.iter().all(|walker| walker.dead)
|
||||
}
|
||||
|
||||
/// The region the next page asks for.
|
||||
fn region(&self) -> Window {
|
||||
Window {
|
||||
until_ms: self.cursor,
|
||||
since_ms: self.since_ms,
|
||||
}
|
||||
}
|
||||
|
||||
fn live(&self) -> impl Iterator<Item = usize> + '_ {
|
||||
(0..self.relays.len()).filter(|&index| !self.relays[index].dead)
|
||||
}
|
||||
|
||||
fn url(&self, index: usize) -> &RelayUrl {
|
||||
&self.relays[index].url
|
||||
}
|
||||
|
||||
fn reject(&mut self, index: usize) {
|
||||
self.relays[index].dead = true;
|
||||
self.errors += 1;
|
||||
}
|
||||
|
||||
fn accept(&mut self, page: BTreeSet<Event>, limit: usize) -> Vec<Event> {
|
||||
if page.len() < limit {
|
||||
self.bottom = true;
|
||||
}
|
||||
|
||||
let mut oldest: Option<u64> = None;
|
||||
let mut events = Vec::with_capacity(page.len());
|
||||
|
||||
for event in page {
|
||||
let at_ms = event.created_at.as_secs().saturating_mul(1000);
|
||||
self.newest_ms = Some(self.newest_ms.map_or(at_ms, |newest| newest.max(at_ms)));
|
||||
oldest = Some(oldest.map_or(at_ms, |oldest| oldest.min(at_ms)));
|
||||
|
||||
if self.seen.insert(event.id) {
|
||||
self.raw += 1;
|
||||
events.push(event);
|
||||
}
|
||||
}
|
||||
|
||||
match oldest {
|
||||
Some(oldest) if oldest > 0 => self.cursor = Some(oldest - 1),
|
||||
Some(_) => self.bottom = true,
|
||||
None => {}
|
||||
}
|
||||
|
||||
events
|
||||
}
|
||||
|
||||
fn finish(self, opened: Vec<ChatRumor>) -> WrapPage {
|
||||
let swept = self.bottom && self.errors == 0;
|
||||
|
||||
WrapPage {
|
||||
opened,
|
||||
raw: self.raw,
|
||||
newest_ms: self.newest_ms,
|
||||
oldest_ms: self.oldest_ms,
|
||||
exhausted: swept && self.raw > 0,
|
||||
failed: self.errors > 0 || (self.bottom && self.raw == 0),
|
||||
errors: self.errors,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::cmp::Reverse;
|
||||
|
||||
use concord::cord03::{build_message, seal_rumor};
|
||||
use concord::derive::channel_group_key;
|
||||
|
||||
use super::*;
|
||||
|
||||
const SECRET: [u8; 32] = [0x07u8; 32];
|
||||
const NEXT_SECRET: [u8; 32] = [0x11u8; 32];
|
||||
|
||||
fn serve_page(database: &BTreeSet<Event>, window: Window, limit: usize) -> BTreeSet<Event> {
|
||||
let mut events: Vec<Event> = database
|
||||
.iter()
|
||||
.filter(|event| {
|
||||
let at_ms = event.created_at.as_secs().saturating_mul(1000);
|
||||
window.until_ms.is_none_or(|until| at_ms <= until)
|
||||
&& window.since_ms.is_none_or(|since| at_ms >= since)
|
||||
})
|
||||
.cloned()
|
||||
.collect();
|
||||
|
||||
events.sort_by_key(|event| Reverse(event.created_at));
|
||||
events.truncate(limit);
|
||||
events.into_iter().collect()
|
||||
}
|
||||
|
||||
fn relay_url(host: &str) -> RelayUrl {
|
||||
RelayUrl::parse(&format!("wss://{host}.example.com")).expect("parses")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_walk_pages_back_across_a_rekey() {
|
||||
let channel = ChannelId::from_bytes([0x9cu8; 32]);
|
||||
let author = Keys::generate();
|
||||
let held = [(Epoch(0), SECRET), (Epoch(1), NEXT_SECRET)];
|
||||
let planes = plane_keys(&held, &channel).expect("derives");
|
||||
|
||||
// Three messages a second apart: a page boundary falls between each.
|
||||
let base = 1_700_000_000_000;
|
||||
let mut relay: BTreeSet<Event> = BTreeSet::new();
|
||||
|
||||
for (content, secret, epoch, at_ms) in [
|
||||
("before the rekey", &SECRET, Epoch(0), base),
|
||||
("still before", &SECRET, Epoch(0), base + 1_000),
|
||||
("after the rekey", &NEXT_SECRET, Epoch(1), base + 2_000),
|
||||
] {
|
||||
let group = channel_group_key(secret, &channel, epoch).expect("derives");
|
||||
let rumor = build_message(
|
||||
author.public_key(),
|
||||
&channel,
|
||||
epoch,
|
||||
content,
|
||||
None,
|
||||
at_ms,
|
||||
None,
|
||||
);
|
||||
relay.insert(
|
||||
smol::block_on(seal_rumor(&rumor, &group, &author, false))
|
||||
.expect("seals")
|
||||
.0,
|
||||
);
|
||||
}
|
||||
|
||||
let mut walk = Walk::new(&[relay_url("history")], Window::newest());
|
||||
let mut found = Vec::new();
|
||||
let mut pages = 0;
|
||||
|
||||
while !walk.is_done() && pages < 10 {
|
||||
pages += 1;
|
||||
let page = serve_page(&relay, walk.region(), 2);
|
||||
|
||||
for wrap in walk.accept(page, 2) {
|
||||
let Some((epoch, group)) =
|
||||
planes.iter().find(|(_, group)| group.pk() == wrap.pubkey)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let (_, rumor) = cord03::open(&wrap, group, &channel, *epoch).expect("opens");
|
||||
found.push(rumor);
|
||||
}
|
||||
}
|
||||
|
||||
found.sort_by_key(|rumor| (Reverse(rumor.at_ms), rumor.id));
|
||||
|
||||
let contents: Vec<&str> = found.iter().map(|rumor| rumor.content.as_str()).collect();
|
||||
assert_eq!(
|
||||
contents,
|
||||
["after the rekey", "still before", "before the rekey"]
|
||||
);
|
||||
|
||||
let page = walk.finish(Vec::new());
|
||||
assert!(page.exhausted);
|
||||
assert!(!page.failed);
|
||||
assert_eq!(page.raw, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_answer_never_seals_the_channel() {
|
||||
let database: BTreeSet<Event> = BTreeSet::new();
|
||||
let mut walk = Walk::new(&[relay_url("history")], Window::newest());
|
||||
|
||||
let page = serve_page(&database, walk.region(), 50);
|
||||
assert!(walk.accept(page, 50).is_empty());
|
||||
|
||||
let page = walk.finish(Vec::new());
|
||||
assert!(page.failed);
|
||||
assert!(!page.exhausted);
|
||||
assert_eq!(page.raw, 0);
|
||||
assert_eq!(page.oldest_ms, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_silent_relay_blocks_the_bottom() {
|
||||
let database: BTreeSet<Event> = BTreeSet::new();
|
||||
let mut walk = Walk::new(
|
||||
&[relay_url("history"), relay_url("archive")],
|
||||
Window::newest(),
|
||||
);
|
||||
|
||||
// One relay answered the empty page; the other never answered at all, so
|
||||
// its share of the region was never read and the walk must not seal.
|
||||
walk.reject(1);
|
||||
let page = serve_page(&database, walk.region(), 50);
|
||||
assert!(walk.accept(page, 50).is_empty());
|
||||
|
||||
let page = walk.finish(Vec::new());
|
||||
assert!(page.failed);
|
||||
assert!(!page.exhausted);
|
||||
assert_eq!(page.errors, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_page_boundary_is_exclusive() {
|
||||
let database = BTreeSet::from([event_at(1_700_000_000_000), event_at(1_700_000_001_000)]);
|
||||
let mut walk = Walk::new(&[relay_url("history")], Window::newest());
|
||||
|
||||
let first = walk.accept(serve_page(&database, walk.region(), 1), 1);
|
||||
let second = walk.accept(serve_page(&database, walk.region(), 1), 1);
|
||||
|
||||
assert_eq!(first.len(), 1);
|
||||
assert_eq!(second.len(), 1);
|
||||
assert_ne!(first[0].id, second[0].id);
|
||||
|
||||
let oldest = second
|
||||
.iter()
|
||||
.map(|event| event.created_at.as_secs() * 1000)
|
||||
.min()
|
||||
.expect("one wrap");
|
||||
assert_eq!(oldest, 1_700_000_000_000);
|
||||
}
|
||||
|
||||
fn event_at(at_ms: u64) -> Event {
|
||||
let keys = Keys::generate();
|
||||
EventBuilder::new(Kind::TextNote, "page")
|
||||
.custom_created_at(Timestamp::from_secs(at_ms / 1000))
|
||||
.finalize(&keys)
|
||||
.expect("signs")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user