update

crates/community/src/sync.rs

@@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet};
 use std::path::PathBuf;
 
 use anyhow::{Context, Result};
-use concord::cord01::KIND_WRAP;
+use concord::cord01::KIND_WRAP_EPHEMERAL;
 use concord::cord02::list::{CommunityList, KIND_COMMUNITY_LIST};
 use concord::cord02::{self, ControlFold, ImageRef};
 use concord::cord04::AuthorityCitation;
@@ -16,7 +16,11 @@ use gpui::AsyncApp;
 use nostr_sdk::prelude::*;
 use state::UniversalSigner;
 
-use crate::cache;
+use crate::cache::{self, Observed};
+use crate::history::{CURSOR_OVERLAP_MS, Window};
+
+/// How much of what a relay stores a cold subscription replays per relay.
+const LIVE_REPLAY: usize = 500;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum PlaneKind {
@@ -63,11 +67,13 @@ pub fn planes(state: &CommunityState) -> Result<Vec<Plane>> {
     });
 
     for channel in &state.channels {
-        if channel.private {
-            continue;
-        }
+        let secret = match (channel.private, channel.key) {
+            (true, None) => continue,
+            (true, Some(key)) => key,
+            (false, _) => state.community_root,
+        };
 
-        let group = channel_group_key(&state.community_root, &channel.id, channel.epoch)?;
+        let group = channel_group_key(&secret, &channel.id, channel.epoch)?;
         planes.push(Plane {
             kind: PlaneKind::Channel(channel.id, channel.epoch),
             address: group.pk(),
@@ -78,12 +84,43 @@ pub fn planes(state: &CommunityState) -> Result<Vec<Plane>> {
     Ok(planes)
 }
 
-pub fn subscription_filter(planes: &[Plane]) -> Filter {
+pub fn plane_filter(planes: &[Plane]) -> Filter {
     Filter::new()
-        .kinds([Kind::from(KIND_WRAP)])
+        .kinds([Kind::GiftWrap, Kind::Custom(KIND_WRAP_EPHEMERAL)])
         .authors(planes.iter().map(|plane| plane.address))
 }
 
+pub fn live_filter(planes: &[Plane], window: Window) -> Filter {
+    let mut filter = plane_filter(planes);
+
+    if let Some(until_ms) = window.until_ms {
+        filter = filter.until(Timestamp::from_secs(until_ms / 1000));
+    }
+
+    if let Some(since_ms) = window.since_ms {
+        filter = filter.since(Timestamp::from_secs(since_ms / 1000));
+    }
+
+    filter.limit(LIVE_REPLAY)
+}
+
+/// The window a community's standing subscription opens with.
+pub fn live_window(state: &CommunityState) -> Window {
+    let floor = state
+        .channels
+        .iter()
+        .filter_map(|channel| state.cursors.get(&channel.id)?.newest_ms)
+        .min();
+
+    match floor {
+        Some(floor) => Window {
+            since_ms: Some(floor.saturating_sub(CURSOR_OVERLAP_MS)),
+            until_ms: None,
+        },
+        None => Window::default(),
+    }
+}
+
 /// The subscription id carrying a community's planes.
 pub fn subscription_id(id: &CommunityId) -> SubscriptionId {
     SubscriptionId::new(id.to_hex())
@@ -429,14 +466,20 @@ pub async fn fold(client: &Client, state: &CommunityState) -> Result<Option<Snap
         return Ok(None);
     }
 
-    let wraps = client
-        .database()
-        .query(subscription_filter(&planes))
-        .await?;
+    let wraps = client.database().query(plane_filter(&planes)).await?;
 
     let mut editions = Vec::new();
     let mut observed: BTreeMap<PublicKey, u64> = BTreeMap::new();
     let mut guestbook_rumors = Vec::new();
+    let mut cached: BTreeMap<ChannelId, BTreeMap<EventId, Observed>> = BTreeMap::new();
+
+    for plane in &planes {
+        if let PlaneKind::Channel(channel, _) = plane.kind
+            && !cached.contains_key(&channel)
+        {
+            cached.insert(channel, cache::wrapper_index(client, &channel).await?);
+        }
+    }
 
     for wrap in &wraps {
         let Some(plane) = planes.iter().find(|plane| plane.address == wrap.pubkey) else {
@@ -457,6 +500,11 @@ pub async fn fold(client: &Client, state: &CommunityState) -> Result<Option<Snap
                 }
             }
             PlaneKind::Channel(channel, epoch) => {
+                if let Some(row) = cached.get(&channel).and_then(|index| index.get(&wrap.id)) {
+                    observe(&mut observed, row.author, row.at_ms);
+                    continue;
+                }
+
                 if let Ok((opened, rumor)) =
                     concord::cord03::open(wrap, &plane.group, &channel, epoch)
                 {
@@ -496,6 +544,7 @@ pub async fn fold(client: &Client, state: &CommunityState) -> Result<Option<Snap
 
     let now_ms = Timestamp::now().as_secs().saturating_mul(1000);
     let coalesced = cord02::guestbook::coalesce(&guestbook_rumors, now_ms, None, can_kick);
+
     let mut members = cord02::guestbook::complete_memberlist(
         &coalesced,
         &observed,
@@ -538,10 +587,11 @@ mod tests {
     }
 
     #[test]
-    fn planes_address_the_control_guestbook_and_only_public_channels() {
+    fn planes_address_the_control_guestbook_and_every_readable_channel() {
         let owner = Keys::generate().public_key();
         let control_pk = Keys::generate().public_key();
         let general = ChannelId::from_bytes([0x9c; 32]);
+        let staff = ChannelId::from_bytes([0x9d; 32]);
 
         let state = CommunityState {
             id: CommunityId::from_bytes([0x42; 32]),
@@ -561,12 +611,19 @@ mod tests {
                     key: None,
                 },
                 concord::state::ChannelKeyRef {
-                    id: ChannelId::from_bytes([0x9d; 32]),
+                    id: staff,
                     name: "staff".to_owned(),
                     private: true,
                     epoch: Epoch(0),
                     key: Some([0x04; 32]),
                 },
+                concord::state::ChannelKeyRef {
+                    id: ChannelId::from_bytes([0x9e; 32]),
+                    name: "locked".to_owned(),
+                    private: true,
+                    epoch: Epoch(0),
+                    key: None,
+                },
             ],
             relays: vec![RelayUrl::parse("wss://relay.example").expect("a url")],
             heads: Vec::new(),
@@ -578,7 +635,7 @@ mod tests {
 
         let planes = planes(&state).expect("planes");
 
-        assert_eq!(planes.len(), 3);
+        assert_eq!(planes.len(), 4);
         assert!(planes.iter().any(|plane| plane.address == control_pk));
         assert!(
             planes
@@ -590,11 +647,53 @@ mod tests {
                 .iter()
                 .any(|plane| matches!(plane.kind, PlaneKind::Channel(id, _) if id == general))
         );
+        assert!(
+            planes
+                .iter()
+                .any(|plane| matches!(plane.kind, PlaneKind::Channel(id, _) if id == staff)),
+            "a private channel whose key is held is subscribed"
+        );
 
-        let filter = subscription_filter(&planes);
+        let filter = plane_filter(&planes);
         let addresses: BTreeSet<PublicKey> = planes.iter().map(|plane| plane.address).collect();
         assert_eq!(filter.authors, Some(addresses));
-        assert_eq!(filter.kinds, Some(BTreeSet::from([Kind::from(KIND_WRAP)])));
+        assert_eq!(
+            filter.kinds,
+            Some(BTreeSet::from([
+                Kind::GiftWrap,
+                Kind::Custom(KIND_WRAP_EPHEMERAL)
+            ])),
+            "the standing subscription asks for both wrap kinds"
+        );
+    }
+
+    /// A cold subscription asks wide; a warm one resumes at the oldest held
+    /// cursor, minus the overlap, so no channel's new region is skipped.
+    #[test]
+    fn the_live_window_is_wide_cold_and_resumes_at_the_oldest_cursor_warm() {
+        let mut state = held(
+            CommunityId::from_bytes([0x42; 32]),
+            Keys::generate().public_key(),
+        );
+        let channel = state.channels[0].id;
+
+        assert_eq!(live_window(&state), Window::default());
+
+        state.cursors.insert(
+            channel,
+            concord::state::ChannelCursor {
+                newest_ms: Some(2_000_000),
+                oldest_ms: Some(1_000),
+                exhausted: false,
+            },
+        );
+        assert_eq!(
+            live_window(&state),
+            Window {
+                since_ms: Some(2_000_000 - CURSOR_OVERLAP_MS),
+                until_ms: None,
+            }
+        );
     }
 
     fn metadata(name: &str) -> cord02::CommunityMetadata {
@@ -661,16 +760,16 @@ mod tests {
                 .expect("loads");
             assert_eq!(loaded, vec![created.clone()]);
 
-            // The subscription filter must address the genesis wraps, or the registry
-            // would listen to a plane nothing is ever published on.
+            // The plane filter must address the genesis wraps, or a fold would
+            // read a plane nothing is ever published on.
             let planes = planes(&created).expect("planes");
             let wraps = client
                 .database()
-                .query(subscription_filter(&planes))
+                .query(plane_filter(&planes))
                 .await
                 .expect("queries");
             assert_eq!(wraps.len(), created.heads.len());
-            assert!(wraps.iter().all(|wrap| wrap.kind == Kind::from(KIND_WRAP)));
+            assert!(wraps.iter().all(|wrap| wrap.kind == Kind::GiftWrap));
 
             let snapshot = fold(&client, &created)
                 .await