Start handling P2P messages

This defines the tart of a very complex series of locks I'm really unhappy
with. At the same time, there's not immediately a better solution. This also
should work without issue.

coordinator/src/main.rs

@@ -52,16 +52,19 @@ async fn run<D: Db, Pro: Processor, P: P2p>(
   serai: Serai,
 ) {
   let add_new_tributary = |db, spec: TributarySpec| async {
+    // Save it to the database
     MainDb(db).add_active_tributary(&spec);
+    // Add it to the queue
+    // If we reboot before this is read from the queue, the fact it was saved to the database
+    // means it'll be handled on reboot
     NEW_TRIBUTARIES.write().await.push_back(spec);
   };
 
+  // Handle new Substrate blocks
   {
     let mut substrate_db = substrate::SubstrateDb::new(raw_db.clone());
     let mut last_substrate_block = substrate_db.last_block();
 
-    let p2p = p2p.clone();
-
     let key = key.clone();
     let mut processor = processor.clone();
     tokio::spawn(async move {
@@ -70,7 +73,6 @@ async fn run<D: Db, Pro: Processor, P: P2p>(
           &mut substrate_db,
           &key,
           add_new_tributary,
-          &p2p,
           &mut processor,
           &serai,
           &mut last_substrate_block,
@@ -87,15 +89,14 @@ async fn run<D: Db, Pro: Processor, P: P2p>(
     });
   }
 
+  // Handle the Tributaries
   {
     struct ActiveTributary<D: Db, P: P2p> {
       spec: TributarySpec,
       tributary: Tributary<D, Transaction, P>,
     }
+    let tributaries = Arc::new(RwLock::new(HashMap::<[u8; 32], ActiveTributary<D, P>>::new()));
 
-    let mut tributaries = HashMap::<[u8; 32], ActiveTributary<D, P>>::new();
-
-    // TODO: Use a db on a distinct volume
     async fn add_tributary<D: Db, P: P2p>(
       db: D,
       key: Zeroizing<<Ristretto as Ciphersuite>::F>,
@@ -104,6 +105,7 @@ async fn run<D: Db, Pro: Processor, P: P2p>(
       spec: TributarySpec,
     ) {
       let tributary = Tributary::<_, Transaction, _>::new(
+        // TODO: Use a db on a distinct volume
         db,
         spec.genesis(),
         spec.start_time(),
@@ -117,40 +119,85 @@ async fn run<D: Db, Pro: Processor, P: P2p>(
       tributaries.insert(tributary.genesis(), ActiveTributary { spec, tributary });
     }
 
+    // Reload active tributaries from the database
     // TODO: Can MainDb take a borrow?
     for spec in MainDb(raw_db.clone()).active_tributaries().1 {
-      add_tributary(raw_db.clone(), key.clone(), p2p.clone(), &mut tributaries, spec).await;
+      add_tributary(
+        raw_db.clone(),
+        key.clone(),
+        p2p.clone(),
+        &mut *tributaries.write().await,
+        spec,
+      )
+      .await;
     }
 
+    // Handle new Tributary blocks
     let mut tributary_db = tributary::TributaryDb::new(raw_db.clone());
-    tokio::spawn(async move {
-      loop {
-        // The following handle_new_blocks function may take an arbitrary amount of time
-        // If registering a new tributary waited for a lock on the tributaries table, the substrate
-        // scanner may wait on a lock for an arbitrary amount of time
-        // By instead using the distinct NEW_TRIBUTARIES, there should be minimal
-        // competition/blocking
-        {
-          let mut new_tributaries = NEW_TRIBUTARIES.write().await;
-          while let Some(spec) = new_tributaries.pop_front() {
-            add_tributary(raw_db.clone(), key.clone(), p2p.clone(), &mut tributaries, spec).await;
+    {
+      let tributaries = tributaries.clone();
+      let p2p = p2p.clone();
+      tokio::spawn(async move {
+        loop {
+          // The following handle_new_blocks function may take an arbitrary amount of time
+          // If registering a new tributary waited for a lock on the tributaries table, the
+          // substrate scanner may wait on a lock for an arbitrary amount of time
+          // By instead using the distinct NEW_TRIBUTARIES, there should be minimal
+          // competition/blocking
+          {
+            let mut new_tributaries = NEW_TRIBUTARIES.write().await;
+            while let Some(spec) = new_tributaries.pop_front() {
+              add_tributary(
+                raw_db.clone(),
+                key.clone(),
+                p2p.clone(),
+                // This is a short-lived write acquisition, which is why it should be fine
+                &mut *tributaries.write().await,
+                spec,
+              )
+              .await;
+            }
+          }
+
+          // Unknown-length read acquisition. This would risk screwing over the P2P process EXCEPT
+          // they both use read locks. Accordingly, they can co-exist
+          for ActiveTributary { spec, tributary } in tributaries.read().await.values() {
+            tributary::scanner::handle_new_blocks::<_, _, P>(
+              &mut tributary_db,
+              &key,
+              &mut processor,
+              spec,
+              tributary,
+            )
+            .await;
+          }
+
+          sleep(Duration::from_secs(3)).await;
+        }
+      });
+    }
+
+    // Handle P2P messages
+    {
+      tokio::spawn(async move {
+        loop {
+          let msg = p2p.receive().await;
+          match msg.kind {
+            P2pMessageKind::Tributary(genesis) => {
+              let tributaries_read = tributaries.read().await;
+              let Some(tributary) = tributaries_read.get(&genesis) else {
+                log::debug!("received p2p message for unknown network");
+                continue;
+              };
+
+              if tributary.tributary.handle_message(&msg.msg).await {
+                P2p::broadcast(&p2p, msg.kind, msg.msg).await;
+              }
+            }
           }
         }
-
-        for ActiveTributary { spec, tributary } in tributaries.values() {
-          tributary::scanner::handle_new_blocks::<_, _, P>(
-            &mut tributary_db,
-            &key,
-            &mut processor,
-            spec,
-            tributary,
-          )
-          .await;
-        }
-
-        sleep(Duration::from_secs(3)).await;
-      }
-    });
+      });
+    }
   }
 
   loop {