Add a dedicated db crate with a basic DB trait

It's needed by the processor and tributary (coordinator).
This commit is contained in:
Luke Parker
2023-04-14 11:41:01 -04:00
parent 04e7863dbd
commit 6f6c9f7cdf
20 changed files with 200 additions and 181 deletions

View File

@@ -1,62 +1,9 @@
use core::{marker::PhantomData, fmt::Debug};
use std::{
sync::{Arc, RwLock},
collections::HashMap,
};
use core::marker::PhantomData;
pub use serai_db::*;
use crate::{Plan, coins::Coin};
pub trait DbTxn: Send + Sync + Clone + Debug {
fn put(&mut self, key: impl AsRef<[u8]>, value: impl AsRef<[u8]>);
fn get(&self, key: impl AsRef<[u8]>) -> Option<Vec<u8>>;
fn del(&mut self, key: impl AsRef<[u8]>);
fn commit(self);
}
pub trait Db: 'static + Send + Sync + Clone + Debug {
type Transaction: DbTxn;
fn key(db_dst: &'static [u8], item_dst: &'static [u8], key: impl AsRef<[u8]>) -> Vec<u8> {
let db_len = u8::try_from(db_dst.len()).unwrap();
let dst_len = u8::try_from(item_dst.len()).unwrap();
[[db_len].as_ref(), db_dst, [dst_len].as_ref(), item_dst, key.as_ref()].concat().to_vec()
}
fn txn(&mut self) -> Self::Transaction;
fn get(&self, key: impl AsRef<[u8]>) -> Option<Vec<u8>>;
}
// TODO: Replace this with RocksDB
#[derive(Clone, Debug)]
pub struct MemDb(Arc<RwLock<HashMap<Vec<u8>, Vec<u8>>>>);
impl MemDb {
#[allow(clippy::new_without_default)]
pub fn new() -> MemDb {
MemDb(Arc::new(RwLock::new(HashMap::new())))
}
}
impl DbTxn for MemDb {
fn put(&mut self, key: impl AsRef<[u8]>, value: impl AsRef<[u8]>) {
self.0.write().unwrap().insert(key.as_ref().to_vec(), value.as_ref().to_vec());
}
fn get(&self, key: impl AsRef<[u8]>) -> Option<Vec<u8>> {
self.0.read().unwrap().get(key.as_ref()).cloned()
}
fn del(&mut self, key: impl AsRef<[u8]>) {
self.0.write().unwrap().remove(key.as_ref());
}
fn commit(self) {}
}
impl Db for MemDb {
type Transaction = MemDb;
fn txn(&mut self) -> MemDb {
Self(self.0.clone())
}
fn get(&self, key: impl AsRef<[u8]>) -> Option<Vec<u8>> {
self.0.read().unwrap().get(key.as_ref()).cloned()
}
}
#[derive(Debug)]
pub struct MainDb<C: Coin, D: Db>(D, PhantomData<C>);
impl<C: Coin, D: Db> MainDb<C, D> {

View File

@@ -18,7 +18,7 @@ use log::info;
use serai_client::{primitives::BlockHash, validator_sets::primitives::ValidatorSet};
use messages::key_gen::*;
use crate::{DbTxn, Db, coins::Coin};
use crate::{Get, DbTxn, Db, coins::Coin};
#[derive(Debug)]
pub enum KeyGenEvent<C: Ciphersuite> {
@@ -40,12 +40,7 @@ impl<C: Coin, D: Db> KeyGenDb<C, D> {
fn params_key(set: &ValidatorSet) -> Vec<u8> {
Self::key_gen_key(b"params", bincode::serialize(set).unwrap())
}
fn save_params(
&mut self,
txn: &mut D::Transaction,
set: &ValidatorSet,
params: &ThresholdParams,
) {
fn save_params(txn: &mut D::Transaction<'_>, set: &ValidatorSet, params: &ThresholdParams) {
txn.put(Self::params_key(set), bincode::serialize(params).unwrap());
}
fn params(&self, set: &ValidatorSet) -> ThresholdParams {
@@ -60,8 +55,7 @@ impl<C: Coin, D: Db> KeyGenDb<C, D> {
Self::key_gen_key(b"commitments", bincode::serialize(id).unwrap())
}
fn save_commitments(
&mut self,
txn: &mut D::Transaction,
txn: &mut D::Transaction<'_>,
id: &KeyGenId,
commitments: &HashMap<Participant, Vec<u8>>,
) {
@@ -78,8 +72,7 @@ impl<C: Coin, D: Db> KeyGenDb<C, D> {
Self::key_gen_key(b"generated_keys", bincode::serialize(id).unwrap())
}
fn save_keys(
&mut self,
txn: &mut D::Transaction,
txn: &mut D::Transaction<'_>,
id: &KeyGenId,
substrate_keys: &ThresholdCore<Ristretto>,
coin_keys: &ThresholdCore<C::Curve>,
@@ -93,11 +86,11 @@ impl<C: Coin, D: Db> KeyGenDb<C, D> {
Self::key_gen_key(b"keys", key.to_bytes())
}
#[allow(clippy::type_complexity)]
fn read_keys(
&self,
fn read_keys<G: Get>(
getter: &G,
key: &[u8],
) -> (Vec<u8>, (ThresholdKeys<Ristretto>, ThresholdKeys<C::Curve>)) {
let keys_vec = self.0.get(key).unwrap();
let keys_vec = getter.get(key).unwrap();
let mut keys_ref: &[u8] = keys_vec.as_ref();
let substrate_keys = ThresholdKeys::new(ThresholdCore::read(&mut keys_ref).unwrap());
let mut coin_keys = ThresholdKeys::new(ThresholdCore::read(&mut keys_ref).unwrap());
@@ -105,11 +98,10 @@ impl<C: Coin, D: Db> KeyGenDb<C, D> {
(keys_vec, (substrate_keys, coin_keys))
}
fn confirm_keys(
&mut self,
txn: &mut D::Transaction,
txn: &mut D::Transaction<'_>,
id: &KeyGenId,
) -> (ThresholdKeys<Ristretto>, ThresholdKeys<C::Curve>) {
let (keys_vec, keys) = self.read_keys(&Self::generated_keys_key(id));
let (keys_vec, keys) = Self::read_keys(txn, &Self::generated_keys_key(id));
txn.put(Self::keys_key(&keys.1.group_key()), keys_vec);
keys
}
@@ -117,7 +109,7 @@ impl<C: Coin, D: Db> KeyGenDb<C, D> {
&self,
key: &<C::Curve as Ciphersuite>::G,
) -> (ThresholdKeys<Ristretto>, ThresholdKeys<C::Curve>) {
self.read_keys(&Self::keys_key(key)).1
Self::read_keys(&self.0, &Self::keys_key(key)).1
}
}
@@ -191,7 +183,7 @@ impl<C: Coin, D: Db> KeyGen<C, D> {
// This may overwrite previously written params if we rebooted, yet that isn't a
// concern
let mut txn = self.db.0.txn();
self.db.save_params(&mut txn, &id.set, &params);
KeyGenDb::<C, D>::save_params(&mut txn, &id.set, &params);
txn.commit();
}
@@ -268,7 +260,7 @@ impl<C: Coin, D: Db> KeyGen<C, D> {
}
let mut txn = self.db.0.txn();
self.db.save_commitments(&mut txn, &id, &commitments);
KeyGenDb::<C, D>::save_commitments(&mut txn, &id, &commitments);
txn.commit();
KeyGenEvent::ProcessorMessage(ProcessorMessage::Shares { id, shares })
@@ -349,7 +341,7 @@ impl<C: Coin, D: Db> KeyGen<C, D> {
let coin_keys = handle_machine(&mut rng, params, machines.1, &mut shares_ref);
let mut txn = self.db.0.txn();
self.db.save_keys(&mut txn, &id, &substrate_keys, &coin_keys);
KeyGenDb::<C, D>::save_keys(&mut txn, &id, &substrate_keys, &coin_keys);
txn.commit();
let mut coin_keys = ThresholdKeys::new(coin_keys);
@@ -363,7 +355,7 @@ impl<C: Coin, D: Db> KeyGen<C, D> {
CoordinatorMessage::ConfirmKeyPair { context, id } => {
let mut txn = self.db.0.txn();
let (substrate_keys, coin_keys) = self.db.confirm_keys(&mut txn, &id);
let (substrate_keys, coin_keys) = KeyGenDb::<C, D>::confirm_keys(&mut txn, &id);
txn.commit();
info!(

View File

@@ -15,7 +15,7 @@ use tokio::{
};
use crate::{
DbTxn, Db,
Get, DbTxn, Db,
coins::{Output, Transaction, EventualitiesTracker, Block, Coin},
};
@@ -48,17 +48,12 @@ impl<C: Coin, D: Db> ScannerDb<C, D> {
fn block_number_key(id: &<C::Block as Block<C>>::Id) -> Vec<u8> {
Self::scanner_key(b"block_number", id)
}
fn save_block(
&mut self,
txn: &mut D::Transaction,
number: usize,
id: &<C::Block as Block<C>>::Id,
) {
fn save_block(txn: &mut D::Transaction<'_>, number: usize, id: &<C::Block as Block<C>>::Id) {
txn.put(Self::block_number_key(id), u64::try_from(number).unwrap().to_le_bytes());
txn.put(Self::block_key(number), id);
}
fn block(&self, number: usize) -> Option<<C::Block as Block<C>>::Id> {
self.0.get(Self::block_key(number)).map(|id| {
fn block<G: Get>(getter: &G, number: usize) -> Option<<C::Block as Block<C>>::Id> {
getter.get(Self::block_key(number)).map(|id| {
let mut res = <C::Block as Block<C>>::Id::default();
res.as_mut().copy_from_slice(&id);
res
@@ -74,8 +69,8 @@ impl<C: Coin, D: Db> ScannerDb<C, D> {
fn active_keys_key() -> Vec<u8> {
Self::scanner_key(b"active_keys", b"")
}
fn add_active_key(&mut self, txn: &mut D::Transaction, key: <C::Curve as Ciphersuite>::G) {
let mut keys = self.0.get(Self::active_keys_key()).unwrap_or(vec![]);
fn add_active_key(txn: &mut D::Transaction<'_>, key: <C::Curve as Ciphersuite>::G) {
let mut keys = txn.get(Self::active_keys_key()).unwrap_or(vec![]);
let key_bytes = key.to_bytes();
@@ -130,8 +125,7 @@ impl<C: Coin, D: Db> ScannerDb<C, D> {
Self::scanner_key(b"outputs", [key.to_bytes().as_ref(), block.as_ref()].concat())
}
fn save_outputs(
&mut self,
txn: &mut D::Transaction,
txn: &mut D::Transaction<'_>,
key: &<C::Curve as Ciphersuite>::G,
block: &<C::Block as Block<C>>::Id,
outputs: &[C::Output],
@@ -165,11 +159,11 @@ impl<C: Coin, D: Db> ScannerDb<C, D> {
next
}
fn outputs(
&self,
txn: &D::Transaction<'_>,
key: &<C::Curve as Ciphersuite>::G,
block: &<C::Block as Block<C>>::Id,
) -> Option<Vec<C::Output>> {
let bytes_vec = self.0.get(Self::outputs_key(key, block))?;
let bytes_vec = txn.get(Self::outputs_key(key, block))?;
let mut bytes: &[u8] = bytes_vec.as_ref();
let mut res = vec![];
@@ -183,13 +177,12 @@ impl<C: Coin, D: Db> ScannerDb<C, D> {
Self::scanner_key(b"scanned_block", key.to_bytes())
}
fn save_scanned_block(
&mut self,
txn: &mut D::Transaction,
txn: &mut D::Transaction<'_>,
key: &<C::Curve as Ciphersuite>::G,
block: usize,
) -> Vec<C::Output> {
let new_key = self.0.get(Self::scanned_block_key(key)).is_none();
let outputs = self.block(block).and_then(|id| self.outputs(key, &id));
let new_key = txn.get(Self::scanned_block_key(key)).is_none();
let outputs = Self::block(txn, block).and_then(|id| Self::outputs(txn, key, &id));
// Either this is a new key, with no outputs, or we're acknowledging this block
// If we're acknowledging it, we should have outputs available
assert_eq!(new_key, outputs.is_none());
@@ -278,8 +271,8 @@ impl<C: Coin, D: Db> ScannerHandle<C, D> {
info!("Rotating to key {}", hex::encode(key.to_bytes()));
let mut txn = scanner.db.0.txn();
assert!(scanner.db.save_scanned_block(&mut txn, &key, activation_number).is_empty());
scanner.db.add_active_key(&mut txn, key);
assert!(ScannerDb::<C, D>::save_scanned_block(&mut txn, &key, activation_number).is_empty());
ScannerDb::<C, D>::add_active_key(&mut txn, key);
txn.commit();
scanner.keys.push(key);
}
@@ -300,7 +293,7 @@ impl<C: Coin, D: Db> ScannerHandle<C, D> {
scanner.db.block_number(&id).expect("main loop trying to operate on data we haven't scanned");
let mut txn = scanner.db.0.txn();
let outputs = scanner.db.save_scanned_block(&mut txn, &key, number);
let outputs = ScannerDb::<C, D>::save_scanned_block(&mut txn, &key, number);
txn.commit();
for output in &outputs {
@@ -395,7 +388,7 @@ impl<C: Coin, D: Db> Scanner<C, D> {
};
let block_id = block.id();
if let Some(id) = scanner.db.block(i) {
if let Some(id) = ScannerDb::<C, D>::block(&scanner.db.0, i) {
// TODO2: Also check this block builds off the previous block
if id != block_id {
panic!("reorg'd from finalized {} to {}", hex::encode(id), hex::encode(block_id));
@@ -403,7 +396,7 @@ impl<C: Coin, D: Db> Scanner<C, D> {
} else {
info!("Found new block: {}", hex::encode(&block_id));
let mut txn = scanner.db.0.txn();
scanner.db.save_block(&mut txn, i, &block_id);
ScannerDb::<C, D>::save_block(&mut txn, i, &block_id);
txn.commit();
}
@@ -461,7 +454,7 @@ impl<C: Coin, D: Db> Scanner<C, D> {
// Save the outputs to disk
let mut txn = scanner.db.0.txn();
let batch = scanner.db.save_outputs(&mut txn, &key, &block_id, &outputs);
let batch = ScannerDb::<C, D>::save_outputs(&mut txn, &key, &block_id, &outputs);
txn.commit();
const TIME_TOLERANCE: u64 = 15;

View File

@@ -21,7 +21,7 @@ use tokio::{
use messages::sign::*;
use crate::{
DbTxn, Db,
Get, DbTxn, Db,
coins::{Transaction, Eventuality, Coin},
};
@@ -46,8 +46,7 @@ impl<C: Coin, D: Db> SignerDb<C, D> {
Self::sign_key(b"completed", id)
}
fn complete(
&mut self,
txn: &mut D::Transaction,
txn: &mut D::Transaction<'_>,
id: [u8; 32],
tx: &<C::Transaction as Transaction<C>>::Id,
) {
@@ -77,12 +76,7 @@ impl<C: Coin, D: Db> SignerDb<C, D> {
fn eventuality_key(id: [u8; 32]) -> Vec<u8> {
Self::sign_key(b"eventuality", id)
}
fn save_eventuality(
&mut self,
txn: &mut D::Transaction,
id: [u8; 32],
eventuality: C::Eventuality,
) {
fn save_eventuality(txn: &mut D::Transaction<'_>, id: [u8; 32], eventuality: C::Eventuality) {
txn.put(Self::eventuality_key(id), eventuality.serialize());
}
fn eventuality(&self, id: [u8; 32]) -> Option<C::Eventuality> {
@@ -94,14 +88,14 @@ impl<C: Coin, D: Db> SignerDb<C, D> {
fn attempt_key(id: &SignId) -> Vec<u8> {
Self::sign_key(b"attempt", bincode::serialize(id).unwrap())
}
fn attempt(&mut self, txn: &mut D::Transaction, id: &SignId) {
fn attempt(txn: &mut D::Transaction<'_>, id: &SignId) {
txn.put(Self::attempt_key(id), []);
}
fn has_attempt(&mut self, id: &SignId) -> bool {
self.0.get(Self::attempt_key(id)).is_some()
}
fn save_transaction(&mut self, txn: &mut D::Transaction, tx: &C::Transaction) {
fn save_transaction(txn: &mut D::Transaction<'_>, tx: &C::Transaction) {
txn.put(Self::sign_key(b"tx", tx.id()), tx.serialize());
}
}
@@ -231,8 +225,8 @@ impl<C: Coin, D: Db> Signer<C, D> {
// Stop trying to sign for this TX
let mut txn = self.db.0.txn();
self.db.save_transaction(&mut txn, &tx);
self.db.complete(&mut txn, id, tx_id);
SignerDb::<C, D>::save_transaction(&mut txn, &tx);
SignerDb::<C, D>::complete(&mut txn, id, tx_id);
txn.commit();
self.signable.remove(&id);
@@ -345,9 +339,9 @@ impl<C: Coin, D: Db> Signer<C, D> {
// Save the transaction in case it's needed for recovery
let mut txn = self.db.0.txn();
self.db.save_transaction(&mut txn, &tx);
SignerDb::<C, D>::save_transaction(&mut txn, &tx);
let tx_id = tx.id();
self.db.complete(&mut txn, id.id, &tx_id);
SignerDb::<C, D>::complete(&mut txn, id.id, &tx_id);
txn.commit();
// Publish it
@@ -481,7 +475,7 @@ impl<C: Coin, D: Db> Signer<C, D> {
}
let mut txn = signer.db.0.txn();
signer.db.attempt(&mut txn, &id);
SignerDb::<C, D>::attempt(&mut txn, &id);
txn.commit();
// Attempt to create the TX
@@ -552,7 +546,7 @@ impl<C: Coin, D: Db> SignerHandle<C, D> {
}
let mut txn = signer.db.0.txn();
signer.db.save_eventuality(&mut txn, id, eventuality);
SignerDb::<C, D>::save_eventuality(&mut txn, id, eventuality);
txn.commit();
signer.signable.insert(id, (start, tx));

View File

@@ -51,7 +51,7 @@ impl<D: Db> SubstrateSignerDb<D> {
fn completed_key(id: [u8; 32]) -> Vec<u8> {
Self::sign_key(b"completed", id)
}
fn complete(&mut self, txn: &mut D::Transaction, id: [u8; 32]) {
fn complete(txn: &mut D::Transaction<'_>, id: [u8; 32]) {
txn.put(Self::completed_key(id), [1]);
}
fn completed(&self, id: [u8; 32]) -> bool {
@@ -61,14 +61,14 @@ impl<D: Db> SubstrateSignerDb<D> {
fn attempt_key(id: &SignId) -> Vec<u8> {
Self::sign_key(b"attempt", bincode::serialize(id).unwrap())
}
fn attempt(&mut self, txn: &mut D::Transaction, id: &SignId) {
fn attempt(txn: &mut D::Transaction<'_>, id: &SignId) {
txn.put(Self::attempt_key(id), []);
}
fn has_attempt(&mut self, id: &SignId) -> bool {
self.0.get(Self::attempt_key(id)).is_some()
}
fn save_batch(&mut self, txn: &mut D::Transaction, batch: &SignedBatch) {
fn save_batch(txn: &mut D::Transaction<'_>, batch: &SignedBatch) {
txn.put(Self::sign_key(b"batch", batch.batch.block), batch.encode());
}
}
@@ -252,8 +252,8 @@ impl<D: Db> SubstrateSigner<D> {
// Save the batch in case it's needed for recovery
let mut txn = self.db.0.txn();
self.db.save_batch(&mut txn, &batch);
self.db.complete(&mut txn, id.id);
SubstrateSignerDb::<D>::save_batch(&mut txn, &batch);
SubstrateSignerDb::<D>::complete(&mut txn, id.id);
txn.commit();
// Stop trying to sign for this batch
@@ -267,7 +267,7 @@ impl<D: Db> SubstrateSigner<D> {
CoordinatorMessage::BatchSigned { key: _, block } => {
// Stop trying to sign for this batch
let mut txn = self.db.0.txn();
self.db.complete(&mut txn, block.0);
SubstrateSignerDb::<D>::complete(&mut txn, block.0);
txn.commit();
self.signable.remove(&block.0);
@@ -377,7 +377,7 @@ impl<D: Db> SubstrateSigner<D> {
}
let mut txn = signer.db.0.txn();
signer.db.attempt(&mut txn, &id);
SubstrateSignerDb::<D>::attempt(&mut txn, &id);
txn.commit();
// b"substrate" is a literal from sp-core

View File

@@ -7,11 +7,13 @@ use frost::{Participant, ThresholdKeys};
use tokio::time::timeout;
use serai_db::MemDb;
use crate::{
Plan, Db,
coins::{OutputType, Output, Block, Coin},
scanner::{ScannerEvent, Scanner, ScannerHandle},
tests::{util::db::MemDb, sign},
tests::sign,
};
async fn spend<C: Coin, D: Db>(

View File

@@ -7,6 +7,8 @@ use rand_core::{RngCore, OsRng};
use group::GroupEncoding;
use frost::{Participant, ThresholdParams, tests::clone_without};
use serai_db::MemDb;
use serai_client::{
primitives::{MONERO_NET_ID, BlockHash},
validator_sets::primitives::{Session, ValidatorSet},
@@ -16,7 +18,6 @@ use messages::{SubstrateContext, key_gen::*};
use crate::{
coins::Coin,
key_gen::{KeyGenEvent, KeyGen},
tests::util::db::MemDb,
};
const ID: KeyGenId =

View File

@@ -1,5 +1,3 @@
pub(crate) mod util;
mod key_gen;
pub(crate) use key_gen::test_key_gen;

View File

@@ -7,10 +7,11 @@ use frost::Participant;
use tokio::time::timeout;
use serai_db::MemDb;
use crate::{
coins::{OutputType, Output, Block, Coin},
scanner::{ScannerEvent, Scanner, ScannerHandle},
tests::util::db::MemDb,
};
pub async fn test_scanner<C: Coin>(coin: C) {

View File

@@ -13,12 +13,13 @@ use frost::{
use tokio::time::timeout;
use serai_db::MemDb;
use messages::sign::*;
use crate::{
Payment, Plan,
coins::{Output, Transaction, Coin},
signer::{SignerEvent, Signer},
tests::util::db::MemDb,
};
#[allow(clippy::type_complexity)]

View File

@@ -17,13 +17,12 @@ use tokio::time::timeout;
use scale::Encode;
use sp_application_crypto::{RuntimePublic, sr25519::Public};
use serai_db::MemDb;
use serai_client::{primitives::*, in_instructions::primitives::*};
use messages::{sign::SignId, coordinator::*};
use crate::{
substrate_signer::{SubstrateSignerEvent, SubstrateSigner},
tests::util::db::MemDb,
};
use crate::substrate_signer::{SubstrateSignerEvent, SubstrateSigner};
#[tokio::test]
async fn test_substrate_signer() {

View File

@@ -1,42 +0,0 @@
use std::{
sync::{Arc, RwLock},
collections::HashMap,
};
use crate::{DbTxn, Db};
#[derive(Clone, Debug)]
pub struct MemDb(Arc<RwLock<HashMap<Vec<u8>, Vec<u8>>>>);
impl MemDb {
pub(crate) fn new() -> MemDb {
MemDb(Arc::new(RwLock::new(HashMap::new())))
}
}
impl Default for MemDb {
fn default() -> MemDb {
MemDb::new()
}
}
impl DbTxn for MemDb {
fn put(&mut self, key: impl AsRef<[u8]>, value: impl AsRef<[u8]>) {
self.0.write().unwrap().insert(key.as_ref().to_vec(), value.as_ref().to_vec());
}
fn get(&self, key: impl AsRef<[u8]>) -> Option<Vec<u8>> {
self.0.read().unwrap().get(key.as_ref()).cloned()
}
fn del(&mut self, key: impl AsRef<[u8]>) {
self.0.write().unwrap().remove(key.as_ref());
}
fn commit(self) {}
}
impl Db for MemDb {
type Transaction = MemDb;
fn txn(&mut self) -> MemDb {
Self(self.0.clone())
}
fn get(&self, key: impl AsRef<[u8]>) -> Option<Vec<u8>> {
self.0.read().unwrap().get(key.as_ref()).cloned()
}
}

View File

@@ -1 +0,0 @@
pub(crate) mod db;

View File

@@ -6,12 +6,14 @@ use frost::{Participant, dkg::tests::key_gen};
use tokio::time::timeout;
use serai_db::MemDb;
use crate::{
Payment, Plan,
coins::{Output, Transaction, Block, Coin},
scanner::{ScannerEvent, Scanner},
scheduler::Scheduler,
tests::{util::db::MemDb, sign},
tests::sign,
};
// Tests the Scanner, Scheduler, and Signer together