From 84d9c14a17e864058527981e3388cef148827c11 Mon Sep 17 00:00:00 2001 From: Yigit Sever Date: Wed, 7 Apr 2021 04:33:45 +0300 Subject: Implement Block GET/PUT with new schema - `Arc`+`Mutex` is replaced by `parking_lot::RwLock,` decoupled Read+Write and ability to upgrade read locks into write locks if needed - Schema has changed, `Db` is now a struct that implements `new()` to return a new instance of itself, pros/cons listed in code but tl;dr blockchain and pending transactions are separate now - `custom_filters` now supports extracting Block json and Transaction json in separate functions too - /block GET and PUT implemented, `Blocks` currently have one check (transactions appear in pending transaction) - debug is working after something, dunno how I fixed it --- Cargo.lock | 63 +++++++++++++++++++++++++++++++++- Cargo.toml | 1 + TODO.md | 6 +++- src/custom_filters.rs | 11 ++++-- src/handlers.rs | 94 ++++++++++++++++++++++++++++++++++++++------------- src/main.rs | 6 ++-- src/routes.rs | 30 +++++++++++++--- src/schema.rs | 65 ++++++++++++++++++++++++----------- tester.sh | 38 +++++++++++++++++++++ 9 files changed, 258 insertions(+), 56 deletions(-) create mode 100644 tester.sh diff --git a/Cargo.lock b/Cargo.lock index c531d87..45ae0a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -107,6 +107,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + [[package]] name = "cpuid-bool" version = "0.1.2" @@ -273,6 +282,7 @@ version = "0.1.0" dependencies = [ "chrono", "log", + "parking_lot", "pretty_env_logger", "serde", "serde_json", @@ -474,6 +484,15 @@ version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41" +[[package]] +name = "lock_api" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.14" @@ -596,6 +615,30 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "parking_lot" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3" +dependencies = [ + "cfg-if 0.1.10", + "cloudabi", + "libc", + "redox_syscall 0.1.57", + "smallvec", + "winapi 0.3.9", +] + [[package]] name = "percent-encoding" version = "2.1.0" @@ -782,6 +825,12 @@ dependencies = [ "rand_core 0.6.2", ] +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + [[package]] name = "redox_syscall" version = "0.2.5" @@ -835,6 +884,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2" +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + [[package]] name = "serde" version = "1.0.125" @@ -906,6 +961,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +[[package]] +name = "smallvec" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" + [[package]] name = "socket2" version = "0.3.19" @@ -937,7 +998,7 @@ dependencies = [ "cfg-if 1.0.0", "libc", "rand 0.8.3", - "redox_syscall", + "redox_syscall 0.2.5", "remove_dir_all", "winapi 0.3.9", ] diff --git a/Cargo.toml b/Cargo.toml index 3f9c00d..a203a6f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ serde = { version = "1.0.104", features = ["derive"] } chrono = { version = "0.4.10", features = ["serde"] } log = "0.4.8" pretty_env_logger = "0.3.1" +parking_lot = "0.10.0" [dev-dependencies] serde_json = "1.0.44" diff --git a/TODO.md b/TODO.md index 76292d2..4e65094 100644 --- a/TODO.md +++ b/TODO.md @@ -1,4 +1,7 @@ # TODO +## Process +- [ ] we need our own representation of students and their grades, "there is no blockchain" + ## Proof-of-work - [ ] pick a block proposal scheme (= pick hash function) [list of hash functions](https://en.bitcoinwiki.org/wiki/List_of_hash_functions) - [ ] check the nonce for incoming blocks @@ -7,4 +10,5 @@ - [ ] pick a user authentication scheme - [ ] implement it -- [ ] Switch to RwLock (parking_lot) +## Done & Brag +- [x] Switch to RwLock (parking_lot) (done at 2021-04-07 03:43, two possible schemes to represent inner Db (ledger) in code) diff --git a/src/custom_filters.rs b/src/custom_filters.rs index 86a78d4..7caf71a 100644 --- a/src/custom_filters.rs +++ b/src/custom_filters.rs @@ -3,7 +3,7 @@ use std::convert::Infallible; use warp::{Filter, Rejection}; -use crate::schema::{Db, Transaction}; // `Block` coming later +use crate::schema::{Block, Db, Transaction}; // Database context for routes pub fn with_db(db: Db) -> impl Filter + Clone { @@ -15,7 +15,12 @@ pub fn with_db(db: Db) -> impl Filter + Clo // warp::query::() // } -// Accept only JSON body and reject big payloads -pub fn json_body() -> impl Filter + Clone { +// Accept only json encoded Transaction body and reject big payloads +pub fn transaction_json_body() -> impl Filter + Clone { + warp::body::content_length_limit(1024 * 32).and(warp::body::json()) +} + +// Accept only json encoded Transaction body and reject big payloads +pub fn block_json_body() -> impl Filter + Clone { warp::body::content_length_limit(1024 * 32).and(warp::body::json()) } diff --git a/src/handlers.rs b/src/handlers.rs index 51c7b63..ecf5a92 100644 --- a/src/handlers.rs +++ b/src/handlers.rs @@ -1,45 +1,93 @@ -// API handlers, the ends of each filter chain - -use log::debug; +/// API handlers, the ends of each filter chain +use log::debug; // this is more useful than debug! learn how to use this +use parking_lot::RwLockUpgradableReadGuard; use std::convert::Infallible; use warp::{http::StatusCode, reply}; -use crate::schema::{Db, Transaction}; // `Block` coming later +use crate::schema::{Block, Db, Transaction}; + +/// GET /transaction +/// Returns JSON array of transactions +/// Cannot fail +pub async fn list_transactions(db: Db) -> Result { + debug!("list all transactions"); + let mut result = Vec::new(); + + let transactions = db.pending_transactions.read(); + // let transactions = transactions.clone().into_iter().collect(); + + for (_, value) in transactions.iter() { + result.push(value) + } + + Ok(reply::with_status(reply::json(&result), StatusCode::OK)) +} + +/// GET /block +/// Returns JSON array of blocks +/// Cannot fail +/// Mostly around for debug purposes +pub async fn list_blocks(db: Db) -> Result { + debug!("list all blocks"); + + let mut result = Vec::new(); + let blocks = db.blockchain.read(); + + for block in blocks.iter() { + result.push(block); + } -// PROPOSE Transaction -// POST /transaction + Ok(reply::with_status(reply::json(&result), StatusCode::OK)) +} + +/// POST /transaction +/// Pushes a new transaction for pending transaction pool +/// Can reject the transaction proposal +/// TODO: when is a new transaction rejected <07-04-21, yigit> // pub async fn propose_transaction( new_transaction: Transaction, db: Db, ) -> Result { debug!("new transaction request {:?}", new_transaction); - let mut transactions = db.lock().await; + // let mut transactions = db.lock().await; + let mut transactions = db.pending_transactions.write(); - transactions.push(new_transaction); + transactions.insert(new_transaction.source.to_owned(), new_transaction); Ok(StatusCode::CREATED) } -// GET Transaction List -// GET /transaction -// Returns JSON array of transactions -// Cannot fail? -pub async fn list_transactions(db: Db) -> Result { - debug!("list all transactions"); +/// POST /block +/// Proposes a new block for the next round +/// Can reject the block +pub async fn propose_block(new_block: Block, db: Db) -> Result { + debug!("new block request {:?}", new_block); - let transactions = db.lock().await; + // https://blog.logrocket.com/create-an-async-crud-web-service-in-rust-with-warp/ (this has + // error.rs, error struct, looks very clean) - let transactions: Vec = transactions.clone().into_iter().collect(); + let pending_transactions = db.pending_transactions.upgradable_read(); + let blockchain = db.blockchain.upgradable_read(); - Ok(reply::with_status( - reply::json(&transactions), - StatusCode::OK, - )) -} + // TODO: check 1, new_block.transaction_list from pending_transactions pool? <07-04-21, yigit> // + for transaction_hash in new_block.transaction_list.iter() { + if !pending_transactions.contains_key(transaction_hash) { + return Ok(StatusCode::BAD_REQUEST); + } + } -// PROPOSE Block -// POST /block + // TODO: check 2, block hash (\w nonce) asserts $hash_condition? <07-04-21, yigit> // + // assume it is for now + + let mut blockchain = RwLockUpgradableReadGuard::upgrade(blockchain); + blockchain.push(new_block); + + let mut pending_transactions = RwLockUpgradableReadGuard::upgrade(pending_transactions); + pending_transactions.clear(); + + Ok(StatusCode::CREATED) +} // `GET /games` // Returns JSON array of todos diff --git a/src/main.rs b/src/main.rs index bcd4173..7ef2597 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,15 +11,15 @@ mod schema; async fn main() { // Show debug logs by default by setting `RUST_LOG=restful_rust=debug` if env::var_os("RUST_LOG").is_none() { - env::set_var("RUST_LOG", "restful_rust=debug"); + env::set_var("RUST_LOG", "gradecoin=debug"); } pretty_env_logger::init(); - let db = schema::ledger(); // 1. we need this to return a _simple_ db + let db = schema::create_database(); let api = routes::consensus_routes(db); - let routes = api.with(warp::log("restful_rust")); + let routes = api.with(warp::log("gradecoin")); // Start the server warp::serve(routes).run(([127, 0, 0, 1], 8080)).await; diff --git a/src/routes.rs b/src/routes.rs index fc4426a..9054fb6 100644 --- a/src/routes.rs +++ b/src/routes.rs @@ -4,12 +4,15 @@ use crate::custom_filters; use crate::handlers; use crate::schema::Db; -// Root, all routes combined +/// Root, all routes combined pub fn consensus_routes(db: Db) -> impl Filter + Clone { - transaction_list(db.clone()).or(transaction_propose(db.clone())) + transaction_list(db.clone()) + .or(transaction_propose(db.clone())) + .or(block_propose(db.clone())) + .or(block_list(db.clone())) } -// GET /transaction +/// GET /transaction pub fn transaction_list(db: Db) -> impl Filter + Clone { warp::path!("transaction") .and(warp::get()) @@ -17,15 +20,32 @@ pub fn transaction_list(db: Db) -> impl Filter impl Filter + Clone { + warp::path!("block") + .and(warp::get()) + .and(custom_filters::with_db(db)) + .and_then(handlers::list_blocks) +} + +/// POST /transaction pub fn transaction_propose(db: Db) -> impl Filter + Clone { warp::path!("transaction") .and(warp::post()) - .and(custom_filters::json_body()) + .and(custom_filters::transaction_json_body()) .and(custom_filters::with_db(db)) .and_then(handlers::propose_transaction) } +/// POST /block +pub fn block_propose(db: Db) -> impl Filter + Clone { + warp::path!("block") + .and(warp::post()) + .and(custom_filters::block_json_body()) + .and(custom_filters::with_db(db)) + .and_then(handlers::propose_block) +} + /////////////////////////// // below are not mine. // /////////////////////////// diff --git a/src/schema.rs b/src/schema.rs index ea36a70..57210a3 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -1,31 +1,46 @@ -// Common types used across API - -use chrono::{NaiveDate, NaiveDateTime}; +use chrono::NaiveDateTime; +use parking_lot::RwLock; use serde::{Deserialize, Serialize}; +use std::collections::HashMap; use std::sync::Arc; -use tokio::sync::Mutex; // use crate::validators; -pub fn ledger() -> Db { - // TODO: there was something simpler in one of the other tutorials? <07-04-21, yigit> // - - Arc::new(Mutex::new(vec![ - Transaction { - source: String::from("Myself"), - target: String::from("Nobody"), - amount: 4, - timestamp: NaiveDate::from_ymd(2021, 4, 7).and_hms(00, 17, 00), - }, - ])) +// In memory data structure + +// Two approaches here +// 1. Db is a type +// pub type Db = Arc>>; +// Ledger is a struct, we wrap the ledger with arc + mutex in ledger() +// to access transactions we need to unwrap blocks as well, vice versa +// +// 2. Db is a struct, attributes are wrapped +// we can offload ::new() to it's member method +// blocks and transactions are accessible separately, which is the biggest pro + +/// Creates a new database +pub fn create_database() -> Db { + Db::new() } +#[derive(Debug, Clone)] +pub struct Db { + // heh. also https://doc.rust-lang.org/std/collections/struct.LinkedList.html says Vec is generally faster + pub blockchain: Arc>>, + // every proposer can have _one_ pending transaction, a way to enforce this, String is proposer identifier + pub pending_transactions: Arc>>, +} -// For presentation purposes keep mocked data in in-memory structure -// In real life scenario connection with regular database would be established - -pub type Db = Arc>>; +impl Db { + fn new() -> Self { + Db { + blockchain: Arc::new(RwLock::new(Vec::new())), + pending_transactions: Arc::new(RwLock::new(HashMap::new())), + } + } +} +/// A transaction between `source` and `target` that moves `amount` #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct Transaction { pub source: String, @@ -34,14 +49,22 @@ pub struct Transaction { pub timestamp: NaiveDateTime, } +/// A block that was proposed with `transaction_list` and `nonce` that made `hash` valid #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct Block { - pub transaction_list: Vec, // [Transaction; N] + pub transaction_list: Vec, // hashes of the transactions (or just "source" for now) pub nonce: i32, pub timestamp: NaiveDateTime, pub hash: String, // future proof'd baby } +// pub struct Ledger { +// // heh. also https://doc.rust-lang.org/std/collections/struct.LinkedList.html says Vec is generally faster +// blockchain: Vec, +// // every proposer can have _one_ pending transaction, a way to enforce this, String is proposer identifier +// pending_transactions: HashMap, +// } + // #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] // #[serde(rename_all = "camelCase")] // pub struct Game { @@ -99,6 +122,8 @@ pub struct Block { // )) // } +// TODO: these tests are amazing, we should write some when schema is decided upon <07-04-21, yigit> // + // #[cfg(test)] // mod tests { // use super::*; diff --git a/tester.sh b/tester.sh new file mode 100644 index 0000000..a0396d6 --- /dev/null +++ b/tester.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +## When in doubt just write a shell script + +curl --request POST \ + --header 'Content-Type: application/json' \ + --data '{ + "source": "foo", + "target": "bar", + "amount": 5, + "timestamp": "2021-04-07T00:17:00" +}' \ + http://localhost:8080/transaction + +curl --request POST \ + --header 'Content-Type: application/json' \ + --data '{ + "source": "saz", + "target": "quux", + "amount": 12, + "timestamp": "2021-04-07T00:17:00" +}' \ + http://localhost:8080/transaction + +curl localhost:8080/transaction + +curl --header "Content-Type: application/json" \ + --request POST \ + --data '{ + "transaction_list": [ + "foo", + "saz" + ], + "nonce": 4, + "timestamp": "2021-04-07T04:17:00", + "hash": "aaaaaa" +}' \ + http://localhost:8080/block -- cgit v1.2.3-70-g09d2