Reenable alpha fees

Cargo.toml

@@ -61,6 +61,7 @@ pallet-subtensor-swap-runtime-api = { path = "pallets/swap/runtime-api", default
 pallet-subtensor-swap-rpc = { path = "pallets/swap/rpc", default-features = false }
 procedural-fork = { path = "support/procedural-fork", default-features = false }
 safe-math = { path = "primitives/safe-math", default-features = false }
+safe-bigmath = { rev = "013c499", package = "safe-bigmath", default-features = false, git = "https://github.com/sam0x17/safe-bigmath" }
 share-pool = { path = "primitives/share-pool", default-features = false }
 subtensor-macros = { path = "support/macros", default-features = false }
 subtensor-custom-rpc = { default-features = false, path = "pallets/subtensor/rpc" }
@@ -82,6 +83,7 @@ hex = { version = "0.4", default-features = false }
 hex-literal = "0.4.1"
 jsonrpsee = { version = "0.24.9", default-features = false }
 libsecp256k1 = { version = "0.7.2", default-features = false }
+lencode = "0.1.6"
 log = { version = "0.4.21", default-features = false }
 memmap2 = "0.9.8"
 ndarray = { version = "0.16.1", default-features = false }
@@ -306,6 +308,8 @@ pow-faucet = []
 
 [patch.crates-io]
 w3f-bls = { git = "https://github.com/opentensor/bls", branch = "fix-no-std" }
+zstd-sys = { path = "patches/zstd-sys" }
+zstd-safe = { path = "patches/zstd-safe" }
 
 # Patches automatically generated with `diener`:
 # `diener patch --target https://github.com/paritytech/polkadot-sdk --point-to-git https://github.com/opentensor/polkadot-sdk.git --point-to-git-commit 81fa2c54e94f824eba7dabe9dffd063481cb2d80 --crates-to-patch ../polkadot-sdk --ignore-unused`

common/src/lib.rs

@@ -257,7 +257,7 @@ pub trait BalanceOps<AccountId> {
         hotkey: &AccountId,
         netuid: NetUid,
         alpha: AlphaCurrency,
-    ) -> Result<AlphaCurrency, DispatchError>;
+    ) -> Result<(), DispatchError>;
 }
 
 pub mod time {

docs/rust-setup.md

@@ -67,7 +67,21 @@ Open the Terminal application and execute the following commands:
 
 # Make sure Homebrew is up-to-date, install protobuf and openssl
 brew update
-brew install protobuf openssl
+brew install protobuf openssl llvm@16
+```
+
+Also, add the following lines at the end of your ~/.zshrc:
+
+```
+# LLVM 16 from Homebrew
+export PATH="/opt/homebrew/opt/llvm@16/bin:$PATH"
+
+export CC="/opt/homebrew/opt/llvm@16/bin/clang"
+export CXX="/opt/homebrew/opt/llvm@16/bin/clang++"
+export LIBCLANG_PATH="/opt/homebrew/opt/llvm@16/lib/libclang.dylib"
+
+export LDFLAGS="-L/opt/homebrew/opt/llvm@16/lib"
+export CPPFLAGS="-I/opt/homebrew/opt/llvm@16/include"
 ```
 
 ### Windows

pallets/subtensor/src/coinbase/block_step.rs

@@ -33,6 +33,7 @@ impl<T: Config + pallet_drand::Config> Pallet<T> {
         Self::run_auto_claim_root_divs(last_block_hash);
         // --- 9. Populate root coldkey maps.
         Self::populate_root_coldkey_staking_maps();
+        Self::populate_root_coldkey_staking_maps_v2();
 
         // Return ok.
         Ok(())

pallets/subtensor/src/coinbase/run_coinbase.rs

@@ -516,15 +516,15 @@ impl<T: Config> Pallet<T> {
             log::debug!(
                 "owner_hotkey: {owner_hotkey:?} owner_coldkey: {owner_coldkey:?}, owner_cut: {owner_cut:?}"
             );
-            let real_owner_cut = Self::increase_stake_for_hotkey_and_coldkey_on_subnet(
+            Self::increase_stake_for_hotkey_and_coldkey_on_subnet(
                 &owner_hotkey,
                 &owner_coldkey,
                 netuid,
                 owner_cut,
             );
             // If the subnet is leased, notify the lease logic that owner cut has been distributed.
             if let Some(lease_id) = SubnetUidToLeaseId::<T>::get(netuid) {
-                Self::distribute_leased_network_dividends(lease_id, real_owner_cut);
+                Self::distribute_leased_network_dividends(lease_id, owner_cut);
             }
         }
 
@@ -618,7 +618,7 @@ impl<T: Config> Pallet<T> {
             root_alpha = root_alpha.saturating_sub(alpha_take);
             // Give the validator their take.
             log::debug!("hotkey: {hotkey:?} alpha_take: {alpha_take:?}");
-            let _validator_stake = Self::increase_stake_for_hotkey_and_coldkey_on_subnet(
+            Self::increase_stake_for_hotkey_and_coldkey_on_subnet(
                 &hotkey,
                 &Owner::<T>::get(hotkey.clone()),
                 netuid,

pallets/subtensor/src/lib.rs

@@ -88,6 +88,7 @@ pub mod pallet {
     use frame_system::pallet_prelude::*;
     use pallet_drand::types::RoundNumber;
     use runtime_common::prod_or_fast;
+    use share_pool::SafeFloatSerializable;
     use sp_core::{ConstU32, H160, H256};
     use sp_runtime::traits::{Dispatchable, TrailingZeroInput};
     use sp_std::collections::btree_map::BTreeMap;
@@ -1437,11 +1438,41 @@ pub mod pallet {
         ValueQuery,
     >;
 
+    /// DMAP ( hot, netuid ) --> total_alpha_shares | Returns the number of alpha shares for a hotkey on a subnet, stores bigmath vector.
+    #[pallet::storage]
+    pub type TotalHotkeySharesV2<T: Config> = StorageDoubleMap<
+        _,
+        Blake2_128Concat,
+        T::AccountId, // hot
+        Identity,
+        NetUid,                // subnet
+        SafeFloatSerializable, // Hotkey shares in unlimited precision
+        ValueQuery,
+    >;
+
+    /// --- NMAP ( hot, cold, netuid ) --> alpha | Returns the alpha shares for a hotkey, coldkey, netuid triplet, stores bigmath vector.
+    #[pallet::storage]
+    pub type AlphaV2<T: Config> = StorageNMap<
+        _,
+        (
+            NMapKey<Blake2_128Concat, T::AccountId>, // hot
+            NMapKey<Blake2_128Concat, T::AccountId>, // cold
+            NMapKey<Identity, NetUid>,               // subnet
+        ),
+        SafeFloatSerializable, // Shares in unlimited precision
+        ValueQuery,
+    >;
+
     /// Contains last Alpha storage map key to iterate (check first)
     #[pallet::storage]
     pub type AlphaMapLastKey<T: Config> =
         StorageValue<_, Option<Vec<u8>>, ValueQuery, DefaultAlphaIterationLastKey<T>>;
 
+    /// Contains last AlphaV2 storage map key to iterate (check first)
+    #[pallet::storage]
+    pub type AlphaV2MapLastKey<T: Config> =
+        StorageValue<_, Option<Vec<u8>>, ValueQuery, DefaultAlphaIterationLastKey<T>>;
+
     /// --- MAP ( netuid ) --> token_symbol | Returns the token symbol for a subnet.
     #[pallet::storage]
     pub type TokenSymbol<T: Config> =
@@ -2635,20 +2666,25 @@ impl<T: Config + pallet_balances::Config<Balance = u64>>
         hotkey: &T::AccountId,
         netuid: NetUid,
         alpha: AlphaCurrency,
-    ) -> Result<AlphaCurrency, DispatchError> {
+    ) -> Result<(), DispatchError> {
         ensure!(
             Self::hotkey_account_exists(hotkey),
             Error::<T>::HotKeyAccountNotExists
         );
 
+        ensure!(
+            Self::get_stake_for_hotkey_and_coldkey_on_subnet(hotkey, coldkey, netuid) >= alpha,
+            Error::<T>::InsufficientBalance
+        );
+
         // Decrese alpha out counter
         SubnetAlphaOut::<T>::mutate(netuid, |total| {
             *total = total.saturating_sub(alpha);
         });
 
-        Ok(Self::decrease_stake_for_hotkey_and_coldkey_on_subnet(
-            hotkey, coldkey, netuid, alpha,
-        ))
+        Self::decrease_stake_for_hotkey_and_coldkey_on_subnet(hotkey, coldkey, netuid, alpha);
+
+        Ok(())
     }
 }
 

pallets/subtensor/src/macros/errors.rs

@@ -266,7 +266,5 @@ mod errors {
         InvalidRootClaimThreshold,
         /// Exceeded subnet limit number or zero.
         InvalidSubnetNumber,
-        /// Unintended precision loss when unstaking alpha
-        PrecisionLoss,
     }
 }

pallets/subtensor/src/macros/genesis.rs

@@ -4,6 +4,7 @@ use frame_support::pallet_macros::pallet_section;
 /// This can later be imported into the pallet using [`import_section`].
 #[pallet_section]
 mod genesis {
+    use share_pool::SafeFloat;
     use sp_core::crypto::Pair;
     use sp_core::sr25519::Pair as Sr25519Pair;
 
@@ -91,20 +92,20 @@ mod genesis {
             SubnetOwner::<T>::insert(netuid, hotkey.clone());
             SubnetLocked::<T>::insert(netuid, TaoCurrency::from(1));
             LargestLocked::<T>::insert(netuid, 1);
-            Alpha::<T>::insert(
+            AlphaV2::<T>::insert(
                 // Lock the initial funds making this key the owner.
                 (hotkey.clone(), hotkey.clone(), netuid),
-                U64F64::saturating_from_num(1_000_000_000),
+                SafeFloatSerializable::from(&SafeFloat::from(1_000_000_000)),
             );
             TotalHotkeyAlpha::<T>::insert(
                 hotkey.clone(),
                 netuid,
                 AlphaCurrency::from(1_000_000_000),
             );
-            TotalHotkeyShares::<T>::insert(
+            TotalHotkeySharesV2::<T>::insert(
                 hotkey.clone(),
                 netuid,
-                U64F64::saturating_from_num(1_000_000_000),
+                SafeFloatSerializable::from(&SafeFloat::from(1_000_000_000)),
             );
             // TotalColdkeyAlpha::<T>::insert(hotkey.clone(), netuid, 1_000_000_000);
             SubnetAlphaOut::<T>::insert(netuid, AlphaCurrency::from(1_000_000_000));

pallets/subtensor/src/rpc_info/delegate_info.rs

@@ -65,8 +65,8 @@ impl<T: Config> Pallet<T> {
                 alpha_share_pools.push(alpha_share_pool);
             }
 
-            for ((nominator, netuid), alpha_stake) in Alpha::<T>::iter_prefix((delegate.clone(),)) {
-                if alpha_stake == 0 {
+            for (nominator, netuid, alpha_stake) in Self::alpha_iter_single_prefix(&delegate) {
+                if alpha_stake.is_zero() {
                     continue;
                 }
 
@@ -166,7 +166,7 @@ impl<T: Config> Pallet<T> {
         )> = Vec::new();
         for delegate in <Delegates<T> as IterableStorageMap<T::AccountId, u16>>::iter_keys() {
             // Staked to this delegate, so add to list
-            for (netuid, _) in Alpha::<T>::iter_prefix((delegate.clone(), delegatee.clone())) {
+            for (netuid, _) in Self::alpha_iter_prefix((&delegate, &delegatee)) {
                 let delegate_info = Self::get_delegate_by_existing_account(delegate.clone(), true);
                 delegates.push((
                     delegate_info,

pallets/subtensor/src/staking/helpers.rs

@@ -6,6 +6,7 @@ use frame_support::traits::{
     },
 };
 use safe_math::*;
+use share_pool::SafeFloat;
 use substrate_fixed::types::U96F32;
 use subtensor_runtime_common::{NetUid, TaoCurrency};
 use subtensor_swap_interface::{Order, SwapHandler};
@@ -68,7 +69,7 @@ impl<T: Config> Pallet<T> {
         hotkeys
             .iter()
             .map(|hotkey| {
-                Alpha::<T>::iter_prefix((hotkey, coldkey))
+                Self::alpha_iter_prefix((hotkey, coldkey))
                     .map(|(netuid, _)| {
                         let alpha_stake = Self::get_stake_for_hotkey_and_coldkey_on_subnet(
                             hotkey, coldkey, netuid,
@@ -101,7 +102,7 @@ impl<T: Config> Pallet<T> {
         hotkeys
             .iter()
             .map(|hotkey| {
-                Alpha::<T>::iter_prefix((hotkey, coldkey))
+                Self::alpha_iter_prefix((hotkey, coldkey))
                     .map(|(netuid_on_storage, _)| {
                         if netuid_on_storage == netuid {
                             let alpha_stake = Self::get_stake_for_hotkey_and_coldkey_on_subnet(
@@ -261,7 +262,7 @@ impl<T: Config> Pallet<T> {
     /// used with caution.
     pub fn clear_small_nominations() {
         // Loop through all staking accounts to identify and clear nominations below the minimum stake.
-        for ((hotkey, coldkey, netuid), _) in Alpha::<T>::iter() {
+        for ((hotkey, coldkey, netuid), _) in Self::alpha_iter() {
             Self::clear_small_nomination_if_required(&hotkey, &coldkey, netuid);
         }
     }
@@ -413,7 +414,117 @@ impl<T: Config> Pallet<T> {
         }
     }
 
+    // Same thing as populate_root_coldkey_staking_maps, but for AlphaV2
+    pub fn populate_root_coldkey_staking_maps_v2() {
+        // Get starting key for the batch. Get the first key if we restart the process.
+        let mut new_starting_raw_key = AlphaV2MapLastKey::<T>::get();
+        let mut starting_key = None;
+        if new_starting_raw_key.is_none() {
+            starting_key = AlphaV2::<T>::iter_keys().next();
+            new_starting_raw_key = starting_key.as_ref().map(AlphaV2::<T>::hashed_key_for);
+        }
+
+        if let Some(starting_raw_key) = new_starting_raw_key {
+            // Get the key batch
+            let mut keys = AlphaV2::<T>::iter_keys_from(starting_raw_key)
+                .take(ALPHA_MAP_BATCH_SIZE)
+                .collect::<Vec<_>>();
+
+            // New iteration: insert the starting key in the batch if it's a new iteration
+            // iter_keys_from() skips the starting key
+            if let Some(starting_key) = starting_key {
+                if keys.len() == ALPHA_MAP_BATCH_SIZE {
+                    keys.remove(keys.len().saturating_sub(1));
+                }
+                keys.insert(0, starting_key);
+            }
+
+            let mut new_starting_key = None;
+            let new_iteration = keys.len() < ALPHA_MAP_BATCH_SIZE;
+
+            // Check and remove alphas if necessary
+            for key in keys {
+                let (_, coldkey, netuid) = key.clone();
+
+                if netuid == NetUid::ROOT {
+                    Self::maybe_add_coldkey_index(&coldkey);
+                }
+
+                new_starting_key = Some(AlphaV2::<T>::hashed_key_for(key));
+            }
+
+            // Restart the process if it's the last batch
+            if new_iteration {
+                new_starting_key = None;
+            }
+
+            AlphaV2MapLastKey::<T>::put(new_starting_key);
+        }
+    }
+
     pub fn burn_subnet_alpha(_netuid: NetUid, _amount: AlphaCurrency) {
         // Do nothing; TODO: record burned alpha in a tracker
     }
+
+    /// Several alpha iteration helpers that merge key space from Alpha and AlphaV2 maps
+    pub fn alpha_iter() -> impl Iterator<Item = ((T::AccountId, T::AccountId, NetUid), SafeFloat)> {
+        // Old Alpha shares format: U64F64 -> SafeFloat
+        let legacy = Alpha::<T>::iter().map(|(key, val_u64f64)| {
+            let sf: SafeFloat = val_u64f64.into();
+            (key, sf)
+        });
+
+        // New Alpha shares format: SafeFloatSerializable -> SafeFloat
+        let v2 = AlphaV2::<T>::iter().map(|(key, val_sf_ser)| {
+            let sf: SafeFloat = SafeFloat::from(&val_sf_ser);
+            (key, sf)
+        });
+
+        legacy.chain(v2)
+    }
+
+    pub fn alpha_iter_prefix(
+        prefix: (&T::AccountId, &T::AccountId),
+    ) -> impl Iterator<Item = (NetUid, SafeFloat)>
+    where
+        T::AccountId: Clone,
+    {
+        // Old Alpha shares format: U64F64 -> SafeFloat
+        let legacy = Alpha::<T>::iter_prefix(prefix).map(|(netuid, val_u64f64)| {
+            let sf: SafeFloat = val_u64f64.into();
+            (netuid, sf)
+        });
+
+        // New Alpha shares format: SafeFloatSerializable -> SafeFloat
+        let v2 = AlphaV2::<T>::iter_prefix(prefix).map(|(netuid, val_sf_ser)| {
+            let sf: SafeFloat = SafeFloat::from(&val_sf_ser);
+            (netuid, sf)
+        });
+
+        legacy
+            .chain(v2)
+            .filter(|(_, alpha_share)| !alpha_share.is_zero())
+    }
+
+    pub fn alpha_iter_single_prefix(
+        prefix: &T::AccountId,
+    ) -> impl Iterator<Item = (T::AccountId, NetUid, SafeFloat)>
+    where
+        T::AccountId: Clone,
+    {
+        // Old Alpha shares format: U64F64 -> SafeFloat
+        let legacy =
+            Alpha::<T>::iter_prefix((prefix.clone(),)).map(|((coldkey, netuid), val_u64f64)| {
+                let sf: SafeFloat = val_u64f64.into();
+                (coldkey, netuid, sf)
+            });
+
+        // New Alpha shares format: SafeFloatSerializable -> SafeFloat
+        let v2 = AlphaV2::<T>::iter_prefix((prefix,)).map(|((coldkey, netuid), val_sf_ser)| {
+            let sf: SafeFloat = SafeFloat::from(&val_sf_ser);
+            (coldkey, netuid, sf)
+        });
+
+        legacy.chain(v2)
+    }
 }