module Ganeti.WConfd.TempRes
( TempRes
, mkTempRes
, TempResState(..)
, emptyTempResState
, NodeUUID
, InstanceUUID
, DiskUUID
, NetworkUUID
, DRBDMinor
, DRBDMap
, trsDRBDL
, computeDRBDMap
, computeDRBDMap'
, allocateDRBDMinor
, releaseDRBDMinors
, MAC
, generateMAC
, reserveMAC
, generateDRBDSecret
, reserveLV
, IPv4ResAction(..)
, IPv4Reservation(..)
, reserveIp
, releaseIp
, generateIp
, commitReleaseIp
, commitReservedIps
, listReservedIps
, dropAllReservations
, isReserved
, reserve
, dropReservationsFor
, reserved
) where
import Control.Lens.At
import Control.Monad.Error
import Control.Monad.State
import Control.Monad.Trans.Maybe
import qualified Data.ByteString as BS
import qualified Data.ByteString.UTF8 as UTF8
import qualified Data.Foldable as F
import Data.Maybe
import Data.Map (Map)
import qualified Data.Map as M
import Data.Monoid
import qualified Data.Semigroup as Sem
import qualified Data.Set as S
import System.Random
import qualified Text.JSON as J
import Ganeti.BasicTypes
import Ganeti.Config
import qualified Ganeti.Constants as C
import Ganeti.Errors
import qualified Ganeti.JSON as J
import Ganeti.Lens
import qualified Ganeti.Network as N
import Ganeti.Locking.Locks (ClientId)
import Ganeti.Logging
import Ganeti.Objects
import Ganeti.THH
import Ganeti.Objects.Lens (configNetworksL)
import Ganeti.Utils
import Ganeti.Utils.Monad
import Ganeti.Utils.Random
import qualified Ganeti.Utils.MultiMap as MM
type NodeUUID = BS.ByteString
type InstanceUUID = BS.ByteString
type DiskUUID = BS.ByteString
type NetworkUUID = BS.ByteString
type DRBDMinor = Int
type DRBDMap = Map NodeUUID (Map DRBDMinor DiskUUID)
type DRBDMap' = Map NodeUUID (Map DRBDMinor [DiskUUID])
data IPv4ResAction = IPv4Reserve | IPv4Release
deriving (Eq, Ord, Show, Bounded, Enum)
instance J.JSON IPv4ResAction where
showJSON IPv4Reserve = J.JSString . J.toJSString $ C.reserveAction
showJSON IPv4Release = J.JSString . J.toJSString $ C.releaseAction
readJSON = J.readEitherString
>=> \s -> case () of
_ | s == C.reserveAction -> return IPv4Reserve
| s == C.releaseAction -> return IPv4Release
| otherwise -> fail $ "Invalid IP reservation action: "
++ s
data IPv4Reservation = IPv4Res
{ ipv4ResAction :: IPv4ResAction
, ipv4ResNetwork :: NetworkUUID
, ipv4ResAddr :: Ip4Address
} deriving (Eq, Ord, Show)
instance J.JSON IPv4Reservation where
showJSON (IPv4Res a net addr) = J.showJSON (a, addr, net)
readJSON = fmap (\(a, addr, net) -> IPv4Res a net addr) . J.readJSON
newtype TempRes j a = TempRes { getTempRes :: MM.MultiMap j a }
deriving (Eq, Ord, Show)
instance (Ord j, Ord a) => Sem.Semigroup (TempRes j a) where
(TempRes x) <> (TempRes y) = TempRes $ x <> y
instance (Ord j, Ord a) => Monoid (TempRes j a) where
mempty = TempRes mempty
mappend = (Sem.<>)
instance (J.JSON j, Ord j, J.JSON a, Ord a) => J.JSON (TempRes j a) where
showJSON = J.showJSON . getTempRes
readJSON = liftM TempRes . J.readJSON
mkTempRes :: MM.MultiMap j a -> TempRes j a
mkTempRes = TempRes
$(buildObject "TempResState" "trs"
[ simpleField "dRBD" [t| DRBDMap |]
, simpleField "mACs" [t| TempRes ClientId MAC |]
, simpleField "dRBDSecrets" [t| TempRes ClientId DRBDSecret |]
, simpleField "lVs" [t| TempRes ClientId LogicalVolume |]
, simpleField "iPv4s" [t| TempRes ClientId IPv4Reservation |]
])
emptyTempResState :: TempResState
emptyTempResState = TempResState M.empty mempty mempty mempty mempty
$(makeCustomLenses ''TempResState)
resError :: (MonadError GanetiException m) => String -> m a
resError = throwError . ReservationError
toResError :: (MonadError GanetiException m) => m a -> m a
toResError = flip catchError (throwError . f)
where
f (GenericError msg) = ReservationError msg
f e = e
filterNested :: (Ord a, Ord b)
=> (c -> Bool) -> Map a (Map b c) -> Map a (Map b c)
filterNested p = M.filter (not . M.null) . fmap (M.filter p)
maybeLens :: (Monoid a, Monoid b, Eq b)
=> Lens s t (Maybe a) (Maybe b) -> Lens s t a b
maybeLens l f = l (fmap (mfilter (/= mempty) . Just) . f . fromMaybe mempty)
computeDRBDMap' :: (MonadError GanetiException m)
=> ConfigData -> TempResState -> m DRBDMap'
computeDRBDMap' cfg trs =
flip execStateT (fmap (fmap (: [])) (trsDRBD trs))
$ F.forM_ (configDisks cfg) addMinors
where
nodeMinor :: NodeUUID -> DRBDMinor -> Lens' DRBDMap' [DiskUUID]
nodeMinor node minor = maybeLens (at node) . maybeLens (at minor)
addMinors disk = do
let minors = getDrbdMinorsForDisk disk
forM_ minors $ \(minor, node) ->
nodeMinor (UTF8.fromString node) minor %=
(UTF8.fromString (uuidOf disk) :)
computeDRBDMap :: (MonadError GanetiException m)
=> ConfigData -> TempResState -> m DRBDMap
computeDRBDMap cfg trs = do
m <- computeDRBDMap' cfg trs
let dups = filterNested ((>= 2) . length) m
unless (M.null dups) . resError
$ "Duplicate DRBD ports detected: " ++ show (M.toList $ fmap M.toList dups)
return $ fmap (fmap head . M.filter ((== 1) . length)) m
`M.union` (fmap (const mempty) . J.fromContainer . configNodes $ cfg)
allocateDRBDMinor :: (MonadError GanetiException m, MonadState TempResState m)
=> ConfigData -> DiskUUID -> [NodeUUID]
-> m [DRBDMinor]
allocateDRBDMinor cfg disk nodes = do
unless (nodes == ordNub nodes) . resError
$ "Duplicate nodes detected in list '" ++ show nodes ++ "'"
dMap <- computeDRBDMap' cfg =<< get
let usedMap = fmap M.keysSet dMap
let alloc :: S.Set DRBDMinor -> Map DRBDMinor DiskUUID
-> (DRBDMinor, Map DRBDMinor DiskUUID)
alloc used m = let k = findFirst 0 (M.keysSet m `S.union` used)
in (k, M.insert k disk m)
forM nodes $ \node -> trsDRBDL . maybeLens (at node)
%%= alloc (M.findWithDefault mempty node usedMap)
releaseDRBDMinors :: (MonadState TempResState m) => DiskUUID -> m ()
releaseDRBDMinors disk = trsDRBDL %= filterNested (/= disk)
isReserved :: (Ord a, Ord j) => a -> TempRes j a -> Bool
isReserved x = MM.elem x . getTempRes
reserve :: (MonadError GanetiException m, Show a, Ord a, Ord j)
=> j -> a -> TempRes j a -> m (TempRes j a)
reserve jobid x tr = do
when (isReserved x tr) . resError $ "Duplicate reservation for resource '"
++ show x ++ "'"
return . TempRes . MM.insert jobid x $ getTempRes tr
dropReservationsFor :: (Ord a, Ord j) => j -> TempRes j a -> TempRes j a
dropReservationsFor jobid = TempRes . MM.deleteAll jobid . getTempRes
reservedFor :: (Ord a, Ord j) => j -> TempRes j a -> S.Set a
reservedFor jobid = MM.lookup jobid . getTempRes
reserved :: (Ord a, Ord j) => TempRes j a -> S.Set a
reserved = MM.values . getTempRes
withReserved :: (MonadError GanetiException m, Show a, Ord a, Ord j)
=> j -> (S.Set a -> m a) -> TempRes j a -> m (a, TempRes j a)
withReserved jobid genfn tr = do
x <- genfn (reserved tr)
(,) x `liftM` reserve jobid x tr
generate :: (MonadError GanetiException m, Show a, Ord a, Ord j)
=> j -> S.Set a -> m (Maybe a) -> TempRes j a -> m (a, TempRes j a)
generate jobid existing genfn = withReserved jobid f
where
retries = 64 :: Int
f res = do
let vals = res `S.union` existing
xOpt <- retryMaybeN retries
(\_ -> mfilter (`S.notMember` vals) (MaybeT genfn))
maybe (resError "Not able generate new resource")
return xOpt
generateRand
:: (MonadError GanetiException m, Show a, Ord a, Ord j, RandomGen g)
=> g -> j -> S.Set a -> (g -> (Maybe a, g)) -> TempRes j a
-> m (a, TempRes j a)
generateRand rgen jobid existing genfn tr =
evalStateT (generate jobid existing (state genfn) tr) rgen
stateM :: (MonadState s m) => (s -> m (a, s)) -> m a
stateM f = get >>= f >>= \(x, s) -> liftM (const x) (put s)
modifyM :: (MonadState s m) => (s -> m s) -> m ()
modifyM f = get >>= f >>= put
dropAllReservations :: ClientId -> State TempResState ()
dropAllReservations jobId = modify $
(trsMACsL %~ dropReservationsFor jobId)
. (trsDRBDSecretsL %~ dropReservationsFor jobId)
. (trsLVsL %~ dropReservationsFor jobId)
. (trsIPv4sL %~ dropReservationsFor jobId)
lookupNetwork :: (MonadError GanetiException m)
=> ConfigData -> NetworkUUID -> m Network
lookupNetwork cd netId =
J.lookupContainer (resError $ "Network '" ++ show netId ++ "' not found")
netId (configNetworks cd)
generateMAC
:: (RandomGen g, MonadError GanetiException m, Functor m)
=> g -> ClientId -> Maybe NetworkUUID -> ConfigData
-> StateT TempResState m MAC
generateMAC rgen jobId netId cd = do
net <- case netId of
Just n -> Just <$> lookupNetwork cd n
Nothing -> return Nothing
let prefix = fromMaybe (clusterMacPrefix . configCluster $ cd)
(networkMacPrefix =<< net)
let existing = S.fromList $ getAllMACs cd
StateT
$ traverseOf2 trsMACsL
(generateRand rgen jobId existing
(over _1 Just . generateOneMAC prefix))
reserveMAC
:: (MonadError GanetiException m, MonadState TempResState m, Functor m)
=> ClientId -> MAC -> ConfigData -> m ()
reserveMAC jobId mac cd = do
let existing = S.fromList $ getAllMACs cd
when (S.member mac existing)
$ resError "MAC already in use"
modifyM $ traverseOf trsMACsL (reserve jobId mac)
generateDRBDSecret
:: (RandomGen g, MonadError GanetiException m, Functor m)
=> g -> ClientId -> ConfigData -> StateT TempResState m DRBDSecret
generateDRBDSecret rgen jobId cd = do
let existing = S.fromList $ getAllDrbdSecrets cd
StateT $ traverseOf2 trsDRBDSecretsL
(generateRand rgen jobId existing
(over _1 Just . generateSecret C.drbdSecretLength))
reserveLV
:: (MonadError GanetiException m, MonadState TempResState m, Functor m)
=> ClientId -> LogicalVolume -> ConfigData -> m ()
reserveLV jobId lv cd = do
existing <- toError $ getAllLVs cd
when (S.member lv existing)
$ resError "MAC already in use"
modifyM $ traverseOf trsLVsL (reserve jobId lv)
usedIPv4Addrs :: NetworkUUID -> S.Set IPv4Reservation -> S.Set Ip4Address
usedIPv4Addrs netuuid =
S.map ipv4ResAddr . S.filter ((== netuuid) . ipv4ResNetwork)
reserveIp
:: (MonadError GanetiException m, MonadState TempResState m, Functor m)
=> ClientId -> NetworkUUID -> Ip4Address
-> Bool
-> ConfigData -> m ()
reserveIp jobId netuuid addr checkExt cd = toResError $ do
net <- lookupNetwork cd netuuid
isres <- N.isReserved N.PoolInstances addr net
when isres . resError $ "IP address already in use"
when checkExt $ do
isextres <- N.isReserved N.PoolExt addr net
when isextres . resError $ "IP is externally reserved"
let action = IPv4Res IPv4Reserve netuuid addr
modifyM $ traverseOf trsIPv4sL (reserve jobId action)
releaseIp
:: (MonadError GanetiException m, MonadState TempResState m, Functor m)
=> ClientId -> NetworkUUID -> Ip4Address -> m ()
releaseIp jobId netuuid addr =
let action = IPv4Res { ipv4ResAction = IPv4Release
, ipv4ResNetwork = netuuid
, ipv4ResAddr = addr }
in modifyM $ traverseOf trsIPv4sL (reserve jobId action)
generateIp
:: (MonadError GanetiException m, MonadState TempResState m, Functor m)
=> ClientId -> NetworkUUID -> ConfigData -> m Ip4Address
generateIp jobId netuuid cd = toResError $ do
net <- lookupNetwork cd netuuid
let f res = do
let ips = usedIPv4Addrs netuuid res
addr <- N.findFree (`S.notMember` ips) net
maybe (resError "Cannot generate IP. Network is full")
(return . IPv4Res IPv4Reserve netuuid) addr
liftM ipv4ResAddr . stateM $ traverseOf2 trsIPv4sL (withReserved jobId f)
commitIp
:: (MonadError GanetiException m, Functor m)
=> IPv4Reservation -> ConfigData -> m ConfigData
commitIp (IPv4Res actType netuuid addr) cd = toResError $ do
let call = case actType of
IPv4Reserve -> N.reserve
IPv4Release -> N.release
f Nothing = resError $ "Network '" ++ show netuuid ++ "' not found"
f (Just net) = Just `liftM` call N.PoolInstances addr net
traverseOf (configNetworksL . J.alterContainerL netuuid) f cd
commitReleaseIp
:: (MonadError GanetiException m, Functor m)
=> NetworkUUID -> Ip4Address -> ConfigData -> m ConfigData
commitReleaseIp netuuid addr =
commitIp (IPv4Res IPv4Release netuuid addr)
commitReservedIps
:: (MonadError GanetiException m, Functor m, MonadLog m)
=> ClientId -> TempResState -> ConfigData -> m ConfigData
commitReservedIps jobId tr cd = do
let res = reservedFor jobId (trsIPv4s tr)
logDebug $ "Commiting reservations: " ++ show res
F.foldrM commitIp cd res
listReservedIps :: ClientId -> TempResState -> S.Set IPv4Reservation
listReservedIps jobid = reservedFor jobid . trsIPv4s