source: trunk/misc/operations_helpers/provisioning/reliability.py

#! /usr/bin/python


import math
from allmydata.util import statistics
from numpy import array, matrix, dot

DAY=24*60*60
MONTH=31*DAY
YEAR=365*DAY

class ReliabilityModel(object):
    """Generate a model of system-wide reliability, given several input
    parameters.

    This runs a simulation in which time is quantized down to 'delta' seconds
    (default is one month): a smaller delta will result in a more accurate
    simulation, but will take longer to run. 'report_span' simulated seconds
    will be run.

    The encoding parameters are provided as 'k' (minimum number of shares
    needed to recover the file) and 'N' (total number of shares generated).
    The default parameters are 3-of-10.

    The first step is to build a probability of individual drive loss during
    any given delta. This uses a simple exponential model, in which the
    average drive lifetime is specified by the 'drive_lifetime' parameter
    (default is 8 years).

    The second step is to calculate a 'transition matrix': a table of
    probabilities that shows, given A shares at the start of the delta, what
    the chances are of having B shares left at the end of the delta. The
    current code optimistically assumes all drives are independent. A
    subclass could override that assumption.

    An additional 'repair matrix' is created to show what happens when the
    Checker/Repairer is run. In the simulation, the Checker will be run every
    'check_period' seconds (default is one month), and the Repairer will be
    run if it sees fewer than 'R' shares (default 7).

    The third step is to finally run the simulation. An initial probability
    vector is created (with a 100% chance of N shares and a 0% chance of
    fewer than N shares), then it is multiplied by the transition matrix for
    every delta of time. Each time the Checker is to be run, the repair
    matrix is multiplied in, and some additional stats are accumulated
    (average number of repairs that occur, average number of shares
    regenerated per repair).

    The output is a ReliabilityReport instance, which contains a table that
    samples the state of the simulation once each 'report_period' seconds
    (defaults to 3 months). Each row of this table will contain the
    probability vector for one sample period (chance of having X shares, from
    0 to N, at the end of the period). The report will also contain other
    information.

    """

    @classmethod
    def run(klass,
            drive_lifetime=8*YEAR,
            k=3, R=7, N=10,
            delta=1*MONTH,
            check_period=1*MONTH,
            report_period=3*MONTH,
            report_span=5*YEAR,
            ):
        self = klass()

        check_period = check_period-1
        P = self.p_in_period(drive_lifetime, delta)

        decay = self.build_decay_matrix(N, P)

        repair = self.build_repair_matrix(k, N, R)

        #print("DECAY:", decay)
        #print("OLD-POST-REPAIR:", old_post_repair)
        #print("NEW-POST-REPAIR:", decay * repair)
        #print("REPAIR:", repair)
        #print("DIFF:", (old_post_repair - decay * repair))

        START = array([0]*N + [1])
        DEAD = array([1]*k + [0]*(1+N-k))
        REPAIRp = array([0]*k + [1]*(R-k) + [0]*(1+N-R))
        REPAIR_newshares = array([0]*k +
                                 [N-i for i in range(k, R)] +
                                 [0]*(1+N-R))
        assert REPAIR_newshares.shape[0] == N+1
        #print("START", START)
        #print("REPAIRp", REPAIRp)
        #print("REPAIR_newshares", REPAIR_newshares)

        unmaintained_state = START
        maintained_state = START
        last_check = 0
        last_report = 0
        P_repaired_last_check_period = 0.0
        needed_repairs = []
        needed_new_shares = []
        report = ReliabilityReport()

        for t in range(0, report_span+delta, delta):
            # the .A[0] turns the one-row matrix back into an array
            unmaintained_state = (unmaintained_state * decay).A[0]
            maintained_state = (maintained_state * decay).A[0]
            if (t-last_check) > check_period:
                last_check = t
                # we do a check-and-repair this frequently
                need_repair = dot(maintained_state, REPAIRp)

                P_repaired_last_check_period = need_repair
                new_shares = dot(maintained_state, REPAIR_newshares)
                needed_repairs.append(need_repair)
                needed_new_shares.append(new_shares)

                maintained_state = (maintained_state * repair).A[0]

            if (t-last_report) > report_period:
                last_report = t
                P_dead_unmaintained = dot(unmaintained_state, DEAD)
                P_dead_maintained = dot(maintained_state, DEAD)
                cumulative_number_of_repairs = sum(needed_repairs)
                cumulative_number_of_new_shares = sum(needed_new_shares)
                report.add_sample(t, unmaintained_state, maintained_state,
                                  P_repaired_last_check_period,
                                  cumulative_number_of_repairs,
                                  cumulative_number_of_new_shares,
                                  P_dead_unmaintained, P_dead_maintained)

        # record one more sample at the end of the run
        P_dead_unmaintained = dot(unmaintained_state, DEAD)
        P_dead_maintained = dot(maintained_state, DEAD)
        cumulative_number_of_repairs = sum(needed_repairs)
        cumulative_number_of_new_shares = sum(needed_new_shares)
        report.add_sample(t, unmaintained_state, maintained_state,
                          P_repaired_last_check_period,
                          cumulative_number_of_repairs,
                          cumulative_number_of_new_shares,
                          P_dead_unmaintained, P_dead_maintained)

        #def yandm(seconds):
        #    return "%dy.%dm" % (int(seconds/YEAR), int( (seconds%YEAR)/MONTH))
        #needed_repairs_total = sum(needed_repairs)
        #needed_new_shares_total = sum(needed_new_shares)
        #print("at 2y:")
        #print(" unmaintained", unmaintained_state)
        #print(" maintained", maintained_state)
        #print(" number of repairs", needed_repairs_total)
        #print(" new shares generated", needed_new_shares_total)
        #repair_rate_inv = report_span / needed_repairs_total
        #print("  avg repair rate: once every %s" % yandm(repair_rate_inv))
        #print("  avg repair download: one share every %s" % yandm(repair_rate_inv/k))
        #print("  avg repair upload: one share every %s" % yandm(report_span / needed_new_shares_total))

        return report

    def p_in_period(self, avg_lifetime, period):
        """Given an average lifetime of a disk (using an exponential model),
        what is the chance that a live disk will survive the next 'period'
        seconds?"""

        # eg p_in_period(8*YEAR, MONTH) = 98.94%
        return math.exp(-1.0*period/avg_lifetime)

    def build_decay_matrix(self, N, P):
        """Return a decay matrix. decay[start_shares][end_shares] is the
        conditional probability of finishing with end_shares, given that we
        started with start_shares."""
        decay_rows = []
        decay_rows.append( [0.0]*(N+1) )
        for start_shares in range(1, (N+1)):
            end_shares = self.build_decay_row(start_shares, P)
            decay_row = end_shares + [0.0] * (N-start_shares)
            assert len(decay_row) == (N+1), len(decay_row)
            decay_rows.append(decay_row)

        decay = matrix(decay_rows)
        return decay

    def build_decay_row(self, start_shares, P):
        """Return a decay row 'end_shares'. end_shares[i] is the chance that
        we finish with i shares, given that we started with start_shares, for
        all i between 0 and start_shares, inclusive. This implementation
        assumes that all shares are independent (IID), but a more complex
        model could incorporate inter-share failure correlations like having
        two shares on the same server."""
        end_shares = statistics.binomial_distribution_pmf(start_shares, P)
        return end_shares

    def build_repair_matrix(self, k, N, R):
        """Return a repair matrix. repair[start][end]: is the conditional
        probability of the repairer finishing with 'end' shares, given that
        it began with 'start' shares (repair if fewer than R shares). The
        repairer's behavior is deterministic, so all values in this matrix
        are either 0 or 1. This matrix should be applied *after* the decay
        matrix."""
        new_repair_rows = []
        for start_shares in range(0, N+1):
            new_repair_row = [0] * (N+1)
            if start_shares < k:
                new_repair_row[start_shares] = 1
            elif start_shares < R:
                new_repair_row[N] = 1
            else:
                new_repair_row[start_shares] = 1
            new_repair_rows.append(new_repair_row)

        repair = matrix(new_repair_rows)
        return repair

class ReliabilityReport(object):
    def __init__(self):
        self.samples = []

    def add_sample(self, when, unmaintained_shareprobs, maintained_shareprobs,
                   P_repaired_last_check_period,
                   cumulative_number_of_repairs,
                   cumulative_number_of_new_shares,
                   P_dead_unmaintained, P_dead_maintained):
        """
        when: the timestamp at the end of the report period
        unmaintained_shareprobs: a vector of probabilities, element[S]
                                 is the chance that there are S shares
                                 left at the end of the report period.
                                 This tracks what happens if no repair
                                 is ever done.
        maintained_shareprobs: same, but for 'maintained' grids, where
                               check and repair is done at the end
                               of each check period
        P_repaired_last_check_period: a float, with the probability
                                      that a repair was performed
                                      at the end of the most recent
                                      check period.
        cumulative_number_of_repairs: a float, with the average number
                                      of repairs that will have been
                                      performed by the end of the
                                      report period
        cumulative_number_of_new_shares: a float, with the average number
                                         of new shares that repair proceses
                                         generated by the end of the report
                                         period
        P_dead_unmaintained: a float, with the chance that the file will
                             be unrecoverable at the end of the period
        P_dead_maintained: same, but for maintained grids

        """
        row = (when, unmaintained_shareprobs, maintained_shareprobs,
               P_repaired_last_check_period,
               cumulative_number_of_repairs,
               cumulative_number_of_new_shares,
               P_dead_unmaintained, P_dead_maintained)
        self.samples.append(row)