Ticket #999: test_backends.py

import mock
from twisted.trial import unittest

from allmydata.test.common_util import ReallyEqualMixin
from allmydata.test.mock_s3 import MockS3Bucket

# This is the code that we're going to be testing.
from allmydata import node
from allmydata.storage.server import StorageServer
from allmydata.storage.backends.null.null_backend import NullBackend
from allmydata.storage.backends.disk.disk_backend import DiskBackend, si_si2dir
from allmydata.storage.backends.s3.s3_backend import S3Backend


# The following share file content was generated with
# storage.immutable.ShareFile from Tahoe-LAFS v1.8.2
# with share data == 'a'. The total size of this input
# is 85 bytes.
shareversionnumber = '\x00\x00\x00\x01'
sharedatalength = '\x00\x00\x00\x01'
numberofleases = '\x00\x00\x00\x01'
shareinputdata = 'a'
ownernumber = '\x00\x00\x00\x00'
renewsecret  = 'x'*32
cancelsecret = 'y'*32
expirationtime = '\x00(\xde\x80'
nextlease = ''
containerdata = shareversionnumber + sharedatalength + numberofleases
client_data = shareinputdata + ownernumber + renewsecret + \
    cancelsecret + expirationtime + nextlease
share_data = containerdata + client_data
testnodeid = 'testnodeidxxxxxxxxxx'


class TestServerWithNullBackend(unittest.TestCase, ReallyEqualMixin):
    """ NullBackend is just for testing and executable documentation, so
    this test is actually a test of StorageServer in which we're using
    NullBackend as helper code for the test, rather than a test of
    NullBackend. """
    def setUp(self):
        self.ss = StorageServer(testnodeid, NullBackend())

    @mock.patch('os.mkdir')
    @mock.patch('__builtin__.open')
    @mock.patch('os.listdir')
    @mock.patch('os.path.isdir')
    def test_write_share(self, mockisdir, mocklistdir, mockopen, mockmkdir):
        """
        Write a new share. This tests that StorageServer's remote_allocate_buckets
        generates the correct return types when given test-vector arguments. That
        bs is of the correct type is verified by attempting to invoke remote_write
        on bs[0].
        """
        alreadygot, bs = self.ss.remote_allocate_buckets('teststorage_index', 'x'*32, 'y'*32, set((0,)), 1, mock.Mock())
        bs[0].remote_write(0, 'a')
        self.failIf(mockisdir.called)
        self.failIf(mocklistdir.called)
        self.failIf(mockopen.called)
        self.failIf(mockmkdir.called)


#class ServerWithS3Backend(Server):
#    def create(self, name, reserved_space=0):
#        workdir = self.workdir(name)
#        s3bucket = MockS3Bucket(workdir)
#        backend = S3Backend(s3bucket, readonly=False, reserved_space=reserved_space)
#        ss = StorageServer("\x00" * 20, backend, workdir,
#                           stats_provider=FakeStatsProvider())
#        ss.setServiceParent(self.sparent)
#        return ss


#class ServerWithDiskBackend(Server):
#    def create(self, name, reserved_space=0, klass=StorageServer):
#        workdir = self.workdir(name)
#        backend = DiskBackend(workdir)
#        ss = StorageServer("\x00" * 20, backend, workdir,
#                           stats_provider=FakeStatsProvider())
#        ss.setServiceParent(self.sparent)
#        return ss


class Server(ReallyEqualMixin):
    """ This tests the StorageServer with a given back-end. """
    def set_up(self):
        self.backend = DiskBackend(self.storedir)
        self.ss = StorageServer(testnodeid, self.backend)

        self.backendwithreserve = DiskBackend(self.storedir, reserved_space = 1)
        self.sswithreserve = StorageServer(testnodeid, self.backendwithreserve)

    @mock.patch('time.time')
    def test_write_and_read_share(self, mocktime):
        """
        Write a new share, read it, and test the server's (and disk backend's)
        handling of simultaneous and successive attempts to write the same
        share.
        """
        mocktime.return_value = 0

        shareset = self.ss.backend.get_shareset('teststorage_index')

        self.failIf(shareset.has_incoming(0))

        # Populate incoming with the sharenum: 0.
        alreadygot, bs = self.ss.remote_allocate_buckets('teststorage_index', 'x'*32, 'y'*32, frozenset((0,)), 1, mock.Mock())

        # This is a white-box test: Inspect incoming and fail unless the sharenum: 0 is listed there.
        self.failUnless(shareset.has_incoming(0))

        # Attempt to create a second share writer with the same sharenum.
        alreadygota, bsa = self.ss.remote_allocate_buckets('teststorage_index', 'x'*32, 'y'*32, frozenset((0,)), 1, mock.Mock())

        # Show that no sharewriter results from a remote_allocate_buckets
        # with the same si and sharenum, until BucketWriter.remote_close()
        # has been called.
        self.failIf(bsa)

        # Test allocated size.
        spaceint = self.ss.allocated_size()
        self.failUnlessReallyEqual(spaceint, 1)

        # Write 'a' to shnum 0. Only tested together with close and read.
        bs[0].remote_write(0, 'a')

        # Preclose: Inspect final, failUnless nothing there.
        self.failUnlessReallyEqual(len(list(self.backend.get_shares('teststorage_index'))), 0)
        bs[0].remote_close()

        # Postclose: (Omnibus) failUnless written data is in final.
        sharesinfinal = list(self.backend.get_shares('teststorage_index'))
        self.failUnlessReallyEqual(len(sharesinfinal), 1)
        contents = sharesinfinal[0].read_share_data(0, 73)
        self.failUnlessReallyEqual(contents, client_data)

        # Exercise the case that the share we're asking to allocate is
        # already (completely) uploaded.
        self.ss.remote_allocate_buckets('teststorage_index', 'x'*32, 'y'*32, set((0,)), 1, mock.Mock())


    def test_read_old_share(self):
        """ This tests whether the code correctly finds and reads
        shares written out by old (Tahoe-LAFS <= v1.8.2)
        servers. There is a similar test in test_download, but that one
        is from the perspective of the client and exercises a deeper
        stack of code. This one is for exercising just the
        StorageServer object. """
        # Contruct a file with the appropriate contents in the mockfilesystem.
        datalen = len(share_data)
        finalhome = si_si2dir(self.basedir, 'teststorage_index').child(str(0))
        finalhome.setContent(share_data)

        # Now begin the test.
        bs = self.ss.remote_get_buckets('teststorage_index')

        self.failUnlessEqual(len(bs), 1)
        b = bs['0']
        # These should match by definition, the next two cases cover cases without (completely) unambiguous behaviors.
        self.failUnlessReallyEqual(b.remote_read(0, datalen), client_data)
        # If you try to read past the end you get the as much data as is there.
        self.failUnlessReallyEqual(b.remote_read(0, datalen+20), client_data)
        # If you start reading past the end of the file you get the empty string.
        self.failUnlessReallyEqual(b.remote_read(datalen+1, 3), '')


class MockConfigParser(object):
    def __init__(self, configged_vals):
        self.configged_vals = configged_vals
    def read(self, fname):
        pass
    def getboolean(self, section, option):
        raise NotImplementedError
    def get(self, section, option):
        return self.configged_vals[(section, option)]


class TestConfigureBackends(ReallyEqualMixin, unittest.TestCase):
    @mock.patch('ConfigParser.SafeConfigParser')
    def test_configure_disk_backend_bad_reserved_space(self, mockCP):
        mockCP.return_value = MockConfigParser({
                ("storage", "backend"): "disk",
                ("storage", "reserved_space"): "not a real answer",
                })

        self.failUnlessRaises(node.MissingConfigEntry, Client)