diff '--exclude=_version.py' '--exclude=*.rej' '--exclude=.git' -u -r merge-leasedb-and-cloud-backend-from-ds/rejects.txt merge-leasedb-and-cloud-backend/rejects.txt --- merge-leasedb-and-cloud-backend-from-ds/rejects.txt 2012-11-26 21:05:06.000000000 -0700 +++ merge-leasedb-and-cloud-backend/rejects.txt 2012-11-27 09:25:03.620391737 -0700 @@ -206,7 +206,7 @@ Hunk #3 succeeded at 600 (offset 28 lines). Hunk #4 succeeded at 641 (offset 28 lines). Hunk #5 succeeded at 1303 (offset 28 lines). -1 out of 5 hunks FAILED -- saving rejects to file src/allmydata/test/common.py.rej +DONE 1 out of 5 hunks FAILED -- saving rejects to file src/allmydata/test/common.py.rej patching file src/allmydata/test/no_network.py Hunk #6 succeeded at 299 with fuzz 2. Hunk #7 succeeded at 319 (offset 1 line). @@ -223,13 +223,13 @@ Hunk #3 FAILED at 58. Hunk #4 FAILED at 92. Hunk #5 FAILED at 336. -5 out of 5 hunks FAILED -- saving rejects to file src/allmydata/test/test_crawler.py.rej +DONE 5 out of 5 hunks FAILED -- saving rejects to file src/allmydata/test/test_crawler.py.rej patching file src/allmydata/test/test_deepcheck.py patching file src/allmydata/test/test_download.py patching file src/allmydata/test/test_encode.py patching file src/allmydata/test/test_hung_server.py Hunk #4 FAILED at 49. -1 out of 8 hunks FAILED -- saving rejects to file src/allmydata/test/test_hung_server.py.rej +DONE 1 out of 8 hunks FAILED -- saving rejects to file src/allmydata/test/test_hung_server.py.rej patching file src/allmydata/test/test_immutable.py patching file src/allmydata/test/test_mutable.py Hunk #2 succeeded at 1911 (offset 2 lines). @@ -243,7 +243,7 @@ patching file src/allmydata/test/test_node.py patching file src/allmydata/test/test_repairer.py Hunk #8 FAILED at 700. -1 out of 8 hunks FAILED -- saving rejects to file src/allmydata/test/test_repairer.py.rej +DONE 1 out of 8 hunks FAILED -- saving rejects to file src/allmydata/test/test_repairer.py.rej patching file src/allmydata/test/test_runner.py Hunk #1 succeeded at 216 (offset 2 lines). patching file src/allmydata/test/test_storage.py @@ -343,7 +343,7 @@ Hunk #25 succeeded at 1844 (offset -3 lines). Hunk #26 succeeded at 1887 (offset -3 lines). Hunk #27 succeeded at 1935 (offset -3 lines). -2 out of 27 hunks FAILED -- saving rejects to file src/allmydata/test/test_system.py.rej +DONE 2 out of 27 hunks FAILED -- saving rejects to file src/allmydata/test/test_system.py.rej patching file src/allmydata/test/test_upload.py patching file src/allmydata/test/test_util.py patching file src/allmydata/test/test_web.py @@ -359,7 +359,7 @@ Hunk #8 FAILED at 5035. Hunk #9 FAILED at 5048. Hunk #10 FAILED at 5139. -5 out of 10 hunks FAILED -- saving rejects to file src/allmydata/test/test_web.py.rej +DONE 5 out of 10 hunks FAILED -- saving rejects to file src/allmydata/test/test_web.py.rej patching file src/allmydata/util/deferredutil.py Hunk #1 FAILED at 1. Hunk #2 FAILED at 33. diff '--exclude=_version.py' '--exclude=*.rej' '--exclude=.git' -u -r merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/common.py merge-leasedb-and-cloud-backend/src/allmydata/test/common.py --- merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/common.py 2012-11-22 17:48:58.000000000 -0700 +++ merge-leasedb-and-cloud-backend/src/allmydata/test/common.py 2012-11-27 09:08:41.130308867 -0700 @@ -17,7 +17,7 @@ from allmydata.storage_client import StubServer from allmydata.mutable.layout import unpack_header from allmydata.mutable.publish import MutableData -from allmydata.storage.backends.disk.mutable import MutableShareFile +from allmydata.storage.backends.disk.mutable import MutableDiskShare from allmydata.util import hashutil, log, fileutil, pollmixin from allmydata.util.assertutil import precondition from allmydata.util.consumer import download_to_data diff '--exclude=_version.py' '--exclude=*.rej' '--exclude=.git' -u -r merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/test_crawler.py merge-leasedb-and-cloud-backend/src/allmydata/test/test_crawler.py --- merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/test_crawler.py 2012-11-22 17:48:58.000000000 -0700 +++ merge-leasedb-and-cloud-backend/src/allmydata/test/test_crawler.py 2012-11-27 09:20:44.330369820 -0700 @@ -87,13 +87,19 @@ def write(self, i, aa, serverid, tail=0): si = self.si(i) si = si[:-1] + chr(tail) - had,made = aa.remote_allocate_buckets(si, - self.rs(i, serverid), - self.cs(i, serverid), - set([0]), 99, FakeCanary()) - made[0].remote_write(0, "data") - made[0].remote_close() - return si_b2a(si) + d = defer.succeed(None) + d.addCallback(lambda ign: aa.remote_allocate_buckets(si, + self.rs(i, serverid), + self.cs(i, serverid), + set([0]), 99, FakeCanary())) + def _allocated( (had, made) ): + d2 = defer.succeed(None) + d2.addCallback(lambda ign: made[0].remote_write(0, "data")) + d2.addCallback(lambda ign: made[0].remote_close()) + d2.addCallback(lambda ign: si_b2a(si)) + return d2 + d.addCallback(_allocated) + return d def test_service(self): server = self.create("crawler/Basic/service") diff '--exclude=_version.py' '--exclude=*.rej' '--exclude=.git' -u -r merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/test_hung_server.py merge-leasedb-and-cloud-backend/src/allmydata/test/test_hung_server.py --- merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/test_hung_server.py 2012-11-22 17:48:58.000000000 -0700 +++ merge-leasedb-and-cloud-backend/src/allmydata/test/test_hung_server.py 2012-11-27 08:48:21.060204651 -0700 @@ -42,53 +42,35 @@ def _hang_shares(self, shnums, **kwargs): # hang all servers who are holding the given shares hung_serverids = set() - for (i_shnum, i_serverid, i_sharefile) in self.shares: + for (i_shnum, i_serverid, i_sharefp) in self.shares: if i_shnum in shnums: if i_serverid not in hung_serverids: self.g.hang_server(i_serverid, **kwargs) hung_serverids.add(i_serverid) def _delete_all_shares_from(self, servers): - serverids = [id for (id, ss) in servers] - for (i_shnum, i_serverid, i_sharefile) in self.shares: + serverids = [ss.original.get_serverid() for ss in servers] + for (i_shnum, i_serverid, i_sharefp) in self.shares: if i_serverid in serverids: - os.unlink(i_sharefile) + i_sharefp.remove() def _corrupt_all_shares_in(self, servers, corruptor_func): - serverids = [id for (id, ss) in servers] - for (i_shnum, i_serverid, i_sharefile) in self.shares: + serverids = [ss.original.get_serverid() for ss in servers] + for (i_shnum, i_serverid, i_sharefp) in self.shares: if i_serverid in serverids: - self._corrupt_share((i_shnum, i_sharefile), corruptor_func) + self.corrupt_share((i_shnum, i_serverid, i_sharefp), corruptor_func) def _copy_all_shares_from(self, from_servers, to_server): - serverids = [id for (id, ss) in from_servers] - for (i_shnum, i_serverid, i_sharefile) in self.shares: + serverids = [ss.original.get_serverid() for ss in from_servers] + for (i_shnum, i_serverid, i_sharefp) in self.shares: if i_serverid in serverids: - self._copy_share((i_shnum, i_sharefile), to_server) + self.copy_share((i_shnum, i_serverid, i_sharefp), self.uri, to_server.original) - def _copy_share(self, share, to_server): - (sharenum, sharefile) = share - (id, ss) = to_server - original_server = ss.original.server - shares_dir = os.path.join(original_server.storedir, "shares") - si = uri.from_string(self.uri).get_storage_index() - si_dir = os.path.join(shares_dir, storage_index_to_dir(si)) - if not os.path.exists(si_dir): - os.makedirs(si_dir) - new_sharefile = os.path.join(si_dir, str(sharenum)) - shutil.copy(sharefile, new_sharefile) - self.shares = self.find_uri_shares(self.uri) - # Make sure that the storage server has the share. - self.failUnlessIn((sharenum, original_server.get_nodeid(), new_sharefile), self.shares) - - def _corrupt_share(self, share, corruptor_func): - (sharenum, sharefile) = share - data = open(sharefile, "rb").read() - newdata = corruptor_func(data) - os.unlink(sharefile) - wf = open(sharefile, "wb") - wf.write(newdata) - wf.close() + d = self.find_uri_shares(self.uri) + def _got_shares(shares): + self.shares = shares + d.addCallback(_got_shares) + return d def _set_up(self, mutable, testdir, num_clients=1, num_servers=10): self.mutable = mutable @@ -157,7 +139,7 @@ self._download_and_check) else: return self.shouldFail(NotEnoughSharesError, self.basedir, - "ran out of shares", + "ran out of shares", self._download_and_check) diff '--exclude=_version.py' '--exclude=*.rej' '--exclude=.git' -u -r merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/test_repairer.py merge-leasedb-and-cloud-backend/src/allmydata/test/test_repairer.py --- merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/test_repairer.py 2012-11-22 17:48:58.000000000 -0700 +++ merge-leasedb-and-cloud-backend/src/allmydata/test/test_repairer.py 2012-11-26 22:42:00.454524942 -0700 @@ -704,7 +704,7 @@ # Cause one of the servers to not respond during the pre-repair # filecheck, but then *do* respond to the post-repair filecheck. ss = self.g.servers_by_number[0] - self.g.break_server(ss.get_nodeid(), count=1) + self.g.break_server(ss.get_serverid(), count=1) shares = self.find_uri_shares(self.uri) self.failUnlessEqual(len(shares), 10) diff '--exclude=_version.py' '--exclude=*.rej' '--exclude=.git' -u -r merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/test_system.py merge-leasedb-and-cloud-backend/src/allmydata/test/test_system.py --- merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/test_system.py 2012-11-22 17:48:58.000000000 -0700 +++ merge-leasedb-and-cloud-backend/src/allmydata/test/test_system.py 2012-11-27 07:38:25.823004955 -0700 @@ -7,12 +7,14 @@ import allmydata from allmydata import uri -from allmydata.storage.backends.disk.mutable import MutableShareFile +from allmydata.storage.backends.disk.disk_backend import DiskBackend +from allmydata.storage.backends.disk.mutable import load_mutable_disk_share +from allmydata.storage.backends.cloud import cloud_common, mock_cloud from allmydata.storage.server import si_a2b from allmydata.immutable import offloaded, upload from allmydata.immutable.literal import LiteralFileNode from allmydata.immutable.filenode import ImmutableFileNode -from allmydata.util import idlib, mathutil +from allmydata.util import idlib, mathutil, fileutil from allmydata.util import log, base32 from allmydata.util.verlib import NormalizedVersion from allmydata.util.encodingutil import quote_output, unicode_to_argv, get_filesystem_encoding @@ -431,7 +433,7 @@ def _corrupt_mutable_share(self, ign, what, which): (storageindex, filename, shnum) = what d = defer.succeed(None) - d.addCallback(lambda ign: load_mutable_disk_share(FilePath(filename), storageindex, shnum)) + d.addCallback(lambda ign: load_mutable_disk_share(filename, storageindex, shnum)) def _got_share(msf): d2 = msf.readv([ (0, 1000000) ]) def _got_data(datav): @@ -516,23 +518,27 @@ filename], stdout=out, stderr=err) output = out.getvalue() + self.failUnlessEqual(err.getvalue(), "") self.failUnlessEqual(rc, 0) try: - self.failUnless("Mutable slot found:\n" in output) - self.failUnless("share_type: SDMF\n" in output) + self.failUnlessIn("Mutable slot found:\n", output) + self.failUnlessIn("share_type: SDMF\n", output) peerid = idlib.nodeid_b2a(self.clients[client_num].nodeid) - self.failUnless(" WE for nodeid: %s\n" % peerid in output) - self.failUnless(" SDMF contents:\n" in output) - self.failUnless(" seqnum: 1\n" in output) - self.failUnless(" required_shares: 3\n" in output) - self.failUnless(" total_shares: 10\n" in output) - self.failUnless(" segsize: 27\n" in output, (output, filename)) - self.failUnless(" datalen: 25\n" in output) + self.failUnlessIn(" WE for nodeid: %s\n" % peerid, output) + self.failUnlessIn(" num_extra_leases: 0\n", output) + if isinstance(self.clients[client_num], DiskBackend): + self.failUnlessIn(" secrets are for nodeid: %s\n" % peerid, output) + self.failUnlessIn(" SDMF contents:\n", output) + self.failUnlessIn(" seqnum: 1\n", output) + self.failUnlessIn(" required_shares: 3\n", output) + self.failUnlessIn(" total_shares: 10\n", output) + self.failUnlessIn(" segsize: 27\n", output) + self.failUnlessIn(" datalen: 25\n", output) # the exact share_hash_chain nodes depends upon the sharenum, # and is more of a hassle to compute than I want to deal with # now - self.failUnless(" share_hash_chain: " in output) - self.failUnless(" block_hash_tree: 1 nodes\n" in output) + self.failUnlessIn(" share_hash_chain: ", output) + self.failUnlessIn(" block_hash_tree: 1 nodes\n", output) expected = (" verify-cap: URI:SSK-Verifier:%s:" % base32.b2a(storage_index)) self.failUnless(expected in output) diff '--exclude=_version.py' '--exclude=*.rej' '--exclude=.git' -u -r merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/test_web.py merge-leasedb-and-cloud-backend/src/allmydata/test/test_web.py --- merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/test_web.py 2012-11-22 17:48:58.000000000 -0700 +++ merge-leasedb-and-cloud-backend/src/allmydata/test/test_web.py 2012-11-27 09:04:35.290287927 -0700 @@ -5029,18 +5029,15 @@ d.addErrback(self.explain_web_error) return d - def _assert_leasecount(self, ignored, which, expected): - u = self.uris[which] - si = uri.from_string(u).get_storage_index() - num_leases = 0 - for server in self.g.servers_by_number.values(): - ss = server.get_accountant().get_anonymous_account() - ss2 = server.get_accountant().get_starter_account() - num_leases += len(ss.get_leases(si)) + len(ss2.get_leases(si)) - - if num_leases != expected: - self.fail("expected %d leases, have %d, on '%s'" % - (expected, num_leases, which)) + def _assert_leasecount(self, which, expected): + d = self.count_leases(self.uris[which]) + def _got_counts(lease_counts): + for (fn, num_leases) in lease_counts: + if num_leases != expected: + self.fail("expected %d leases, have %d, on %s" % + (expected, num_leases, fn)) + d.addCallback(_got_counts) + return d def test_add_lease(self): self.basedir = "web/Grid/add_lease" @@ -5070,9 +5067,9 @@ self.fileurls[which] = "uri/" + urllib.quote(self.uris[which]) d.addCallback(_compute_fileurls) - d.addCallback(self._assert_leasecount, "one", N) - d.addCallback(self._assert_leasecount, "two", N) - d.addCallback(self._assert_leasecount, "mutable", N) + d.addCallback(lambda ign: self._assert_leasecount("one", N)) + d.addCallback(lambda ign: self._assert_leasecount("two", N)) + d.addCallback(lambda ign: self._assert_leasecount("mutable", N)) d.addCallback(self.CHECK, "one", "t=check") # no add-lease def _got_html_good(res): @@ -5080,23 +5077,25 @@ self.failIfIn("Not Healthy", res) d.addCallback(_got_html_good) - d.addCallback(self._assert_leasecount, "one", N) - d.addCallback(self._assert_leasecount, "two", N) - d.addCallback(self._assert_leasecount, "mutable", N) + d.addCallback(lambda ign: self._assert_leasecount("one", N)) + d.addCallback(lambda ign: self._assert_leasecount("two", N)) + d.addCallback(lambda ign: self._assert_leasecount("mutable", N)) # this CHECK uses the original client, which uses the same # lease-secrets, so it will just renew the original lease d.addCallback(self.CHECK, "one", "t=check&add-lease=true") d.addCallback(_got_html_good) - d.addCallback(self._assert_leasecount, "one", N) - d.addCallback(self._assert_leasecount, "two", N) - d.addCallback(self._assert_leasecount, "mutable", N) + d.addCallback(lambda ign: self._assert_leasecount("one", N)) + d.addCallback(lambda ign: self._assert_leasecount("two", N)) + d.addCallback(lambda ign: self._assert_leasecount("mutable", N)) # this CHECK uses an alternate client, which adds a second lease d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1) d.addCallback(_got_html_good) + # XXX why are the checks below commented out? --Zooko 2012-11-27 + #d.addCallback(self._assert_leasecount, "one", 2*N) #d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true") @@ -5151,9 +5150,9 @@ self.failUnlessReallyEqual(len(units), 4+1) d.addCallback(_done) - d.addCallback(self._assert_leasecount, "root", N) - d.addCallback(self._assert_leasecount, "one", N) - d.addCallback(self._assert_leasecount, "mutable", N) + d.addCallback(lambda ign: self._assert_leasecount("root", N)) + d.addCallback(lambda ign: self._assert_leasecount("one", N)) + d.addCallback(lambda ign: self._assert_leasecount("mutable", N)) d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true") d.addCallback(_done) -----------------------------6656230412292714931255239840 Content-Disposition: form-data; name="description"