Ticket #1818: merge-some-test-rejs.diff.txt

File merge-some-test-rejs.diff.txt, 18.8 KB (added by zooko, at 2012-11-27T16:29:18Z)
Line 
1diff '--exclude=_version.py' '--exclude=*.rej' '--exclude=.git' -u -r merge-leasedb-and-cloud-backend-from-ds/rejects.txt merge-leasedb-and-cloud-backend/rejects.txt
2--- merge-leasedb-and-cloud-backend-from-ds/rejects.txt 2012-11-26 21:05:06.000000000 -0700
3+++ merge-leasedb-and-cloud-backend/rejects.txt 2012-11-27 09:25:03.620391737 -0700
4@@ -206,7 +206,7 @@
5 Hunk #3 succeeded at 600 (offset 28 lines).
6 Hunk #4 succeeded at 641 (offset 28 lines).
7 Hunk #5 succeeded at 1303 (offset 28 lines).
8-1 out of 5 hunks FAILED -- saving rejects to file src/allmydata/test/common.py.rej
9+DONE 1 out of 5 hunks FAILED -- saving rejects to file src/allmydata/test/common.py.rej
10 patching file src/allmydata/test/no_network.py
11 Hunk #6 succeeded at 299 with fuzz 2.
12 Hunk #7 succeeded at 319 (offset 1 line).
13@@ -223,13 +223,13 @@
14 Hunk #3 FAILED at 58.
15 Hunk #4 FAILED at 92.
16 Hunk #5 FAILED at 336.
17-5 out of 5 hunks FAILED -- saving rejects to file src/allmydata/test/test_crawler.py.rej
18+DONE 5 out of 5 hunks FAILED -- saving rejects to file src/allmydata/test/test_crawler.py.rej
19 patching file src/allmydata/test/test_deepcheck.py
20 patching file src/allmydata/test/test_download.py
21 patching file src/allmydata/test/test_encode.py
22 patching file src/allmydata/test/test_hung_server.py
23 Hunk #4 FAILED at 49.
24-1 out of 8 hunks FAILED -- saving rejects to file src/allmydata/test/test_hung_server.py.rej
25+DONE 1 out of 8 hunks FAILED -- saving rejects to file src/allmydata/test/test_hung_server.py.rej
26 patching file src/allmydata/test/test_immutable.py
27 patching file src/allmydata/test/test_mutable.py
28 Hunk #2 succeeded at 1911 (offset 2 lines).
29@@ -243,7 +243,7 @@
30 patching file src/allmydata/test/test_node.py
31 patching file src/allmydata/test/test_repairer.py
32 Hunk #8 FAILED at 700.
33-1 out of 8 hunks FAILED -- saving rejects to file src/allmydata/test/test_repairer.py.rej
34+DONE 1 out of 8 hunks FAILED -- saving rejects to file src/allmydata/test/test_repairer.py.rej
35 patching file src/allmydata/test/test_runner.py
36 Hunk #1 succeeded at 216 (offset 2 lines).
37 patching file src/allmydata/test/test_storage.py
38@@ -343,7 +343,7 @@
39 Hunk #25 succeeded at 1844 (offset -3 lines).
40 Hunk #26 succeeded at 1887 (offset -3 lines).
41 Hunk #27 succeeded at 1935 (offset -3 lines).
42-2 out of 27 hunks FAILED -- saving rejects to file src/allmydata/test/test_system.py.rej
43+DONE 2 out of 27 hunks FAILED -- saving rejects to file src/allmydata/test/test_system.py.rej
44 patching file src/allmydata/test/test_upload.py
45 patching file src/allmydata/test/test_util.py
46 patching file src/allmydata/test/test_web.py
47@@ -359,7 +359,7 @@
48 Hunk #8 FAILED at 5035.
49 Hunk #9 FAILED at 5048.
50 Hunk #10 FAILED at 5139.
51-5 out of 10 hunks FAILED -- saving rejects to file src/allmydata/test/test_web.py.rej
52+DONE 5 out of 10 hunks FAILED -- saving rejects to file src/allmydata/test/test_web.py.rej
53 patching file src/allmydata/util/deferredutil.py
54 Hunk #1 FAILED at 1.
55 Hunk #2 FAILED at 33.
56diff '--exclude=_version.py' '--exclude=*.rej' '--exclude=.git' -u -r merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/common.py merge-leasedb-and-cloud-backend/src/allmydata/test/common.py
57--- merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/common.py        2012-11-22 17:48:58.000000000 -0700
58+++ merge-leasedb-and-cloud-backend/src/allmydata/test/common.py        2012-11-27 09:08:41.130308867 -0700
59@@ -17,7 +17,7 @@
60 from allmydata.storage_client import StubServer
61 from allmydata.mutable.layout import unpack_header
62 from allmydata.mutable.publish import MutableData
63-from allmydata.storage.backends.disk.mutable import MutableShareFile
64+from allmydata.storage.backends.disk.mutable import MutableDiskShare
65 from allmydata.util import hashutil, log, fileutil, pollmixin
66 from allmydata.util.assertutil import precondition
67 from allmydata.util.consumer import download_to_data
68diff '--exclude=_version.py' '--exclude=*.rej' '--exclude=.git' -u -r merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/test_crawler.py merge-leasedb-and-cloud-backend/src/allmydata/test/test_crawler.py
69--- merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/test_crawler.py  2012-11-22 17:48:58.000000000 -0700
70+++ merge-leasedb-and-cloud-backend/src/allmydata/test/test_crawler.py  2012-11-27 09:20:44.330369820 -0700
71@@ -87,13 +87,19 @@
72     def write(self, i, aa, serverid, tail=0):
73         si = self.si(i)
74         si = si[:-1] + chr(tail)
75-        had,made = aa.remote_allocate_buckets(si,
76-                                              self.rs(i, serverid),
77-                                              self.cs(i, serverid),
78-                                              set([0]), 99, FakeCanary())
79-        made[0].remote_write(0, "data")
80-        made[0].remote_close()
81-        return si_b2a(si)
82+        d = defer.succeed(None)
83+        d.addCallback(lambda ign: aa.remote_allocate_buckets(si,
84+                                                             self.rs(i, serverid),
85+                                                             self.cs(i, serverid),
86+                                                             set([0]), 99, FakeCanary()))
87+        def _allocated( (had, made) ):
88+            d2 = defer.succeed(None)
89+            d2.addCallback(lambda ign: made[0].remote_write(0, "data"))
90+            d2.addCallback(lambda ign: made[0].remote_close())
91+            d2.addCallback(lambda ign: si_b2a(si))
92+            return d2
93+        d.addCallback(_allocated)
94+        return d
95 
96     def test_service(self):
97         server = self.create("crawler/Basic/service")
98diff '--exclude=_version.py' '--exclude=*.rej' '--exclude=.git' -u -r merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/test_hung_server.py merge-leasedb-and-cloud-backend/src/allmydata/test/test_hung_server.py
99--- merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/test_hung_server.py      2012-11-22 17:48:58.000000000 -0700
100+++ merge-leasedb-and-cloud-backend/src/allmydata/test/test_hung_server.py      2012-11-27 08:48:21.060204651 -0700
101@@ -42,53 +42,35 @@
102     def _hang_shares(self, shnums, **kwargs):
103         # hang all servers who are holding the given shares
104         hung_serverids = set()
105-        for (i_shnum, i_serverid, i_sharefile) in self.shares:
106+        for (i_shnum, i_serverid, i_sharefp) in self.shares:
107             if i_shnum in shnums:
108                 if i_serverid not in hung_serverids:
109                     self.g.hang_server(i_serverid, **kwargs)
110                     hung_serverids.add(i_serverid)
111 
112     def _delete_all_shares_from(self, servers):
113-        serverids = [id for (id, ss) in servers]
114-        for (i_shnum, i_serverid, i_sharefile) in self.shares:
115+        serverids = [ss.original.get_serverid() for ss in servers]
116+        for (i_shnum, i_serverid, i_sharefp) in self.shares:
117             if i_serverid in serverids:
118-                os.unlink(i_sharefile)
119+                i_sharefp.remove()
120 
121     def _corrupt_all_shares_in(self, servers, corruptor_func):
122-        serverids = [id for (id, ss) in servers]
123-        for (i_shnum, i_serverid, i_sharefile) in self.shares:
124+        serverids = [ss.original.get_serverid() for ss in servers]
125+        for (i_shnum, i_serverid, i_sharefp) in self.shares:
126             if i_serverid in serverids:
127-                self._corrupt_share((i_shnum, i_sharefile), corruptor_func)
128+                self.corrupt_share((i_shnum, i_serverid, i_sharefp), corruptor_func)
129 
130     def _copy_all_shares_from(self, from_servers, to_server):
131-        serverids = [id for (id, ss) in from_servers]
132-        for (i_shnum, i_serverid, i_sharefile) in self.shares:
133+        serverids = [ss.original.get_serverid() for ss in from_servers]
134+        for (i_shnum, i_serverid, i_sharefp) in self.shares:
135             if i_serverid in serverids:
136-                self._copy_share((i_shnum, i_sharefile), to_server)
137+                self.copy_share((i_shnum, i_serverid, i_sharefp), self.uri, to_server.original)
138 
139-    def _copy_share(self, share, to_server):
140-        (sharenum, sharefile) = share
141-        (id, ss) = to_server
142-        original_server = ss.original.server
143-        shares_dir = os.path.join(original_server.storedir, "shares")
144-        si = uri.from_string(self.uri).get_storage_index()
145-        si_dir = os.path.join(shares_dir, storage_index_to_dir(si))
146-        if not os.path.exists(si_dir):
147-            os.makedirs(si_dir)
148-        new_sharefile = os.path.join(si_dir, str(sharenum))
149-        shutil.copy(sharefile, new_sharefile)
150-        self.shares = self.find_uri_shares(self.uri)
151-        # Make sure that the storage server has the share.
152-        self.failUnlessIn((sharenum, original_server.get_nodeid(), new_sharefile), self.shares)
153-
154-    def _corrupt_share(self, share, corruptor_func):
155-        (sharenum, sharefile) = share
156-        data = open(sharefile, "rb").read()
157-        newdata = corruptor_func(data)
158-        os.unlink(sharefile)
159-        wf = open(sharefile, "wb")
160-        wf.write(newdata)
161-        wf.close()
162+            d = self.find_uri_shares(self.uri)
163+            def _got_shares(shares):
164+                self.shares = shares
165+            d.addCallback(_got_shares)
166+            return d
167 
168     def _set_up(self, mutable, testdir, num_clients=1, num_servers=10):
169         self.mutable = mutable
170@@ -157,7 +139,7 @@
171                                    self._download_and_check)
172         else:
173             return self.shouldFail(NotEnoughSharesError, self.basedir,
174-                               "ran out of shares",
175+                                   "ran out of shares",
176                                    self._download_and_check)
177 
178 
179diff '--exclude=_version.py' '--exclude=*.rej' '--exclude=.git' -u -r merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/test_repairer.py merge-leasedb-and-cloud-backend/src/allmydata/test/test_repairer.py
180--- merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/test_repairer.py 2012-11-22 17:48:58.000000000 -0700
181+++ merge-leasedb-and-cloud-backend/src/allmydata/test/test_repairer.py 2012-11-26 22:42:00.454524942 -0700
182@@ -704,7 +704,7 @@
183             # Cause one of the servers to not respond during the pre-repair
184             # filecheck, but then *do* respond to the post-repair filecheck.
185             ss = self.g.servers_by_number[0]
186-            self.g.break_server(ss.get_nodeid(), count=1)
187+            self.g.break_server(ss.get_serverid(), count=1)
188 
189             shares = self.find_uri_shares(self.uri)
190             self.failUnlessEqual(len(shares), 10)
191diff '--exclude=_version.py' '--exclude=*.rej' '--exclude=.git' -u -r merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/test_system.py merge-leasedb-and-cloud-backend/src/allmydata/test/test_system.py
192--- merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/test_system.py   2012-11-22 17:48:58.000000000 -0700
193+++ merge-leasedb-and-cloud-backend/src/allmydata/test/test_system.py   2012-11-27 07:38:25.823004955 -0700
194@@ -7,12 +7,14 @@
195 
196 import allmydata
197 from allmydata import uri
198-from allmydata.storage.backends.disk.mutable import MutableShareFile
199+from allmydata.storage.backends.disk.disk_backend import DiskBackend
200+from allmydata.storage.backends.disk.mutable import load_mutable_disk_share
201+from allmydata.storage.backends.cloud import cloud_common, mock_cloud
202 from allmydata.storage.server import si_a2b
203 from allmydata.immutable import offloaded, upload
204 from allmydata.immutable.literal import LiteralFileNode
205 from allmydata.immutable.filenode import ImmutableFileNode
206-from allmydata.util import idlib, mathutil
207+from allmydata.util import idlib, mathutil, fileutil
208 from allmydata.util import log, base32
209 from allmydata.util.verlib import NormalizedVersion
210 from allmydata.util.encodingutil import quote_output, unicode_to_argv, get_filesystem_encoding
211@@ -431,7 +433,7 @@
212     def _corrupt_mutable_share(self, ign, what, which):
213         (storageindex, filename, shnum) = what
214         d = defer.succeed(None)
215-        d.addCallback(lambda ign: load_mutable_disk_share(FilePath(filename), storageindex, shnum))
216+        d.addCallback(lambda ign: load_mutable_disk_share(filename, storageindex, shnum))
217         def _got_share(msf):
218             d2 = msf.readv([ (0, 1000000) ])
219             def _got_data(datav):
220@@ -516,23 +518,27 @@
221                                 filename],
222                                stdout=out, stderr=err)
223             output = out.getvalue()
224+            self.failUnlessEqual(err.getvalue(), "")
225             self.failUnlessEqual(rc, 0)
226             try:
227-                self.failUnless("Mutable slot found:\n" in output)
228-                self.failUnless("share_type: SDMF\n" in output)
229+                self.failUnlessIn("Mutable slot found:\n", output)
230+                self.failUnlessIn("share_type: SDMF\n", output)
231                 peerid = idlib.nodeid_b2a(self.clients[client_num].nodeid)
232-                self.failUnless(" WE for nodeid: %s\n" % peerid in output)
233-                self.failUnless(" SDMF contents:\n" in output)
234-                self.failUnless("  seqnum: 1\n" in output)
235-                self.failUnless("  required_shares: 3\n" in output)
236-                self.failUnless("  total_shares: 10\n" in output)
237-                self.failUnless("  segsize: 27\n" in output, (output, filename))
238-                self.failUnless("  datalen: 25\n" in output)
239+                self.failUnlessIn(" WE for nodeid: %s\n" % peerid, output)
240+                self.failUnlessIn(" num_extra_leases: 0\n", output)
241+                if isinstance(self.clients[client_num], DiskBackend):
242+                    self.failUnlessIn("  secrets are for nodeid: %s\n" % peerid, output)
243+                self.failUnlessIn(" SDMF contents:\n", output)
244+                self.failUnlessIn("  seqnum: 1\n", output)
245+                self.failUnlessIn("  required_shares: 3\n", output)
246+                self.failUnlessIn("  total_shares: 10\n", output)
247+                self.failUnlessIn("  segsize: 27\n", output)
248+                self.failUnlessIn("  datalen: 25\n", output)
249                 # the exact share_hash_chain nodes depends upon the sharenum,
250                 # and is more of a hassle to compute than I want to deal with
251                 # now
252-                self.failUnless("  share_hash_chain: " in output)
253-                self.failUnless("  block_hash_tree: 1 nodes\n" in output)
254+                self.failUnlessIn("  share_hash_chain: ", output)
255+                self.failUnlessIn("  block_hash_tree: 1 nodes\n", output)
256                 expected = ("  verify-cap: URI:SSK-Verifier:%s:" %
257                             base32.b2a(storage_index))
258                 self.failUnless(expected in output)
259diff '--exclude=_version.py' '--exclude=*.rej' '--exclude=.git' -u -r merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/test_web.py merge-leasedb-and-cloud-backend/src/allmydata/test/test_web.py
260--- merge-leasedb-and-cloud-backend-from-ds/src/allmydata/test/test_web.py      2012-11-22 17:48:58.000000000 -0700
261+++ merge-leasedb-and-cloud-backend/src/allmydata/test/test_web.py      2012-11-27 09:04:35.290287927 -0700
262@@ -5029,18 +5029,15 @@
263         d.addErrback(self.explain_web_error)
264         return d
265 
266-    def _assert_leasecount(self, ignored, which, expected):
267-        u = self.uris[which]
268-        si = uri.from_string(u).get_storage_index()
269-        num_leases = 0
270-        for server in self.g.servers_by_number.values():
271-            ss = server.get_accountant().get_anonymous_account()
272-            ss2 = server.get_accountant().get_starter_account()
273-            num_leases += len(ss.get_leases(si)) + len(ss2.get_leases(si))
274-
275-        if num_leases != expected:
276-            self.fail("expected %d leases, have %d, on '%s'" %
277-                      (expected, num_leases, which))
278+    def _assert_leasecount(self, which, expected):
279+        d = self.count_leases(self.uris[which])
280+        def _got_counts(lease_counts):
281+            for (fn, num_leases) in lease_counts:
282+                if num_leases != expected:
283+                    self.fail("expected %d leases, have %d, on %s" %
284+                              (expected, num_leases, fn))
285+        d.addCallback(_got_counts)
286+        return d
287 
288     def test_add_lease(self):
289         self.basedir = "web/Grid/add_lease"
290@@ -5070,9 +5067,9 @@
291                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
292         d.addCallback(_compute_fileurls)
293 
294-        d.addCallback(self._assert_leasecount, "one", N)
295-        d.addCallback(self._assert_leasecount, "two", N)
296-        d.addCallback(self._assert_leasecount, "mutable", N)
297+        d.addCallback(lambda ign: self._assert_leasecount("one", N))
298+        d.addCallback(lambda ign: self._assert_leasecount("two", N))
299+        d.addCallback(lambda ign: self._assert_leasecount("mutable", N))
300 
301         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
302         def _got_html_good(res):
303@@ -5080,23 +5077,25 @@
304             self.failIfIn("Not Healthy", res)
305         d.addCallback(_got_html_good)
306 
307-        d.addCallback(self._assert_leasecount, "one", N)
308-        d.addCallback(self._assert_leasecount, "two", N)
309-        d.addCallback(self._assert_leasecount, "mutable", N)
310+        d.addCallback(lambda ign: self._assert_leasecount("one", N))
311+        d.addCallback(lambda ign: self._assert_leasecount("two", N))
312+        d.addCallback(lambda ign: self._assert_leasecount("mutable", N))
313 
314         # this CHECK uses the original client, which uses the same
315         # lease-secrets, so it will just renew the original lease
316         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
317         d.addCallback(_got_html_good)
318 
319-        d.addCallback(self._assert_leasecount, "one", N)
320-        d.addCallback(self._assert_leasecount, "two", N)
321-        d.addCallback(self._assert_leasecount, "mutable", N)
322+        d.addCallback(lambda ign: self._assert_leasecount("one", N))
323+        d.addCallback(lambda ign: self._assert_leasecount("two", N))
324+        d.addCallback(lambda ign: self._assert_leasecount("mutable", N))
325 
326         # this CHECK uses an alternate client, which adds a second lease
327         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
328         d.addCallback(_got_html_good)
329 
330+        # XXX why are the checks below commented out? --Zooko 2012-11-27
331+
332         #d.addCallback(self._assert_leasecount, "one", 2*N)
333 
334         #d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
335@@ -5151,9 +5150,9 @@
336             self.failUnlessReallyEqual(len(units), 4+1)
337         d.addCallback(_done)
338 
339-        d.addCallback(self._assert_leasecount, "root", N)
340-        d.addCallback(self._assert_leasecount, "one", N)
341-        d.addCallback(self._assert_leasecount, "mutable", N)
342+        d.addCallback(lambda ign: self._assert_leasecount("root", N))
343+        d.addCallback(lambda ign: self._assert_leasecount("one", N))
344+        d.addCallback(lambda ign: self._assert_leasecount("mutable", N))
345 
346         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
347         d.addCallback(_done)
348
349-----------------------------6656230412292714931255239840
350Content-Disposition: form-data; name="description"
351