Ticket #1556: 1556-fix-miscaptures.darcs.patch

File 1556-fix-miscaptures.darcs.patch, 15.5 KB (added by davidsarah, at 2011-10-07T03:49:42Z)

Fix some potential bugs exposed by check-miscaptures.py. (Version 3.)

Line 
14 patches for repository http://tahoe-lafs.org/source/tahoe/trunk:
2
3Fri Oct  7 03:30:01 BST 2011  david-sarah@jacaranda.org
4  * misc/simulators/hashbasedsig.py: simplify by removing unnecessary local function that captured a variable declared in a for loop (this was not a bug, but the code was unclear). Also fix a pyflakes warning about an import. refs #1556
5
6Fri Oct  7 03:34:43 BST 2011  david-sarah@jacaranda.org
7  * Fix some potential bugs in test code exposed by check-miscaptures.py. refs #1556
8
9Fri Oct  7 04:24:44 BST 2011  david-sarah@jacaranda.org
10  * Fix some potential bugs (in non-test code) exposed by check-miscaptures.py. refs #1556
11
12Fri Oct  7 04:38:47 BST 2011  david-sarah@jacaranda.org
13  * Fix some more potential bugs in test code exposed by check-miscaptures.py. refs #1556
14
15New patches:
16
17[misc/simulators/hashbasedsig.py: simplify by removing unnecessary local function that captured a variable declared in a for loop (this was not a bug, but the code was unclear). Also fix a pyflakes warning about an import. refs #1556
18david-sarah@jacaranda.org**20111007023001
19 Ignore-this: 446c94efae02ded5e85eb3335ca5e69
20] {
21hunk ./misc/simulators/hashbasedsig.py 22
22 Mcycles_per_block = cycles_per_byte * L_block / (8 * 1000000.0)
23 
24 
25-from math import floor, ceil, log, log1p, pow, e, sqrt
26+from math import floor, ceil, log, log1p, pow, e
27 from sys import stderr
28 from gc import collect
29 
30hunk ./misc/simulators/hashbasedsig.py 141
31         for x in xrange(1, j):
32             lg_px[x] = lg_px[x-1] - lg(x) + lg_px_step
33 
34-        def find_min_q():
35-            for q in xrange(1, q_max+1):
36-                lg_q = lg(q)
37-                lg_pforge = [lg_px[x] + (lg_q*x - lg_K2)*q for x in xrange(1, j)]
38-                if max(lg_pforge) < -L_hash + lg(j) and lg_px[j-1] + 1.0 < -L_hash:
39-                    #print "K = %d, K1 = %d, K2 = %d, L_hash = %d, lg_K2 = %.3f, q = %d, lg_pforge_1 = %.3f, lg_pforge_2 = %.3f, lg_pforge_3 = %.3f" \
40-                    #      % (K, K1, K2, L_hash, lg_K2, q, lg_pforge_1, lg_pforge_2, lg_pforge_3)
41-                    return q
42-            return None
43+        q = None
44+        # Find the minimum acceptable value of q.
45+        for q_cand in xrange(1, q_max+1):
46+            lg_q = lg(q_cand)
47+            lg_pforge = [lg_px[x] + (lg_q*x - lg_K2)*q_cand for x in xrange(1, j)]
48+            if max(lg_pforge) < -L_hash + lg(j) and lg_px[j-1] + 1.0 < -L_hash:
49+                #print "K = %d, K1 = %d, K2 = %d, L_hash = %d, lg_K2 = %.3f, q = %d, lg_pforge_1 = %.3f, lg_pforge_2 = %.3f, lg_pforge_3 = %.3f" \
50+                #      % (K, K1, K2, L_hash, lg_K2, q, lg_pforge_1, lg_pforge_2, lg_pforge_3)
51+                q = q_cand
52+                break
53 
54hunk ./misc/simulators/hashbasedsig.py 152
55-        q = find_min_q()
56         if q is None or q == last_q:
57             # if q hasn't decreased, this will be strictly worse than the previous candidate
58             continue
59}
60[Fix some potential bugs in test code exposed by check-miscaptures.py. refs #1556
61david-sarah@jacaranda.org**20111007023443
62 Ignore-this: e48b2c2d200521d6f28c737994ce3a2a
63] {
64hunk ./src/allmydata/test/test_cli.py 760
65             self.failUnlessIn("cannot contain", stderr)
66 
67         for invalid in ['foo:bar', 'foo bar', 'foobar::']:
68-            d.addCallback(lambda res: self.do_cli("create-alias", invalid))
69+            d.addCallback(lambda res, invalid=invalid: self.do_cli("create-alias", invalid))
70             d.addCallback(_check_invalid)
71hunk ./src/allmydata/test/test_cli.py 762
72-            d.addCallback(lambda res: self.do_cli("add-alias", invalid, self.two_uri))
73+            d.addCallback(lambda res, invalid=invalid: self.do_cli("add-alias", invalid, self.two_uri))
74             d.addCallback(_check_invalid)
75 
76         def _test_urls(junk):
77hunk ./src/allmydata/test/test_dirnode.py 1713
78             n.raise_error()
79 
80         for (i, n) in unknown_rw:
81-            self.failUnlessRaises(MustNotBeUnknownRWError, lambda: n.raise_error())
82+            self.failUnlessRaises(MustNotBeUnknownRWError, lambda n=n: n.raise_error())
83 
84         for (i, n) in must_be_ro:
85hunk ./src/allmydata/test/test_dirnode.py 1716
86-            self.failUnlessRaises(MustBeReadonlyError, lambda: n.raise_error())
87+            self.failUnlessRaises(MustBeReadonlyError, lambda n=n: n.raise_error())
88 
89         for (i, n) in must_be_imm:
90hunk ./src/allmydata/test/test_dirnode.py 1719
91-            self.failUnlessRaises(MustBeDeepImmutableError, lambda: n.raise_error())
92+            self.failUnlessRaises(MustBeDeepImmutableError, lambda n=n: n.raise_error())
93 
94         for (i, n) in bad_uri:
95hunk ./src/allmydata/test/test_dirnode.py 1722
96-            self.failUnlessRaises(uri.BadURIError, lambda: n.raise_error())
97+            self.failUnlessRaises(uri.BadURIError, lambda n=n: n.raise_error())
98 
99         for (i, n) in ok:
100             self.failIf(n.get_readonly_uri() is None, i)
101hunk ./src/allmydata/test/test_download.py 1118
102             for i,which,substring in corrupt_me:
103                 # All these tests result in a failed download.
104                 d.addCallback(self._corrupt_flip_all, imm_uri, i)
105-                d.addCallback(lambda ign:
106+                d.addCallback(lambda ign, which=which, substring=substring:
107                               self.shouldFail(NoSharesError, which,
108                                               substring,
109                                               _download, imm_uri))
110}
111[Fix some potential bugs (in non-test code) exposed by check-miscaptures.py. refs #1556
112david-sarah@jacaranda.org**20111007032444
113 Ignore-this: bac9ed65b21c2136c4db2482b3c093f7
114] {
115hunk ./src/allmydata/mutable/filenode.py 1139
116         start_segments = {} # shnum -> start segment
117         end_segments = {} # shnum -> end segment
118         blockhashes = {} # shnum -> blockhash tree
119-        for (shnum, data) in update_data.iteritems():
120-            data = [d[1] for d in data if d[0] == self._version]
121+        for (shnum, original_data) in update_data.iteritems():
122+            data = [d[1] for d in original_data if d[0] == self._version]
123 
124             # Every data entry in our list should now be share shnum for
125             # a particular version of the mutable file, so all of the
126hunk ./src/allmydata/mutable/filenode.py 1146
127             # entries should be identical.
128             datum = data[0]
129-            assert filter(lambda x: x != datum, data) == []
130+            assert [x for x in data if x != datum] == []
131 
132             blockhashes[shnum] = datum[0]
133             start_segments[shnum] = datum[1]
134hunk ./src/allmydata/mutable/servermap.py 683
135                 d.addCallback(lambda results, shnum=shnum, peerid=peerid:
136                     self._try_to_set_pubkey(results, peerid, shnum, lp))
137                 # XXX: Make self._pubkey_query_failed?
138-                d.addErrback(lambda error, shnum=shnum, peerid=peerid:
139+                d.addErrback(lambda error, shnum=shnum, peerid=peerid, data=data:
140                     self._got_corrupt_share(error, shnum, peerid, data, lp))
141             else:
142                 # we already have the public key.
143hunk ./src/allmydata/mutable/servermap.py 698
144             #   bytes of the share on the storage server, so we
145             #   shouldn't need to fetch anything at this step.
146             d2 = reader.get_verinfo()
147-            d2.addErrback(lambda error, shnum=shnum, peerid=peerid:
148+            d2.addErrback(lambda error, shnum=shnum, peerid=peerid, data=data:
149                 self._got_corrupt_share(error, shnum, peerid, data, lp))
150             # - Next, we need the signature. For an SDMF share, it is
151             #   likely that we fetched this when doing our initial fetch
152hunk ./src/allmydata/mutable/servermap.py 706
153             #   the end of the share, so unless the file is quite small,
154             #   we'll need to do a remote fetch to get it.
155             d3 = reader.get_signature()
156-            d3.addErrback(lambda error, shnum=shnum, peerid=peerid:
157+            d3.addErrback(lambda error, shnum=shnum, peerid=peerid, data=data:
158                 self._got_corrupt_share(error, shnum, peerid, data, lp))
159             #  Once we have all three of these responses, we can move on
160             #  to validating the signature
161hunk ./src/allmydata/mutable/servermap.py 717
162                 d4 = reader.get_encprivkey()
163                 d4.addCallback(lambda results, shnum=shnum, peerid=peerid:
164                     self._try_to_validate_privkey(results, peerid, shnum, lp))
165-                d4.addErrback(lambda error, shnum=shnum, peerid=peerid:
166+                d4.addErrback(lambda error, shnum=shnum, peerid=peerid, data=data:
167                     self._privkey_query_failed(error, shnum, data, lp))
168             else:
169                 d4 = defer.succeed(None)
170}
171[Fix some more potential bugs in test code exposed by check-miscaptures.py. refs #1556
172david-sarah@jacaranda.org**20111007033847
173 Ignore-this: aec8a543e9b5c3563b60692c647439a8
174] {
175hunk ./src/allmydata/test/test_hung_server.py 165
176     def test_10_good_sanity_check(self):
177         d = defer.succeed(None)
178         for mutable in [False, True]:
179-            d.addCallback(lambda ign: self._set_up(mutable, "test_10_good_sanity_check"))
180+            d.addCallback(lambda ign, mutable=mutable: self._set_up(mutable, "test_10_good_sanity_check"))
181             d.addCallback(lambda ign: self._download_and_check())
182         return d
183 
184hunk ./src/allmydata/test/test_hung_server.py 172
185     def test_10_good_copied_share(self):
186         d = defer.succeed(None)
187         for mutable in [False, True]:
188-            d.addCallback(lambda ign: self._set_up(mutable, "test_10_good_copied_share"))
189+            d.addCallback(lambda ign, mutable=mutable: self._set_up(mutable, "test_10_good_copied_share"))
190             d.addCallback(lambda ign: self._copy_all_shares_from(self.servers[2:3], self.servers[0]))
191             d.addCallback(lambda ign: self._download_and_check())
192             return d
193hunk ./src/allmydata/test/test_hung_server.py 180
194     def test_3_good_7_noshares(self):
195         d = defer.succeed(None)
196         for mutable in [False, True]:
197-            d.addCallback(lambda ign: self._set_up(mutable, "test_3_good_7_noshares"))
198+            d.addCallback(lambda ign, mutable=mutable: self._set_up(mutable, "test_3_good_7_noshares"))
199             d.addCallback(lambda ign: self._delete_all_shares_from(self.servers[3:]))
200             d.addCallback(lambda ign: self._download_and_check())
201         return d
202hunk ./src/allmydata/test/test_hung_server.py 188
203     def test_2_good_8_broken_fail(self):
204         d = defer.succeed(None)
205         for mutable in [False, True]:
206-            d.addCallback(lambda ign: self._set_up(mutable, "test_2_good_8_broken_fail"))
207+            d.addCallback(lambda ign, mutable=mutable: self._set_up(mutable, "test_2_good_8_broken_fail"))
208             d.addCallback(lambda ign: self._break(self.servers[2:]))
209             d.addCallback(lambda ign: self._should_fail_download())
210         return d
211hunk ./src/allmydata/test/test_hung_server.py 196
212     def test_2_good_8_noshares_fail(self):
213         d = defer.succeed(None)
214         for mutable in [False, True]:
215-            d.addCallback(lambda ign: self._set_up(mutable, "test_2_good_8_noshares_fail"))
216+            d.addCallback(lambda ign, mutable=mutable: self._set_up(mutable, "test_2_good_8_noshares_fail"))
217             d.addCallback(lambda ign: self._delete_all_shares_from(self.servers[2:]))
218             d.addCallback(lambda ign: self._should_fail_download())
219         return d
220hunk ./src/allmydata/test/test_hung_server.py 204
221     def test_2_good_8_broken_copied_share(self):
222         d = defer.succeed(None)
223         for mutable in [False, True]:
224-            d.addCallback(lambda ign: self._set_up(mutable, "test_2_good_8_broken_copied_share"))
225+            d.addCallback(lambda ign, mutable=mutable: self._set_up(mutable, "test_2_good_8_broken_copied_share"))
226             d.addCallback(lambda ign: self._copy_all_shares_from(self.servers[2:3], self.servers[0]))
227             d.addCallback(lambda ign: self._break(self.servers[2:]))
228             d.addCallback(lambda ign: self._download_and_check())
229hunk ./src/allmydata/test/test_hung_server.py 213
230     def test_2_good_8_broken_duplicate_share_fail(self):
231         d = defer.succeed(None)
232         for mutable in [False, True]:
233-            d.addCallback(lambda ign: self._set_up(mutable, "test_2_good_8_broken_duplicate_share_fail"))
234+            d.addCallback(lambda ign, mutable=mutable: self._set_up(mutable, "test_2_good_8_broken_duplicate_share_fail"))
235             d.addCallback(lambda ign: self._copy_all_shares_from(self.servers[1:2], self.servers[0]))
236             d.addCallback(lambda ign: self._break(self.servers[2:]))
237             d.addCallback(lambda ign: self._should_fail_download())
238hunk ./src/allmydata/test/test_mutable.py 193
239             reader = MDMFSlotReadProxy(None, None, shnum, data)
240             # We need to get the offsets for the next part.
241             d = reader.get_verinfo()
242-            def _do_corruption(verinfo, data, shnum):
243+            def _do_corruption(verinfo, data, shnum, shares):
244                 (seqnum,
245                  root_hash,
246                  IV,
247hunk ./src/allmydata/test/test_mutable.py 218
248                 else:
249                     f = flip_bit
250                 shares[shnum] = f(data, real_offset)
251-            d.addCallback(_do_corruption, data, shnum)
252+            d.addCallback(_do_corruption, data, shnum, shares)
253             ds.append(d)
254     dl = defer.DeferredList(ds)
255     dl.addCallback(lambda ignored: res)
256hunk ./src/allmydata/test/test_mutable.py 286
257         self.nodemaker.default_encoding_parameters['n'] = 1
258         d = defer.succeed(None)
259         for v in (SDMF_VERSION, MDMF_VERSION):
260-            d.addCallback(lambda ignored:
261+            d.addCallback(lambda ignored, v=v:
262                 self.nodemaker.create_mutable_file(version=v))
263             def _created(n):
264                 self.failUnless(isinstance(n, MutableFileNode))
265}
266
267Context:
268
269[no_network.py: Clean up whitespace around code changed by previous patch.
270david-sarah@jacaranda.org**20111004010407
271 Ignore-this: 647ec8a9346dca1a41212ab250619b72
272]
273[no_network.py: Fix potential bugs in some tests due to capture of slots in for loops.
274david-sarah@jacaranda.org**20111004010231
275 Ignore-this: 9c496877613a3befd54979e5de6e63d2
276]
277[docs: fix the rst formatting of COPYING.TGPPL.rst
278zooko@zooko.com**20111003043333
279 Ignore-this: c5fbc83f4a3db81a0c95b27053c463c5
280 Now it renders correctly both on trac and with rst2html --verbose from docutils v0.8.1.
281]
282[MDMF: remove extension fields from caps, tolerate arbitrary ones. Fixes #1526
283Brian Warner <warner@lothar.com>**20111001233553
284 Ignore-this: 335e1690aef1146a2c0b8d8c18c1cb21
285 
286 The filecaps used to be produced with hints for 'k' and segsize, but they
287 weren't actually used, and doing so had the potential to limit how we change
288 those filecaps in the future. Also the parsing code had some problems dealing
289 with other numbers of extensions. Removing the existing fields and making the
290 parser tolerate (and ignore) extra ones makes MDMF more future-proof.
291]
292[test/test_runner.py: BinTahoe.test_path has rare nondeterministic failures; this patch probably fixes a problem where the actual cause of failure is masked by a string conversion error.
293david-sarah@jacaranda.org**20110927225336
294 Ignore-this: 6f1ad68004194cc9cea55ace3745e4af
295]
296[docs/configuration.rst: add section about the types of node, and clarify when setting web.port enables web-API service. fixes #1444
297zooko@zooko.com**20110926203801
298 Ignore-this: ab94d470c68e720101a7ff3c207a719e
299]
300[TAG allmydata-tahoe-1.9.0a2
301warner@lothar.com**20110925234811
302 Ignore-this: e9649c58f9c9017a7d55008938dba64f
303]
304Patch bundle hash:
305058e6f84c9bbf9869cd770ce646f57166c7a9ce9