Ticket #948: test-dirnode-lit-in-deep-check.darcspatch.txt

Tue Feb 23 23:45:49 MST 2010  zooko@zooko.com
  * directories: add immutable directories to test_deepcheck.py

New patches:

[directories: add immutable directories to test_deepcheck.py
zooko@zooko.com**20100224064549
 Ignore-this: 679f2b3cbbf02d59a44a7727c6eeb16d
] {
hunk ./src/allmydata/nodemaker.py 48
         return DirectoryNode(filenode, self, self.uploader)
 
     def create_from_cap(self, writecap, readcap=None, deep_immutable=False, name=u"<unknown name>"):
+        """ The 'name' argument is solely for debugging -- it is not
+        necessarily the name of this node in any Tahoe-LAFS namespace. """
         # this returns synchronously. It starts with a "cap string".
         assert isinstance(writecap, (str, type(None))), type(writecap)
         assert isinstance(readcap,  (str, type(None))), type(readcap)
hunk ./src/allmydata/test/test_deepcheck.py 149
             if not unit:
                 # stream should end with a newline, so split returns ""
                 continue
-            yield simplejson.loads(unit)
+            try:
+                yield simplejson.loads(unit)
+            except ValueError, le:
+                le.args = tuple(le.args + (unit,))
+                raise
 
     def web(self, n, method="GET", **kwargs):
         # returns (data, url)
hunk ./src/allmydata/test/test_deepcheck.py 207
 
 class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
     # construct a small directory tree (with one dir, one immutable file, one
-    # mutable file, one LIT file, and a loop), and then check/examine it in
-    # various ways.
+    # mutable file, one LIT file, one DIR2:LIT empty dir, one DIR2:LIT tiny
+    # dir, and a loop), and then check/examine it in various ways.
 
     def set_up_tree(self):
         # 2.9s
hunk ./src/allmydata/test/test_deepcheck.py 213
 
-        # root
-        #   mutable
-        #   large
-        #   small
-        #   small2
-        #   loop -> root
         c0 = self.g.clients[0]
         d = c0.create_dirnode()
         def _created_root(n):
hunk ./src/allmydata/test/test_deepcheck.py 247
             self.small2_uri = n.get_uri()
         d.addCallback(_created_small2)
 
+        empty_litdir_uri = "URI:DIR2-LIT:"
+        tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
+
+        d.addCallback(lambda ign: self.root._create_and_validate_node(None, empty_litdir_uri, name=u"test_deepcheck empty_lit_dir"))
+        def _created_empty_lit_dir(n):
+            self.empty_lit_dir = n
+            self.empty_lit_dir_uri = n.get_uri()
+            self.root.set_node(u"empty_lit_dir", n)
+        d.addCallback(_created_empty_lit_dir)
+
+        d.addCallback(lambda ign: self.root._create_and_validate_node(None, tiny_litdir_uri, name=u"test_deepcheck tiny_lit_dir"))
+        def _created_tiny_lit_dir(n):
+            self.tiny_lit_dir = n
+            self.tiny_lit_dir_uri = n.get_uri()
+            self.root.set_node(u"tiny_lit_dir", n)
+        d.addCallback(_created_tiny_lit_dir)
+
         d.addCallback(lambda ign: self.root.set_node(u"loop", self.root))
         return d
 
hunk ./src/allmydata/test/test_deepcheck.py 348
         return d
 
     def check_stats_good(self, s):
-        self.failUnlessEqual(s["count-directories"], 1)
-        self.failUnlessEqual(s["count-files"], 4)
+        self.failUnlessEqual(s["count-directories"], 3)
+        self.failUnlessEqual(s["count-files"], 5)
         self.failUnlessEqual(s["count-immutable-files"], 1)
hunk ./src/allmydata/test/test_deepcheck.py 351
-        self.failUnlessEqual(s["count-literal-files"], 2)
+        self.failUnlessEqual(s["count-literal-files"], 3)
         self.failUnlessEqual(s["count-mutable-files"], 1)
         # don't check directories: their size will vary
         # s["largest-directory"]
hunk ./src/allmydata/test/test_deepcheck.py 356
         # s["size-directories"]
-        self.failUnlessEqual(s["largest-directory-children"], 5)
+        self.failUnlessEqual(s["largest-directory-children"], 7)
         self.failUnlessEqual(s["largest-immutable-file"], 13000)
         # to re-use this function for both the local
         # dirnode.start_deep_stats() and the webapi t=start-deep-stats, we
hunk ./src/allmydata/test/test_deepcheck.py 364
         # returns a list of tuples, but JSON only knows about lists., so
         # t=start-deep-stats returns a list of lists.
         histogram = [tuple(stuff) for stuff in s["size-files-histogram"]]
-        self.failUnlessEqual(histogram, [(11, 31, 2),
+        self.failUnlessEqual(histogram, [(4, 10, 1), (11, 31, 2),
                                          (10001, 31622, 1),
                                          ])
         self.failUnlessEqual(s["size-immutable-files"], 13000)
hunk ./src/allmydata/test/test_deepcheck.py 368
-        self.failUnlessEqual(s["size-literal-files"], 48)
+        self.failUnlessEqual(s["size-literal-files"], 56)
 
     def do_web_stream_manifest(self, ignored):
         d = self.web(self.root, method="POST", t="stream-manifest")
hunk ./src/allmydata/test/test_deepcheck.py 381
         files = [u for u in units if u["type"] in ("file", "directory")]
         assert units[-1]["type"] == "stats"
         stats = units[-1]["stats"]
-        self.failUnlessEqual(len(files), 5)
-        # [root,mutable,large] are distributed, [small,small2] are not
+        self.failUnlessEqual(len(files), 8)
+        # [root,mutable,large] are distributed, [small,small2,empty_litdir,tiny_litdir] are not
         self.failUnlessEqual(len([f for f in files
                                   if f["verifycap"] is not None]), 3)
         self.failUnlessEqual(len([f for f in files
hunk ./src/allmydata/test/test_deepcheck.py 386
-                                  if f["verifycap"] is None]), 2)
+                                  if f["verifycap"] is None]), 5)
         self.failUnlessEqual(len([f for f in files
                                   if f["repaircap"] is not None]), 3)
         self.failUnlessEqual(len([f for f in files
hunk ./src/allmydata/test/test_deepcheck.py 390
-                                  if f["repaircap"] is None]), 2)
+                                  if f["repaircap"] is None]), 5)
         self.failUnlessEqual(len([f for f in files
                                   if f["storage-index"] is not None]), 3)
         self.failUnlessEqual(len([f for f in files
hunk ./src/allmydata/test/test_deepcheck.py 394
-                                  if f["storage-index"] is None]), 2)
+                                  if f["storage-index"] is None]), 5)
         # make sure that a mutable file has filecap==repaircap!=verifycap
         mutable = [f for f in files
                    if f["cap"] is not None
hunk ./src/allmydata/test/test_deepcheck.py 437
         d.addCallback(self.failUnlessEqual, None, "small")
         d.addCallback(lambda ign: self.small2.check(Monitor()))
         d.addCallback(self.failUnlessEqual, None, "small2")
+        d.addCallback(lambda ign: self.empty_lit_dir.check(Monitor()))
+        d.addCallback(self.failUnlessEqual, None, "empty_lit_dir")
+        d.addCallback(lambda ign: self.tiny_lit_dir.check(Monitor()))
+        d.addCallback(self.failUnlessEqual, None, "tiny_lit_dir")
 
         # and again with verify=True
         d.addCallback(lambda ign: self.root.check(Monitor(), verify=True))
hunk ./src/allmydata/test/test_deepcheck.py 453
         d.addCallback(self.failUnlessEqual, None, "small")
         d.addCallback(lambda ign: self.small2.check(Monitor(), verify=True))
         d.addCallback(self.failUnlessEqual, None, "small2")
+        d.addCallback(lambda ign: self.empty_lit_dir.check(Monitor(), verify=True))
+        d.addCallback(self.failUnlessEqual, None, "empty_lit_dir")
+        d.addCallback(lambda ign: self.tiny_lit_dir.check(Monitor(), verify=True))
+        d.addCallback(self.failUnlessEqual, None, "tiny_lit_dir")
 
         # and check_and_repair(), which should be a nop
         d.addCallback(lambda ign: self.root.check_and_repair(Monitor()))
hunk ./src/allmydata/test/test_deepcheck.py 463
         d.addCallback(self.check_and_repair_is_healthy, self.root, "root")
         d.addCallback(lambda ign: self.mutable.check_and_repair(Monitor()))
         d.addCallback(self.check_and_repair_is_healthy, self.mutable, "mutable")
-        #TODO d.addCallback(lambda ign: self.large.check_and_repair(Monitor()))
-        #TODO d.addCallback(self.check_and_repair_is_healthy, self.large, "large")
-        #TODO d.addCallback(lambda ign: self.small.check_and_repair(Monitor()))
-        #TODO d.addCallback(self.failUnlessEqual, None, "small")
-        #TODO d.addCallback(lambda ign: self.small2.check_and_repair(Monitor()))
-        #TODO d.addCallback(self.failUnlessEqual, None, "small2")
+        d.addCallback(lambda ign: self.large.check_and_repair(Monitor()))
+        d.addCallback(self.check_and_repair_is_healthy, self.large, "large")
+        d.addCallback(lambda ign: self.small.check_and_repair(Monitor()))
+        d.addCallback(self.failUnlessEqual, None, "small")
+        d.addCallback(lambda ign: self.small2.check_and_repair(Monitor()))
+        d.addCallback(self.failUnlessEqual, None, "small2")
+        d.addCallback(lambda ign: self.empty_lit_dir.check_and_repair(Monitor()))
+        d.addCallback(self.failUnlessEqual, None, "empty_lit_dir")
+        d.addCallback(lambda ign: self.tiny_lit_dir.check_and_repair(Monitor()))
 
         # check_and_repair(verify=True)
         d.addCallback(lambda ign: self.root.check_and_repair(Monitor(), verify=True))
hunk ./src/allmydata/test/test_deepcheck.py 478
         d.addCallback(self.check_and_repair_is_healthy, self.root, "root")
         d.addCallback(lambda ign: self.mutable.check_and_repair(Monitor(), verify=True))
         d.addCallback(self.check_and_repair_is_healthy, self.mutable, "mutable")
-        #TODO d.addCallback(lambda ign: self.large.check_and_repair(Monitor(), verify=True))
-        #TODO d.addCallback(self.check_and_repair_is_healthy, self.large, "large",
-        #TODO               incomplete=True)
-        #TODO d.addCallback(lambda ign: self.small.check_and_repair(Monitor(), verify=True))
-        #TODO d.addCallback(self.failUnlessEqual, None, "small")
-        #TODO d.addCallback(lambda ign: self.small2.check_and_repair(Monitor(), verify=True))
-        #TODO d.addCallback(self.failUnlessEqual, None, "small2")
+        d.addCallback(lambda ign: self.large.check_and_repair(Monitor(), verify=True))
+        d.addCallback(self.check_and_repair_is_healthy, self.large, "large", incomplete=True)
+        d.addCallback(lambda ign: self.small.check_and_repair(Monitor(), verify=True))
+        d.addCallback(self.failUnlessEqual, None, "small")
+        d.addCallback(lambda ign: self.small2.check_and_repair(Monitor(), verify=True))
+        d.addCallback(self.failUnlessEqual, None, "small2")
+        d.addCallback(self.failUnlessEqual, None, "small2")
+        d.addCallback(lambda ign: self.empty_lit_dir.check_and_repair(Monitor(), verify=True))
+        d.addCallback(self.failUnlessEqual, None, "empty_lit_dir")
+        d.addCallback(lambda ign: self.tiny_lit_dir.check_and_repair(Monitor(), verify=True))
 
 
         # now deep-check the root, with various verify= and repair= options
hunk ./src/allmydata/test/test_deepcheck.py 631
         d.addCallback(self.json_check_lit, self.small, "small")
         d.addCallback(lambda ign: self.web_json(self.small2, t="check"))
         d.addCallback(self.json_check_lit, self.small2, "small2")
+        d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check"))
+        d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir")
+        d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check"))
+        d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir")
 
         # check and verify
         d.addCallback(lambda ign:
hunk ./src/allmydata/test/test_deepcheck.py 653
         d.addCallback(lambda ign:
                       self.web_json(self.small2, t="check", verify="true"))
         d.addCallback(self.json_check_lit, self.small2, "small2+v")
+        d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check", verify="true"))
+        d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir+v")
+        d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check", verify="true"))
+        d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir+v")
 
         # check and repair, no verify
         d.addCallback(lambda ign:
hunk ./src/allmydata/test/test_deepcheck.py 674
         d.addCallback(lambda ign:
                       self.web_json(self.small2, t="check", repair="true"))
         d.addCallback(self.json_check_lit, self.small2, "small2+r")
+        d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check", repair="true"))
+        d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir+r")
+        d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check", repair="true"))
+        d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir+r")
 
         # check+verify+repair
         d.addCallback(lambda ign:
hunk ./src/allmydata/test/test_deepcheck.py 695
         d.addCallback(lambda ign:
                       self.web_json(self.small2, t="check", repair="true", verify="true"))
         d.addCallback(self.json_check_lit, self.small2, "small2+vr")
+        d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check", repair="true", verify=True))
+        d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir+vr")
+        d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check", repair="true", verify=True))
+        d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir+vr")
 
         # now run a deep-check, with various verify= and repair= flags
         d.addCallback(lambda ign:
hunk ./src/allmydata/test/test_deepcheck.py 723
         d.addCallback(lambda ign: self.web(self.large, t="info"))
         d.addCallback(lambda ign: self.web(self.small, t="info"))
         d.addCallback(lambda ign: self.web(self.small2, t="info"))
+        d.addCallback(lambda ign: self.web(self.empty_lit_dir, t="info"))
+        d.addCallback(lambda ign: self.web(self.tiny_lit_dir, t="info"))
 
         return d
 
hunk ./src/allmydata/test/test_deepcheck.py 780
             self.failUnlessEqual(caps[self.large.get_uri()], "large")
             self.failUnlessEqual(caps[self.small.get_uri()], "small")
             self.failUnlessEqual(caps[self.small2.get_uri()], "small2")
+            self.failUnlessEqual(caps[self.empty_lit_dir.get_uri()], "empty_lit_dir")
+            self.failUnlessEqual(caps[self.tiny_lit_dir.get_uri()], "tiny_lit_dir")
         d.addCallback(_check)
         return d
 
hunk ./src/allmydata/uri.py 197
     STRING_RE=re.compile('^URI:LIT:'+base32.BASE32STR_anybytes+'$')
     HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'LIT'+SEP+base32.BASE32STR_anybytes+'$')
 
-    def __init__(self, data=None):
-        if data is not None:
-            assert isinstance(data, str)
-            self.data = data
+    def __init__(self, data):
+        assert isinstance(data, str)
+        self.data = data
 
     @classmethod
     def init_from_human_encoding(cls, uri):
}

Context:

[More cleanups to test_cli using new utilities for reading and writing files.
david-sarah@jacaranda.org**20100206013855
 Ignore-this: 9fd2294406b346bfe9144fff6a61f789
] 
[Fix race conditions and missing callback in allmydata.test.test_cli.Cp.test_copy_using_filecap, add utilities for one-liner reading and writing of files, and fix cases in test_cli where files were not being closed after writing.
david-sarah@jacaranda.org**20100206013727
 Ignore-this: 49da6c33190d526a4ae84c472f04d5f4
] 
[setup: comment-out the dependency on pycrypto, see #953
zooko@zooko.com**20100215050844
 Ignore-this: 2751120921ff35b8189d8fcd896da149
] 
[Add tests for #939
Kevan Carstensen <kevan@isnotajoke.com>**20100212062137
 Ignore-this: 5459e8c64ba76cca70aa720e68549637
] 
[Alter CLI utilities to handle nonexistent aliases better
Kevan Carstensen <kevan@isnotajoke.com>**20100211024318
 Ignore-this: e698ea4a57f5fe27c24336581ca0cf65
] 
[adding pycrypto to the auto dependencies
secorp@allmydata.com**20100206054314
 Ignore-this: b873fc00a6a5b001d30d479e6053cf2f
] 
[docs running.html - "tahoe run ." does not work with the current installation, replaced with "tahoe start ."
secorp@allmydata.com**20100206165320
 Ignore-this: fdb2dcb0e417d303cd43b1951a4f8c03
] 
[web/storage.py: display total-seen on the last-complete-cycle line. For #940.
Brian Warner <warner@lothar.com>**20100208002010
 Ignore-this: c0ed860f3e9628d3171d2b055d96c5aa
] 
[code coverage: replace figleaf with coverage.py, should work on py2.6 now.
Brian Warner <warner@lothar.com>**20100203165421
 Ignore-this: 46ab590360be6a385cb4fc4e68b6b42c
 
 It still lacks the right HTML report (the builtin report is very pretty, but
 lacks the "lines uncovered" numbers that I want), and the half-finished
 delta-from-last-run measurements.
] 
[More comprehensive changes and ticket references for NEWS
david-sarah@jacaranda.org**20100202061256
 Ignore-this: 696cf0106e8a7fd388afc5b55fba8a1b
] 
[docs: install.html: link into Python 2.5.5 download page
zooko@zooko.com**20100202065852
 Ignore-this: 1a9471b8175b7de5741d8445a7ede29d
] 
[TAG allmydata-tahoe-1.6.0
zooko@zooko.com**20100202061125
 Ignore-this: dee6ade7ac1452cf5d1d9c69a8146d84
] 
Patch bundle hash:
7d1641ff84c408c3275fe709337123b0c4a988e9