diff -rN -u old-trunk/docs/frontends/webapi.txt new-trunk/docs/frontends/webapi.txt --- old-trunk/docs/frontends/webapi.txt 2009-04-07 20:44:50.000000000 -0600 +++ new-trunk/docs/frontends/webapi.txt 2009-04-07 20:44:54.000000000 -0600 @@ -393,8 +393,11 @@ "verify_uri": verify_uri, "size": bytes, "mutable": false, - "metadata": {"ctime": 1202777696.7564139, - "mtime": 1202777696.7564139 + "metadata": { + "ctime": 1202777696.7564139, + "mtime": 1202777696.7564139, + "lcrtime": 1202777696.7564139, + "lmotime": 1202777696.7564139, } } ] @@ -402,7 +405,7 @@ this directory, as a mapping from child name to a set of data about the child (the same data that would appear in a corresponding GET?t=json of the child itself). The child entries also include metadata about each child, - including creation- and modification- timestamps. The output looks like + including link-creation- and link-change- timestamps. The output looks like this: GET /uri/$DIRCAP?t=json : @@ -418,6 +421,8 @@ "metadata": { "ctime": 1202777696.7564139, "mtime": 1202777696.7564139 + "lcrtime": 1202777696.7564139, + "lmotime": 1202777696.7564139, } } ], "subdir": [ "dirnode", { "rw_uri": rwuri, @@ -425,6 +430,8 @@ "metadata": { "ctime": 1202778102.7589991, "mtime": 1202778111.2160511, + "lcrtime": 1202777696.7564139, + "lmotime": 1202777696.7564139, } } ] } } ] diff -rN -u old-trunk/docs/specifications/dirnodes.txt new-trunk/docs/specifications/dirnodes.txt --- old-trunk/docs/specifications/dirnodes.txt 2009-04-07 20:44:50.000000000 -0600 +++ new-trunk/docs/specifications/dirnodes.txt 2009-04-07 20:44:54.000000000 -0600 @@ -176,30 +176,30 @@ netstring(cap) = 4+len(cap) encrypted(cap) = 16+cap+32 JSON({}) = 2 - JSON({ctime=float,mtime=float}): 57 - netstring(metadata) = 4+57 = 61 + JSON({ctime=float,mtime=float,lcrtime=float,lmotime=float}): 117 + netstring(metadata) = 4+117 = 121 so a CHK entry is: - 5+ 4+len(name) + 4+97 + 5+16+97+32 + 4+57 -And a 15-byte filename gives a 336-byte entry. When the entry points at a + 5+ 4+len(name) + 4+97 + 5+16+97+32 + 4+117 +And a 15-byte filename gives a 396-byte entry. When the entry points at a subdirectory instead of a file, the entry is a little bit smaller. So an -empty directory uses 0 bytes, a directory with one child uses about 336 -bytes, a directory with two children uses about 672, etc. +empty directory uses 0 bytes, a directory with one child uses about 396 +bytes, a directory with two children uses about 792, etc. When the dirnode data is encoding using our default 3-of-10, that means we -get 112ish bytes of data in each share per child. +get 132ish bytes of data in each share per child. The pubkey, signature, and hashes form the first 935ish bytes of the container, then comes our data, then about 1216 bytes of encprivkey. So if we read the first: 1kB: we get 65bytes of dirnode data : only empty directories - 1kiB: 89bytes of dirnode data : maybe one short-named subdir - 2kB: 1065bytes: about 9 entries - 3kB: 2065bytes: about 18 entries, or 7.5 entries plus the encprivkey - 4kB: 3065bytes: about 27 entries, or about 16.5 plus the encprivkey + 1kiB: 89bytes of dirnode data : only empty directories + 2kB: 1065bytes: about 2 or 3 entries + 3kB: 2065bytes: about 5 entries, or 2 entries plus the encprivkey + 4kB: 3065bytes: about 8 entries, or about 5 plus the encprivkey -So we've written the code to do an initial read of 2kB from each share when +So we've written the code to do an initial read of 4kB from each share when we read the mutable file, which should give good performance (one RTT) for small directories. diff -rN -u old-trunk/src/allmydata/dirnode.py new-trunk/src/allmydata/dirnode.py --- old-trunk/src/allmydata/dirnode.py 2009-04-07 20:44:53.000000000 -0600 +++ new-trunk/src/allmydata/dirnode.py 2009-04-07 20:44:55.000000000 -0600 @@ -83,15 +83,39 @@ metadata = children[name][1].copy() else: metadata = {"ctime": now, - "mtime": now} - if new_metadata is None: - # update timestamps - if "ctime" not in metadata: - metadata["ctime"] = now - metadata["mtime"] = now - else: - # just replace it - metadata = new_metadata.copy() + "mtime": now, + "lcrtime": now, + "lmotime": now, + } + + if new_metadata is not None: + # Overwrite all metadata. + newmd = new_metadata.copy() + + # Except that callers are not allowed to mess with the + # link-timestamp metadata. + for special in ['ctime', 'mtime', 'lcrtime', 'lmotime']: + if newmd.has_key(special): + del newmd[special] + if metadata.has_key(special): + newmd[special] = metadata[special] + + metadata = newmd + + # update timestamps + if "lcrtime" not in metadata: + if "ctime" in metadata: + # In Tahoe < 1.4.0 we used the word "ctime" to mean what Tahoe >= 1.4.0 calls "lcrtime". + metadata["lcrtime"] = metadata["ctime"] + else: + metadata["lmotime"] = now + metadata["lmotime"] = now + + # For backwards compatibility with Tahoe < 1.4.0: + if "ctime" not in metadata: + metadata["ctime"] = now + metadata["mtime"] = now + children[name] = (child, metadata) new_contents = self.node._pack_contents(children) return new_contents diff -rN -u old-trunk/src/allmydata/mutable/servermap.py new-trunk/src/allmydata/mutable/servermap.py --- old-trunk/src/allmydata/mutable/servermap.py 2009-04-07 20:44:53.000000000 -0600 +++ new-trunk/src/allmydata/mutable/servermap.py 2009-04-07 20:44:55.000000000 -0600 @@ -374,7 +374,7 @@ # fixed-size slots so we can retrieve less data. For now, we'll just # read 2000 bytes, which also happens to read enough actual data to # pre-fetch a 9-entry dirnode. - self._read_size = 2000 + self._read_size = 4000 if mode == MODE_CHECK: # we use unpack_prefix_and_signature, so we need 1k self._read_size = 1000 diff -rN -u old-trunk/src/allmydata/scripts/tahoe_backup.py new-trunk/src/allmydata/scripts/tahoe_backup.py --- old-trunk/src/allmydata/scripts/tahoe_backup.py 2009-04-07 20:44:53.000000000 -0600 +++ new-trunk/src/allmydata/scripts/tahoe_backup.py 2009-04-07 20:44:55.000000000 -0600 @@ -4,6 +4,7 @@ import urllib import simplejson import datetime +import platform from allmydata.scripts.common import get_alias, escape_path, DEFAULT_ALIAS from allmydata.scripts.common_http import do_http from allmydata import uri @@ -69,11 +69,7 @@ metadata = {} # posix stat(2) metadata, depends on the platform - os.stat_float_times(True) s = os.stat(path) - metadata["ctime"] = s.st_ctime - metadata["mtime"] = s.st_mtime - misc_fields = ("st_mode", "st_ino", "st_dev", "st_uid", "st_gid") macos_misc_fields = ("st_rsize", "st_creator", "st_type") for field in misc_fields + macos_misc_fields: diff -rN -u old-trunk/src/allmydata/scripts/tahoe_ls.py new-trunk/src/allmydata/scripts/tahoe_ls.py --- old-trunk/src/allmydata/scripts/tahoe_ls.py 2009-04-07 20:44:53.000000000 -0600 +++ new-trunk/src/allmydata/scripts/tahoe_ls.py 2009-04-07 20:44:55.000000000 -0600 @@ -65,8 +65,20 @@ name = unicode(name) child = children[name] childtype = child[0] - ctime = child[1]["metadata"].get("ctime") - mtime = child[1]["metadata"].get("mtime") + + # lcrtime is not really what unix filesystems mean by "ctime", but it + # *is* apparently what many or even most unix programmers and users + # think that a unix filesystem means by "ctime"... + ctime = child[1]["metadata"].get("lcrtime") + if not ctime: + ctime = child[1]["metadata"].get("ctime") + + # lmotime is not really what unix filesystems mean by "mtime", because + # lmotime is a property of the link and mtime is a property of the file + # contents... + mtime = child[1]["metadata"].get("lmotime") + if not mtime: + mtime = child[1]["metadata"].get("mtime") rw_uri = child[1].get("rw_uri") ro_uri = child[1].get("ro_uri") if ctime: diff -rN -u old-trunk/src/allmydata/test/test_cli.py new-trunk/src/allmydata/test/test_cli.py --- old-trunk/src/allmydata/test/test_cli.py 2009-04-07 20:44:53.000000000 -0600 +++ new-trunk/src/allmydata/test/test_cli.py 2009-04-07 20:44:55.000000000 -0600 @@ -949,7 +949,7 @@ self.failUnlessEqual(fu, 0) self.failUnlessEqual(fr, 3) # empty, home, home/parent, home/parent/subdir - self.failUnlessEqual(dc, 0) + self.failUnlessEqual(dc, 0, out) self.failUnlessEqual(dr, 4) d.addCallback(_check4a) diff -rN -u old-trunk/src/allmydata/test/test_dirnode.py new-trunk/src/allmydata/test/test_dirnode.py --- old-trunk/src/allmydata/test/test_dirnode.py 2009-04-07 20:44:53.000000000 -0600 +++ new-trunk/src/allmydata/test/test_dirnode.py 2009-04-07 20:44:55.000000000 -0600 @@ -416,7 +416,7 @@ d.addCallback(lambda res: n.get_metadata_for(u"child")) d.addCallback(lambda metadata: self.failUnlessEqual(sorted(metadata.keys()), - ["ctime", "mtime"])) + ["ctime", "lcrtime", "lmotime", "mtime"])) d.addCallback(lambda res: self.shouldFail(NoSuchChildError, "gcamap-no", @@ -439,7 +439,7 @@ self.failUnlessEqual(child.get_uri(), fake_file_uri.to_string()) self.failUnlessEqual(sorted(metadata.keys()), - ["ctime", "mtime"]) + ["ctime", "lcrtime", "lmotime", "mtime"]) d.addCallback(_check_child_and_metadata2) d.addCallback(lambda res: @@ -448,36 +448,41 @@ child, metadata = res self.failUnless(isinstance(child, FakeDirectoryNode)) self.failUnlessEqual(sorted(metadata.keys()), - ["ctime", "mtime"]) + ["ctime", "lcrtime", "lmotime", "mtime"]) d.addCallback(_check_child_and_metadata3) # set_uri + metadata - # it should be possible to add a child without any metadata + # it should not be possible to add a child without any metadata d.addCallback(lambda res: n.set_uri(u"c2", fake_file_uri.to_string(), {})) d.addCallback(lambda res: n.get_metadata_for(u"c2")) - d.addCallback(lambda metadata: self.failUnlessEqual(metadata, {})) + def _has_ltimes(metadata): + self.failUnless(metadata.has_key('ctime')) + self.failUnless(metadata.has_key('mtime')) + self.failUnless(metadata.has_key('lcrtime')) + self.failUnless(metadata.has_key('lmotime')) + d.addCallback(_has_ltimes) + + # nor to override the link timestamps with the "metadata" argument + d.addCallback(lambda res: n.set_uri(u"c2", fake_file_uri.to_string(), {'lcrtime': "bogus"})) + d.addCallback(lambda res: n.get_metadata_for(u"c2")) + def _has_good_lcrtime(metadata): + self.failUnless(metadata.has_key('lcrtime')) + self.failIfEqual(metadata['lcrtime'], 'bogus') + d.addCallback(_has_good_lcrtime) # if we don't set any defaults, the child should get timestamps d.addCallback(lambda res: n.set_uri(u"c3", fake_file_uri.to_string())) d.addCallback(lambda res: n.get_metadata_for(u"c3")) d.addCallback(lambda metadata: self.failUnlessEqual(sorted(metadata.keys()), - ["ctime", "mtime"])) - - # or we can add specific metadata at set_uri() time, which - # overrides the timestamps - d.addCallback(lambda res: n.set_uri(u"c4", fake_file_uri.to_string(), - {"key": "value"})) - d.addCallback(lambda res: n.get_metadata_for(u"c4")) - d.addCallback(lambda metadata: - self.failUnlessEqual(metadata, {"key": "value"})) + ["ctime", "lcrtime", "lmotime", "mtime"])) d.addCallback(lambda res: n.delete(u"c2")) d.addCallback(lambda res: n.delete(u"c3")) d.addCallback(lambda res: n.delete(u"c4")) # set_node + metadata - # it should be possible to add a child without any metadata + # it should be impossible to add a child without any metadata d.addCallback(lambda res: n.set_node(u"d2", n, {})) d.addCallback(lambda res: self.client.create_empty_dirnode()) d.addCallback(lambda n2: diff -rN -u old-trunk/src/allmydata/util/time_format.py new-trunk/src/allmydata/util/time_format.py --- old-trunk/src/allmydata/util/time_format.py 2009-04-07 20:44:53.000000000 -0600 +++ new-trunk/src/allmydata/util/time_format.py 2009-04-07 20:44:55.000000000 -0600 @@ -19,6 +19,11 @@ now = t() return datetime.datetime.utcfromtimestamp(now).isoformat(sep) +def iso_local(now=None, sep='_', t=time.time): + if now is None: + now = t() + return datetime.datetime.fromtimestamp(now).isoformat(sep) + def iso_utc_time_to_seconds(isotime, _conversion_re=re.compile(r"(?P\d{4})-(?P\d{2})-(?P\d{2})[T_ ](?P\d{2}):(?P\d{2}):(?P\d{2})(?P\.\d+)?")): """ The inverse of iso_utc(). diff -rN -u old-trunk/src/allmydata/web/directory.py new-trunk/src/allmydata/web/directory.py --- old-trunk/src/allmydata/web/directory.py 2009-04-07 20:44:53.000000000 -0600 +++ new-trunk/src/allmydata/web/directory.py 2009-04-07 20:44:55.000000000 -0600 @@ -13,7 +13,7 @@ from foolscap.eventual import fireEventually -from allmydata.util import base32 +from allmydata.util import base32, time_format from allmydata.uri import from_string_dirnode from allmydata.interfaces import IDirectoryNode, IFileNode, IMutableFileNode, \ ExistingChildError, NoSuchChildError @@ -592,16 +592,25 @@ ctx.fillSlots("rename", rename) times = [] - TIME_FORMAT = "%H:%M:%S %d-%b-%Y" - if "ctime" in metadata: - ctime = time.strftime(TIME_FORMAT, - time.localtime(metadata["ctime"])) - times.append("c: " + ctime) - if "mtime" in metadata: - mtime = time.strftime(TIME_FORMAT, - time.localtime(metadata["mtime"])) + lcrtime = metadata.get("lcrtime") + if lcrtime is not None: + times.append("lcr: " + time_format.iso_local(lcrtime)) + else: + # For backwards-compatibility with links last modified by Tahoe < 1.4.0: + if "ctime" in metadata: + ctime = time_format.iso_local(metadata["ctime"]) + times.append("c: " + ctime) + lmotime = metadata.get("lmotime") + if lmotime is not None: if times: times.append(T.br()) + times.append("lmo: " + time_format.iso_local(lmotime)) + else: + # For backwards-compatibility with links last modified by Tahoe < 1.4.0: + if "mtime" in metadata: + mtime = time_format.iso_local(metadata["mtime"]) + if times: + times.append(T.br()) times.append("m: " + mtime) ctx.fillSlots("times", times)