Ticket #684: new2-684.diff

docs/frontends/webapi.txt

@@ -340 +340 @@
  mutable file, and return its write-cap in the HTTP respose. The default is
  to create an immutable file, returning the read-cap as a response.
 
+ To use a randomly-generated key as the encryption key for the file, add
+ the query argument "key=random".
+
+ To specify a key, add a query argument of the form "key=encoding-value",
+ where 'encoding' is one of 'hex', 'base16', or 'base32' and value is the
+ 128-bit key encoded with the specified encoding.  For example, the following
+ are all equivalent, and permitted:
+
+          key=hex-B6F39C58C25A501B6FDF4AF94E07BB5D
+          key=base16-B6F39C58C25A501B6FDF4AF94E07BB5D
+          key=base32-w3zzywgcljibw367jl4u4b53lu
+
+ Be VERY careful that you know what you're doing if you use this feature.
+ Choosing bad keys could compromise the security of your files.  Also, make
+ sure that every file is encrypted with a unique key because uploading
+ different files with the same key will result in a storage index collision.
+ Even uploading the same file encoded with different FEC parameters will cause
+ a collision if you use the same key.  It's a good idea to hash the FEC
+ parameters (k, N, segsize) into your key to be sure that doesn't happen.
+
+ Normally, you should omit the 'key' argument and let Tahoe construct a
+ content hash key (CHK) which is secure, unique and will make your uploads
+ idempotent.
+
 === Creating A New Directory ===
 
 POST /uri?t=mkdir
@@ -747 +771 @@
  the upload results page. The default is to create an immutable file,
  returning the upload results page as a response.
 
+ To use a randomly-generated key as the encryption key for the file, add
+ the argument "key=random".
+
+ To specify a key, add an argument of the form "key=encoding-value", where
+ 'encoding' is one of 'hex', 'base16', or 'base32' and value is the 128-bit
+ key encoded with the specified encoding.  For example, the following are all
+ equivalent, and permitted:
+
+          key=hex-B6F39C58C25A501B6FDF4AF94E07BB5D
+          key=base16-B6F39C58C25A501B6FDF4AF94E07BB5D
+          key=base32-w3zzywgcljibw367jl4u4b53lu
+
+ Be VERY careful that you know what you're doing if you use this feature.
+ Choosing bad keys could compromise the security of your files.  Also, make
+ sure that every file is encrypted with a unique key because uploading
+ different files with the same key will result in a storage index collision.
+ Even uploading the same file encoded with different FEC parameters will cause
+ a collision if you use the same key.  It's a good idea to hash the FEC
+ parameters (k, N, segsize) into your key to be sure that doesn't happen.
+
+ Normally, you should omit the 'key' argument and let Tahoe construct a
+ content hash key (CHK) which is secure, unique and will make your uploads
+ idempotent.
 
 POST /uri/$DIRCAP/[SUBDIRS../]?t=upload
 

src/allmydata/immutable/upload.py

@@ -1100 +1100 @@
 class FileHandle(BaseUploadable):
     implements(IUploadable)
 
-    def __init__(self, filehandle, convergence):
+    def __init__(self, filehandle, convergence, key=None):
         """
         Upload the data from the filehandle.  If convergence is None then a
         random encryption key will be used, else the plaintext will be hashed,
@@ -1112 +1112 @@
         self._key = None
         self.convergence = convergence
         self._size = None
+        self.chosen_key = key
+        if key:
+            assert convergence is None # Can't specify both key and convergence
+            assert isinstance(key, str) and len(key) is 16
 
     def _get_encryption_key_convergent(self):
         if self._key is not None:
@@ -1156 +1160 @@
     def get_encryption_key(self):
         if self.convergence is not None:
             return self._get_encryption_key_convergent()
+        elif self.chosen_key is not None:
+            return defer.succeed(self.chosen_key)
         else:
             return self._get_encryption_key_random()
 
@@ -1176 +1182 @@
         pass
 
 class FileName(FileHandle):
-    def __init__(self, filename, convergence):
+    def __init__(self, filename, convergence, key=None):
         """
         Upload the data from the filename.  If convergence is None then a
         random encryption key will be used, else the plaintext will be hashed,
@@ -1184 +1190 @@
         "convergence" argument to form the encryption key.
         """
         assert convergence is None or isinstance(convergence, str), (convergence, type(convergence))
-        FileHandle.__init__(self, open(filename, "rb"), convergence=convergence)
+        FileHandle.__init__(self, open(filename, "rb"), convergence=convergence, key=key)
     def close(self):
         FileHandle.close(self)
         self._filehandle.close()
 
 class Data(FileHandle):
-    def __init__(self, data, convergence):
+    def __init__(self, data, convergence, key=None):
         """
         Upload the data from the data argument.  If convergence is None then a
         random encryption key will be used, else the plaintext will be hashed,
@@ -1198 +1204 @@
         "convergence" argument to form the encryption key.
         """
         assert convergence is None or isinstance(convergence, str), (convergence, type(convergence))
-        FileHandle.__init__(self, StringIO(data), convergence=convergence)
+        FileHandle.__init__(self, StringIO(data), convergence=convergence, key=key)
 
 class Uploader(service.MultiService, log.PrefixingLogMixin):
     """I am a service that allows file uploading. I am a service-child of the

src/allmydata/test/test_system.py

@@ -1 @@
-from base64 import b32encode
+from base64 import b32encode, b16encode
 import os, sys, time, re, simplejson
 from cStringIO import StringIO
 from zope.interface import implements
@@ -1183 +1183 @@
         d.addCallback(lambda res: self.GET(public + "/subdir3/new.txt"))
         d.addCallback(self.failUnlessEqual, "NEWER contents")
 
+        # test unlinked PUT with specified key
+        key = 'd'*16
+        d.addCallback(lambda res: self.PUT("uri?key=hex-" + b16encode(key),
+                                           "data" * 100))
+        def _check_specified_key_uri(res):
+            u = uri.from_string_filenode(res)
+            self.failUnlessEqual(u.key, key)
+            return res
+        d.addCallback(_check_specified_key_uri)
+
+        # test unlinked PUT with content hash key
+        d.addCallback(lambda res: self.PUT("uri", "data" * 100))
+        def _check_CHK_key_uri(res):
+            u = uri.from_string_filenode(res)
+            self.failIfEqual(u.key, key)
+            return res
+        d.addCallback(_check_CHK_key_uri)
+
         # test unlinked POST
         d.addCallback(lambda res: self.POST("uri", t="upload",
                                             file=("new.txt", "data" * 10000)))
@@ -1287 +1305 @@
         d.addCallback(lambda res: self.GET("statistics?t=json"))
         def _got_stats_json(res):
             data = simplejson.loads(res)
-            self.failUnlessEqual(data["counters"]["uploader.files_uploaded"], 5)
+            self.failUnlessEqual(data["counters"]["uploader.files_uploaded"], 7)
             self.failUnlessEqual(data["stats"]["chk_upload_helper.upload_need_upload"], 1)
         d.addCallback(_got_stats_json)
 

src/allmydata/test/test_upload.py

@@ -30 +30 @@
         self.failUnlessEqual(s, expected)
 
     def test_filehandle_random_key(self):
-        return self._test_filehandle(convergence=None)
+        return self._test_filehandle(convergence=None, key=None)
+
+    def test_filehandle_specified_key(self):
+        return self._test_filehandle(convergence=None, key='a'*16)
 
     def test_filehandle_convergent_encryption(self):
-        return self._test_filehandle(convergence="some convergence string")
+        return self._test_filehandle(convergence="some convergence string", key=None)
 
-    def _test_filehandle(self, convergence):
+    def _test_filehandle(self, convergence, key):
         s = StringIO("a"*41)
-        u = upload.FileHandle(s, convergence=convergence)
+        u = upload.FileHandle(s, convergence=convergence, key=key)
         d = u.get_size()
         d.addCallback(self.failUnlessEqual, 41)
         d.addCallback(lambda res: u.read(1))
@@ -217 +220 @@
 SIZE_SMALL = 16
 SIZE_LARGE = len(DATA)
 
-def upload_data(uploader, data):
-    u = upload.Data(data, convergence=None)
+def upload_data(uploader, data, key=None):
+    u = upload.Data(data, convergence=None, key=key)
     return uploader.upload(u)
 def upload_filename(uploader, filename):
     u = upload.FileName(filename, convergence=None)
@@ -256 +259 @@
         self.failUnlessEqual(len(u.key), 16)
         self.failUnlessEqual(u.size, size)
 
+    def _check_provided_key(self, newuri, size):
+        self._check_large(newuri, size)
+        u = IFileURI(newuri)
+        self.failUnlessEqual(u.key, 'b'*16)
+
     def get_data(self, size):
         return DATA[:size]
 
@@ -301 +309 @@
         d.addCallback(self._check_large, SIZE_LARGE)
         return d
 
+    def test_specified_key(self):
+        data = self.get_data(SIZE_LARGE)
+        d = upload_data(self.u, data, 'b'*16)
+        d.addCallback(extract_uri)
+        d.addCallback(self._check_provided_key, SIZE_LARGE)
+        return d
+
     def test_data_large_odd_segments(self):
         data = self.get_data(SIZE_LARGE)
         segsize = int(SIZE_LARGE / 2.5)
@@ -566 +581 @@
         eu = upload.EncryptAnUploadable(u)
         d1salt1a = eu.get_storage_index()
 
+        # and if we specify a custom encryption key it should be different again
+        key = '\x01' * 16
+        u = upload.Data(DATA, convergence=None, key=key)
+        eu = upload.EncryptAnUploadable(u)
+        k1 = eu.get_storage_index()
+
         # and if we change the encoding parameters, it should be different (from the same convergence string with different encoding parameters)
         u = upload.Data(DATA, convergence="")
         u.encoding_param_k = u.default_encoding_param_k + 1
@@ -581 +602 @@
         eu = upload.EncryptAnUploadable(u)
         d4 = eu.get_storage_index()
 
-        d = DeferredListShouldSucceed([d1,d1a,d1salt1,d1salt2,d1salt1a,d2,d3,d4])
+        d = DeferredListShouldSucceed([d1,d1a,d1salt1,d1salt2,d1salt1a,k1,d2,d3,d4])
         def _done(res):
-            si1, si1a, si1salt1, si1salt2, si1salt1a, si2, si3, si4 = res
+            si1, si1a, si1salt1, si1salt2, si1salt1a, sik1, si2, si3, si4 = res
             self.failUnlessEqual(si1, si1a)
             self.failIfEqual(si1, si2)
             self.failIfEqual(si1, si3)
@@ -593 +614 @@
             self.failIfEqual(si1salt1, si1salt2)
             self.failIfEqual(si1salt2, si1)
             self.failUnlessEqual(si1salt1, si1salt1a)
+            self.failIfEqual(sik1, si1)
+            self.failIfEqual(sik1, si1a)
         d.addCallback(_done)
         return d
 

src/allmydata/test/test_web.py

@@ -1 @@
-import os.path, re, urllib
+import os.path, re, urllib, base64
 import simplejson
 from StringIO import StringIO
 from twisted.application import service
@@ -717 +717 @@
                                                       self.NEWFILE_CONTENTS))
         return d
 
+    def PUT_URI_specified_key(self, key, encoding, encoder, data):
+        return self.PUT("/uri?key=" + encoding + '-' + encoder(key), data)
+
+    def test_PUT_URI_random_key(self):
+        d = self.PUT("/uri?key=random", self.NEWFILE_CONTENTS)
+        return d
+
+    def test_PUT_URI_specified_key_hex(self):
+        return self.PUT_URI_specified_key('0'*16, 'hex', base64.b16encode,
+                                          self.NEWFILE_CONTENTS)
+
+    def test_PUT_URI_specified_key_base16(self):
+        return self.PUT_URI_specified_key('1'*16, 'base16', base64.b16encode,
+                                          self.NEWFILE_CONTENTS)
+
+    def test_PUT_URI_specified_key_base32(self):
+        return self.PUT_URI_specified_key('2'*16, 'base32', base32.b2a,
+                                          self.NEWFILE_CONTENTS)
+
+    def test_PUT_URI_specified_key_invalid_format(self):
+        key_str = base32.b2a('3'*16)
+        d = self.PUT("/uri?key=" + key_str, self.NEWFILE_CONTENTS)
+        return self.failUnlessFailure(d, error.Error)
+
+    def test_PUT_URI_specified_key_incorrect_encoding(self):
+        d = self.PUT_URI_specified_key('4'*16, 'hex', base32.b2a,
+                                       self.NEWFILE_CONTENTS)
+        return self.failUnlessFailure(d, error.Error)
+
+    def test_PUT_URI_specified_key_incorrect_length(self):
+        d = self.PUT_URI_specified_key('5'*16, 'base32', base64.b16encode,
+                                       self.NEWFILE_CONTENTS)
+        return self.failUnlessFailure(d, error.Error)
+
+    def test_PUT_NEWFILEURL_specified_key(self):
+        key = '6' * 16
+        key_str = 'base32-'+base32.b2a(key)
+        d = self.PUT(self.public_url + "/foo/new.txt?key=" + key_str,
+                     self.NEWFILE_CONTENTS)
+        # TODO: we lose the response code, so we can't check this
+        #self.failUnlessEqual(responsecode, 201)
+        d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
+        d.addCallback(lambda res:
+                      self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
+                                                      self.NEWFILE_CONTENTS))
+        return d
+
     def test_PUT_NEWFILEURL_not_mutable(self):
         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
                      self.NEWFILE_CONTENTS)
@@ -1250 +1297 @@
                                                       self.NEWFILE_CONTENTS))
         return d
 
+    def test_POST_upload_specified_key(self):
+        key = '\x27' * 16
+        key_str = 'base32-' + base32.b2a(key)
+        d = self.POST(self.public_url + "/foo", t="upload",
+                      file=("new.txt", self.NEWFILE_CONTENTS),
+                      key=key_str)
+        fn = self._foo_node
+        d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
+        d.addCallback(lambda res:
+                      self.failUnlessChildContentsAre(fn, u"new.txt",
+                                                      self.NEWFILE_CONTENTS))
+        return d
+        
+
     def test_POST_upload_unicode(self):
         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
         d = self.POST(self.public_url + "/foo", t="upload",

src/allmydata/web/common.py

@@ -7 +7 @@
 from allmydata.interfaces import ExistingChildError, NoSuchChildError, \
      FileTooLargeError, NotEnoughSharesError
 from allmydata.mutable.common import UnrecoverableFileError
-from allmydata.util import abbreviate # TODO: consolidate
+from allmydata.util import abbreviate, base32 # TODO: consolidate
+import base64
 
 class IOpHandleTable(Interface):
     pass
@@ -46 +47 @@
         return results[0]
     return default
 
+class RandomKey:
+    """Marker to indicate that the user wants a randomly-generated key"""
+
+def get_key_arg(ctx_or_req):
+    """
+    Extract the 'key' argument from the query args.  If not found,
+    return None.  If the argument is "random", return RandomKey.
+    Otherwise, the argument should be of the form "encoding-value",
+    where encoding is one of 'hex', 'base16', or 'base32'.  Parse it
+    and return the value as a binary string, which must be 16 bytes in
+    length.
+    """
+    req = IRequest(ctx_or_req)
+    key_str = get_arg(req, "key", "").strip()
+    if key_str == "":
+        return None
+    elif key_str == "random":
+        return RandomKey
+
+    try:
+        encoding, value = key_str.split('-', 1)
+        if encoding == 'base32':
+            key = base32.a2b(value)
+        elif encoding == 'hex' or encoding == 'base16':
+            key = base64.b16decode(value)
+        else:
+            raise WebError('Unknown key format ' + encoding)
+    except:
+        raise WebError('Invalid key format')
+
+    if len(key) != 16:
+        raise WebError("Key must be 16 bytes in length")
+
+    return key
+
 def abbreviate_time(data):
     # 1.23s, 790ms, 132us
     if data is None:

src/allmydata/web/unlinked.py

@@ -4 +4 @@
 from twisted.internet import defer
 from nevow import rend, url, tags as T
 from allmydata.immutable.upload import FileHandle
-from allmydata.web.common import getxmlfile, get_arg, boolean_of_arg
+from allmydata.web.common import getxmlfile, get_arg, boolean_of_arg, \
+    get_key_arg, RandomKey
 from allmydata.web import status
 
 def PUTUnlinkedCHK(req, client):
     # "PUT /uri", to create an unlinked file.
-    uploadable = FileHandle(req.content, client.convergence)
+    key = get_key_arg(req)
+    if key is not None:
+        convergence = None
+        if key is RandomKey:
+            key = None
+    else:
+        convergence = client.convergence
+
+    uploadable = FileHandle(req.content, convergence=convergence, key=key)
     d = client.upload(uploadable)
     d.addCallback(lambda results: results.uri)
     # that fires with the URI of the new file
@@ -33 +42 @@
 
 def POSTUnlinkedCHK(req, client):
     fileobj = req.fields["file"].file
-    uploadable = FileHandle(fileobj, client.convergence)
+    key = get_key_arg(req)
+    if key is not None:
+        convergence = None
+        if key is RandomKey:
+            key = None
+    else:
+        convergence = client.convergence
+
+    uploadable = FileHandle(fileobj, convergence, key)
     d = client.upload(uploadable)
     when_done = get_arg(req, "when_done", None)
     if when_done: