Ticket #1074: unicode-args-and-rm-tahoe-exe.3.dpatch

Wed Jul 14 03:20:59 GMT Daylight Time 2010  david-sarah@jacaranda.org
  * Changes to Tahoe needed to work with new zetuptoolz (that does not use .exe wrappers on Windows), and to support Unicode arguments and stdout/stderr.

New patches:

[Changes to Tahoe needed to work with new zetuptoolz (that does not use .exe wrappers on Windows), and to support Unicode arguments and stdout/stderr.
david-sarah@jacaranda.org**20100714022059
 Ignore-this: bae64a497f95e834852de7466ca04066
] {
hunk ./bin/tahoe-script.template 8
 where = os.path.realpath(sys.argv[0])
 base = os.path.dirname(os.path.dirname(where))
 
+if sys.platform == "win32":
+    installed_tahoe = os.path.join(os.path.dirname(sys.executable), 'Scripts', 'tahoe.pyscript')
+else:
+    installed_tahoe = "/usr/bin/tahoe"
+
 whoami = '''\
 I am a "bin/tahoe" executable who is only for the convenience of running
 Tahoe from its source distribution -- I work only when invoked as the "tahoe"
hunk ./bin/tahoe-script.template 27
     print '''\
 I just tried to run and found that I am not living in such a directory, so I
 am stopping now. To run Tahoe after it has been is installed, please execute
-my brother, also named "tahoe", who gets installed into the appropriate place
-for executables when you run "make install" (perhaps as /usr/bin/tahoe).
-'''
+my brother, who gets installed into the appropriate place for executables
+when you run "make install" (perhaps as %s).
+''' % (installed_tahoe,)
     sys.exit(1)
 
 # we've found our home. Put the tahoe support/lib etc. in our PYTHONPATH.
hunk ./bin/tahoe-script.template 49
     pp = supportdir
 os.environ["PYTHONPATH"] = pp
 
-# find the location of the tahoe executable.
-bin_dir = "bin"
+# find commandline args and the location of the tahoe executable.
 if sys.platform == "win32":
hunk ./bin/tahoe-script.template 51
-    bin_dir = "Scripts"
-executable = os.path.join(base, "support", bin_dir, "tahoe")
+    import re
+    from ctypes import WINFUNCTYPE, POINTER, byref, c_wchar_p, c_int, windll
+
+    GetCommandLineW = WINFUNCTYPE(c_wchar_p)(("GetCommandLineW", windll.kernel32))
+    CommandLineToArgvW = WINFUNCTYPE(POINTER(c_wchar_p), c_wchar_p, POINTER(c_int)) \
+                            (("CommandLineToArgvW", windll.shell32))
+
+    argc = c_int(0)
+    argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc))
+
+    # See src/allmydata/scripts/runner.py for the corresponding unmangler.
+    def mangle(s):
+        return str(re.sub(r'[^\x20-\x7E]', lambda m: '\x7F%x;' % (ord(m.group(0)),), s))
+
+    argv = [mangle(argv_unicode[i]) for i in xrange(1, argc.value)]
+    local_tahoe = "Scripts\\tahoe.pyscript"
+else:
+    argv = sys.argv
+    local_tahoe = "bin/tahoe"
+
+script = os.path.join(base, "support", local_tahoe)
 
 try:
hunk ./bin/tahoe-script.template 74
-    res = subprocess.call([executable] + sys.argv[1:], env=os.environ)
+    res = subprocess.call([sys.executable, script] + argv[1:], env=os.environ)
 except (OSError, IOError), le:
     if le.args[0] == errno.ENOENT:
         print whoami
hunk ./bin/tahoe-script.template 78
+
         print '''\
 I just tried to run and could not find my brother, named
hunk ./bin/tahoe-script.template 81
-"../support/bin/tahoe". To run Tahoe when it is installed, please execute my
-brother, also named "tahoe", who gets installed into the appropriate place
-for executables when you run "make install" (perhaps as /usr/bin/tahoe).
-'''
+"../support/%s". To run Tahoe when it is installed, please execute my
+brother, who gets installed into the appropriate place for executables
+when you run "make install" (perhaps as %s).
+''' % (local_tahoe, installed_tahoe)
         raise
 except Exception, le:
     print whoami
hunk ./bin/tahoe-script.template 89
     print '''\
-I just tried to invoke my brother, named "../support/bin/tahoe" and got an
+I just tried to invoke my brother, named "../support/%s" and got an
 exception.
hunk ./bin/tahoe-script.template 91
-'''
+''' % (local_tahoe,)
     raise
 else:
     sys.exit(res)
hunk ./setup.py 238
     def run(self):
         bin_tahoe_template = os.path.join("bin", "tahoe-script.template")
 
-        # Create the 'tahoe-script.py' file under the 'bin' directory. The
-        # 'tahoe-script.py' file is exactly the same as the
-        # 'tahoe-script.template' script except that the shebang line is
-        # rewritten to use our sys.executable for the interpreter. On
-        # Windows, create a tahoe.exe will execute it. On non-Windows, make a
-        # symlink to it from 'tahoe'. The tahoe.exe will be copied from the
-        # setuptools egg's cli.exe and this will work from a zip-safe and
-        # non-zip-safe setuptools egg.
+        if sys.platform == 'win32':
+            # 'tahoe' script is needed for cygwin
+            script_names = ["tahoe.pyscript", "tahoe"]
+        else:
+            script_names = ["tahoe"]
+
+        # Create the tahoe script file under the 'bin' directory. This
+        # file is exactly the same as the 'tahoe-script.template' script
+        # except that the shebang line is rewritten to use our sys.executable
+        # for the interpreter.
         f = open(bin_tahoe_template, "rU")
         script_lines = f.readlines()
         f.close()
hunk ./setup.py 251
-        script_lines[0] = "#!%s\n" % sys.executable
-        tahoe_script = os.path.join("bin", "tahoe-script.py")
-        f = open(tahoe_script, "w")
-        for line in script_lines:
-            f.write(line)
-        f.close()
-        if sys.platform == "win32":
-            from pkg_resources import require
-            setuptools_egg = require("setuptools")[0].location
-            if os.path.isfile(setuptools_egg):
-                z = zipfile.ZipFile(setuptools_egg, 'r')
-                for filename in z.namelist():
-                    if 'cli.exe' in filename:
-                        cli_exe = z.read(filename)
-            else:
-                cli_exe = os.path.join(setuptools_egg, 'setuptools', 'cli.exe')
-            tahoe_exe = os.path.join("bin", "tahoe.exe")
-            if os.path.isfile(setuptools_egg):
-                f = open(tahoe_exe, 'wb')
-                f.write(cli_exe)
-                f.close()
-            else:
-                shutil.copy(cli_exe, tahoe_exe)
-        else:
+        script_lines[0] = '#!%s\n' % sys.executable
+        for script_name in script_names:
+            tahoe_script = os.path.join("bin", script_name)
             try:
hunk ./setup.py 255
-                os.remove(os.path.join('bin', 'tahoe'))
-            except:
+                os.remove(tahoe_script)
+            except Exception:
                 # okay, probably it was already gone
                 pass
hunk ./setup.py 259
-            os.symlink('tahoe-script.py', os.path.join('bin', 'tahoe'))
+            f = open(tahoe_script, "wb")
+            for line in script_lines:
+                f.write(line)
+            f.close()
+
+            # chmod +x
+            old_mode = stat.S_IMODE(os.stat(tahoe_script)[stat.ST_MODE])
+            new_mode = old_mode | (stat.S_IXUSR | stat.S_IRUSR |
+                                   stat.S_IXGRP | stat.S_IRGRP |
+                                   stat.S_IXOTH | stat.S_IROTH )
+            os.chmod(tahoe_script, new_mode)
 
hunk ./setup.py 271
-        # chmod +x bin/tahoe-script.py
-        old_mode = stat.S_IMODE(os.stat(tahoe_script)[stat.ST_MODE])
-        new_mode = old_mode | (stat.S_IXUSR | stat.S_IRUSR |
-                               stat.S_IXGRP | stat.S_IRGRP |
-                               stat.S_IXOTH | stat.S_IROTH )
-        os.chmod(tahoe_script, new_mode)
+        old_tahoe_exe = os.path.join("bin", "tahoe.exe")
+        try:
+            os.remove(old_tahoe_exe)
+        except Exception:
+            pass
+        if os.path.exists(old_tahoe_exe):
+            print >>sys.stderr, "%s could not be deleted. Please delete this file manually." % (old_tahoe_exe,)
 
 class MySdist(sdist.sdist):
     """ A hook in the sdist command so that we can determine whether this the
hunk ./src/allmydata/scripts/runner.py 12
 pkg_resources.require('allmydata-tahoe')
 from allmydata.scripts.common import BaseOptions
 from allmydata.scripts import debug, create_node, startstop_node, cli, keygen, stats_gatherer
+from allmydata.util.encodingutil import quote_output, get_argv_encoding, _reload
 
 def GROUP(s):
     # Usage.parseOptions compares argv[1] against command[0], so it will
hunk ./src/allmydata/scripts/runner.py 22
 
 
 class Options(BaseOptions, usage.Options):
-    synopsis = "Usage:  tahoe <command> [command options]"
+    synopsis = "\nUsage:  tahoe <command> [command options]"
     subCommands = ( GROUP("Administration")
                     +   create_node.subCommands
                     +   keygen.subCommands
hunk ./src/allmydata/scripts/runner.py 45
 
 def runner(argv,
            run_by_human=True,
-           stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr,
+           stdin=None, stdout=None, stderr=None,
            install_node_control=True, additional_commands=None):
 
hunk ./src/allmydata/scripts/runner.py 48
+    stdin  = stdin  or sys.stdin
+    stdout = stdout or sys.stdout
+    stderr = stderr or sys.stderr
+
     config = Options()
     if install_node_control:
         config.subCommands.extend(startstop_node.subCommands)
hunk ./src/allmydata/scripts/runner.py 70
         c = config
         while hasattr(c, 'subOptions'):
             c = c.subOptions
-        print str(c)
-        print "%s:  %s" % (sys.argv[0], e)
+        print >>stdout, str(c)
+        try:
+            msg = e.args[0].decode(get_argv_encoding())
+        except Exception:
+            msg = repr(e)
+        print >>stdout, "%s:  %s\n" % (sys.argv[0], quote_output(msg, quotemarks=False))
         return 1
 
     command = config.subCommand
hunk ./src/allmydata/scripts/runner.py 110
 
     return rc
 
+
 def run(install_node_control=True):
hunk ./src/allmydata/scripts/runner.py 112
-    rc = runner(sys.argv[1:])
+    if sys.platform == "win32":
+        windows_fixups()
+
+    rc = runner(sys.argv[1:], install_node_control=install_node_control)
     sys.exit(rc)
hunk ./src/allmydata/scripts/runner.py 117
+
+
+def windows_fixups():
+    import codecs, re
+    from ctypes import WINFUNCTYPE, windll, CFUNCTYPE, cdll, POINTER, byref, \
+        c_wchar_p, c_char_p, c_void_p, c_int, c_size_t
+
+    # Work around <http://bugs.python.org/issue6058>.
+    codecs.register(lambda name: name == 'cp65001' and codecs.lookup('utf-8') or None)
+
+    # Make Unicode console output work independently of the current code page.
+    # This also fixes <http://bugs.python.org/issue1602>.
+    try:
+        STDOUT_FILENO = 1
+        STDERR_FILENO = 2
+        fix_stdout = hasattr(sys.stdout, 'fileno') and sys.stdout.fileno() == STDOUT_FILENO
+        fix_stderr = hasattr(sys.stderr, 'fileno') and sys.stderr.fileno() == STDERR_FILENO
+
+        if fix_stdout or fix_stderr:
+            # FILE * _fdopen(int fd, const char *mode);
+            # #define _IOLBF 0x0040
+            # int setvbuf(FILE *stream, char *buffer, int mode, size_t size);
+            # #define _O_U8TEXT 0x40000
+            # int _setmode(int fd, int mode);
+            # int fputws(const wchar_t *ws, FILE *stream);
+            # int fflush(FILE *stream);
+
+            c_runtime = cdll.msvcrt
+            NULL = None
+            _fdopen = CFUNCTYPE(c_void_p, c_int, c_char_p)(("_fdopen", c_runtime))
+            _IOLBF = 0x0040
+            setvbuf = CFUNCTYPE(c_int, c_void_p, c_char_p, c_int, c_size_t)(("setvbuf", c_runtime))
+            _O_U8TEXT = 0x40000
+            _setmode = CFUNCTYPE(c_int, c_int, c_int)(("_setmode", c_runtime))
+            fputws = CFUNCTYPE(c_int, c_wchar_p, c_void_p)(("fputws", c_runtime));
+            fflush = CFUNCTYPE(c_int, c_void_p)(("fflush", c_runtime));
+
+            buffer_chars = 1024
+
+            class UnicodeOutput:
+                def __init__(self, fileno, name):
+                    self._stream = _fdopen(fileno, "w")
+                    assert self._stream is not NULL
+
+                    # Deep magic. MSVCRT supports writing wide-oriented output to stdout/stderr
+                    # to the console using the Unicode APIs, but it does the conversion in the
+                    # stdio buffer, so you need that buffer to be as large as the maximum amount
+                    # you're going to write in a single call (in bytes, not characters).
+                    setvbuf(self._stream, NULL, _IOLBF, buffer_chars*4 + 100)
+                    _setmode(fileno, _O_U8TEXT)
+
+                    self._fileno = fileno
+                    self.closed = False
+                    self.softspace = False
+                    self.mode = 'w'
+                    self.encoding = 'utf-8'
+                    self.name = name
+
+                def isatty(self):
+                    return False
+                def close(self):
+                    self.closed = True
+                    self.flush()
+                def fileno(self):
+                    return self._fileno
+                def flush(self):
+                    fflush(self._stream)
+
+                def write(self, text):
+                    if not isinstance(text, unicode):
+                        text = str(text).decode('utf-8')
+                    for i in xrange(0, len(text), buffer_chars):
+                        fputws(text[i:(i+buffer_chars)], self._stream)
+                        fflush(self._stream)
+
+                def writelines(self, lines):
+                    for line in lines:
+                        self.write(line)
+
+            if fix_stdout:
+                sys.stdout = UnicodeOutput(STDOUT_FILENO, '<Unicode stdout>')
+
+            if fix_stderr:
+                sys.stderr = UnicodeOutput(STDERR_FILENO, '<Unicode stderr>')
+
+            _reload()  # take account of new output encoding
+    except Exception, e:
+        print >>sys.stderr, "Warning: %r" % (e,)
+
+    # Unmangle command-line arguments.
+    GetCommandLineW = WINFUNCTYPE(c_wchar_p)(("GetCommandLineW", windll.kernel32))
+    CommandLineToArgvW = WINFUNCTYPE(POINTER(c_wchar_p), c_wchar_p, POINTER(c_int)) \
+                            (("CommandLineToArgvW", windll.shell32))
+
+    argc = c_int(0)
+    argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc))
+
+    # The cygwin wrapper script may append '\r' to each argument (bug in cygwin bash).
+    def unmangle(s):
+        return re.sub(ur'\x7f[0-9a-fA-F]*\;', lambda m: unichr(int(m.group(0)[1:-1], 16)),
+                      s.rstrip(u'\r'))
+
+    try:
+        sys.argv = [unmangle(argv_unicode[i]).encode('utf-8') for i in xrange(1, argc.value)]
+    except Exception, e:
+        print >>sys.stderr, "%s:  could not unmangle Unicode arguments" % (sys.argv[0],)
+        print >>sys.stderr, repr(e)
+        print >>sys.stderr, [argv_unicode[i] for i in xrange(1, argc.value)]
+        return 1
hunk ./src/allmydata/test/test_runner.py 16
 import allmydata
 
 bintahoe = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(allmydata.__file__))), 'bin', 'tahoe')
-if sys.platform == "win32":  # TODO: should this include cygwin?
-    bintahoe += ".exe"
+if sys.platform == "win32":
+    bintahoe += ".pyscript"
 
 
 class SkipMixin:
hunk ./src/allmydata/util/encodingutil.py 55
     output_encoding = _canonical_encoding(outenc)
 
     if sys.platform == 'win32':
-        # Unicode arguments are not supported on Windows yet; see #565 and #1074.
-        argv_encoding = 'ascii'
+        # On Windows we reencode the arguments as UTF-8; see scripts/runner.py.
+        argv_encoding = 'utf-8'
     else:
         argv_encoding = output_encoding
     is_unicode_platform = sys.platform in ["win32", "darwin"]
}

Context:

[Rename stringutils to encodingutil, and drop open_unicode (since the Python 'open' function works fine with Unicode paths).
david-sarah@jacaranda.org**20100713042715
 Ignore-this: fa2bb6b5d48ce9ba7ea2b1afc9f3b7b4
] 
[Resolve conflicts between NFC normalization changes, and post-1.7 branch.
david-sarah@jacaranda.org**20100618021642
 Ignore-this: ed3d0d71b761b1317cff9a4c92c2f5cb
] 
[trivial: a copy of tiny code-cleanups suggested by Kevan in reviewing #967
zooko@zooko.com**20100617045339
 Ignore-this: 274b3fdbf1531aa053f484a5d47d7447
] 
[minor code clean-up in dirnode.py
zooko@zooko.com**20100221052527
 Ignore-this: b01bfb47638f5a64256bb306e0022066
 Impose micro-POLA by passing only the writekey instead of the whole node object to {{{_encrypt_rw_uri()}}}. Remove DummyImmutableFileNode in nodemaker.py, which is obviated by this. Add micro-optimization by precomputing the netstring of the empty string and branching on whether the writekey is present or not outside of {{{_encrypt_rw_uri()}}}. Add doc about writekey to docstring.
] 
[Move EncryptedTemporaryFile from SFTP frontend to allmydata.util.fileutil, and make the FTP frontend also use it (fixing #1083).
david-sarah@jacaranda.org**20100711213721
 Ignore-this: e452e8ca66391aa2a1a49afe0114f317
] 
[Add tests of caps from the future that have non-ASCII characters in them (encoded as UTF-8). The changes to test_uri.py, test_client.py, and test_dirnode.py add tests of non-ASCII future caps in addition to the current tests. The changes to test_web.py just replace the tests of all-ASCII future caps with tests of non-ASCII future caps. We also change uses of failUnlessEqual to failUnlessReallyEqual, in order to catch cases where the type of a string is not as expected.
david-sarah@jacaranda.org**20100711200252
 Ignore-this: c2f193352369d32e06865f8f3e951894
] 
[Allow URIs passed in the initial JSON for t=mkdir-with-children, t=mkdir-immutable to be Unicode. Also pass the name of each child into nodemaker.create_from_cap for error reporting.
david-sarah@jacaranda.org**20100711195525
 Ignore-this: deac32d8b91ba26ede18905d3f7d2b93
] 
[docs/logging.txt: note that setting flogging vars might affect tests with race conditions.
david-sarah@jacaranda.org**20100712050721
 Ignore-this: fc1609d215fcd5561a57fd1226206f27
] 
[test_storage.py: potential fix for failures when logging is enabled.
david-sarah@jacaranda.org**19700713040546
 Ignore-this: 5815693a0df3e64c52c3c6b7be2846c7
] 
[SFTP: address some of the comments in zooko's review (#1106).
david-sarah@jacaranda.org**20100712025537
 Ignore-this: c3921638a2d4f1de2a776ae78e4dc37e
] 
[upcase_since_on_welcome
terrellrussell@gmail.com**20100708193903] 
[server_version_on_welcome_page.dpatch.txt
freestorm77@gmail.com**20100605191721
 Ignore-this: b450c76dc875f5ac8cca229a666cbd0a
 
 
 - The storage server version is 0 for all storage nodes in the Welcome Page
 
 
] 
[NEWS: add NEWS snippets about two recent patches
zooko@zooko.com**20100708162058
 Ignore-this: 6c9da6a0ad7351a960bdd60f81532899
] 
[directory_html_top_banner.dpatch
freestorm77@gmail.com**20100622205301
 Ignore-this: 1d770d975e0c414c996564774f049bca
 
 The div tag with the link "Return to Welcome page" on the directory.xhtml page is not correct
 
] 
[tahoe_css_toolbar.dpatch
freestorm77@gmail.com**20100622210046
 Ignore-this: 5b3ebb2e0f52bbba718a932f80c246c0
 
 CSS modification to be correctly diplayed with Internet Explorer 8
 
 The links on the top of page directory.xhtml are not diplayed in the same line as display with Firefox.
 
] 
[runnin_test_tahoe_css.dpatch
freestorm77@gmail.com**20100622214714
 Ignore-this: e0db73d68740aad09a7b9ae60a08c05c
 
 Runnin test for changes in tahoe.css file
 
] 
[runnin_test_directory_xhtml.dpatch
freestorm77@gmail.com**20100622201403
 Ignore-this: f8962463fce50b9466405cb59fe11d43
 
 Runnin test for diretory.xhtml top banner
 
] 
[stringutils.py: tolerate sys.stdout having no 'encoding' attribute.
david-sarah@jacaranda.org**20100626040817
 Ignore-this: f42cad81cef645ee38ac1df4660cc850
] 
[quickstart.html: python 2.5 -> 2.6 as recommended version
david-sarah@jacaranda.org**20100705175858
 Ignore-this: bc3a14645ea1d5435002966ae903199f
] 
[SFTP: don't call .stopProducing on the producer registered with OverwriteableFileConsumer (which breaks with warner's new downloader).
david-sarah@jacaranda.org**20100628231926
 Ignore-this: 131b7a5787bc85a9a356b5740d9d996f
] 
[docs/how_to_make_a_tahoe-lafs_release.txt: trivial correction, install.html should now be quickstart.html.
david-sarah@jacaranda.org**20100625223929
 Ignore-this: 99a5459cac51bd867cc11ad06927ff30
] 
[setup: in the Makefile, refuse to upload tarballs unless someone has passed the environment variable "BB_BRANCH" with value "trunk"
zooko@zooko.com**20100619034928
 Ignore-this: 276ddf9b6ad7ec79e27474862e0f7d6
] 
[trivial: tiny update to in-line comment
zooko@zooko.com**20100614045715
 Ignore-this: 10851b0ed2abfed542c97749e5d280bc
 (I'm actually committing this patch as a test of the new eager-annotation-computation of trac-darcs.)
] 
[docs: about.html link to home page early on, and be decentralized storage instead of cloud storage this time around
zooko@zooko.com**20100619065318
 Ignore-this: dc6db03f696e5b6d2848699e754d8053
] 
[docs: update about.html, especially to have a non-broken link to quickstart.html, and also to comment out the broken links to "for Paranoids" and "for Corporates"
zooko@zooko.com**20100619065124
 Ignore-this: e292c7f51c337a84ebfeb366fbd24d6c
] 
[TAG allmydata-tahoe-1.7.0
zooko@zooko.com**20100619052631
 Ignore-this: d21e27afe6d85e2e3ba6a3292ba2be1
] 
Patch bundle hash:
45b4a4597e7297ca2f8868ebd4d1a4d2f9dd9406