Refactor reST conversion code in setup.py and strip HTML.

.gitignore

@@ -5,7 +5,8 @@
 # Our tox runs convert the doctests in *.rst files to Python 3 prior to
 # running tests.  Ignore these temporary files.
 *.temp2to3.rst
-# The setup.py "prep" command converts *.md to *.temp.rst.
+# The setup.py "prep" command converts *.md to *.temp.rst (via *.temp.md).
+*.temp.md
 *.temp.rst
 # TextMate project file
 *.tmproj

setup.py

@@ -23,24 +23,25 @@
 
 It helps to review this auto-generated file on GitHub prior to uploading
 because the long description will be sent to PyPI and appear there after
-publishing.
+publishing.  PyPI attempts to convert this string to HTML before displaying
+it on the PyPI project page.  If PyPI finds any issues, it will render it
+instead as plain-text, which we do not want.
 
 To check in advance that PyPI will accept and parse the reST file as HTML,
 you can use the rst2html program installed by the docutils package
 (http://docutils.sourceforge.net/).  To install docutils:
 
     $ pip install docutils
 
-You will want to issue a command like the following and check that no
-warnings are issued:
+To check the file, run the following command and confirm that it reports
+no warnings:
 
     $ python setup.py --long-description | rst2html.py -v --no-raw > out.html
 
 See here for more information:
 
     http://docs.python.org/distutils/uploading.html#pypi-package-display
 
-
 (2) Push to PyPI.  To release a new version of Pystache to PyPI--
 
     http://pypi.python.org/pypi/pystache
@@ -119,9 +120,9 @@
 HISTORY_PATH = 'HISTORY.md'
 LICENSE_PATH = 'LICENSE'
 
-DESCRIPTION_PATH = 'setup_description.rst'
+RST_DESCRIPTION_PATH = 'setup_description.rst'
 
-TEMP_REST_EXTENSION = '.temp.rst'
+TEMP_EXTENSION = '.temp'
 
 PREP_COMMAND = 'prep'
 
@@ -141,7 +142,7 @@
 )
 
 # Comments in reST begin with two dots.
-LONG_DESCRIPTION_INTRO = """\
+RST_LONG_DESCRIPTION_INTRO = """\
 .. Do not edit this file.  This file is auto-generated for PyPI by setup.py
 .. using pandoc, so edits should go in the source files rather than here.
 """
@@ -168,7 +169,7 @@ def write(u, path):
     Write a unicode string to a file (as utf-8).
 
     """
-    print("Writing to: %s" % path)
+    print("writing to: %s" % path)
     # This function implementation was chosen to be compatible across Python 2/3.
     f = open(path, "wb")
     try:
@@ -178,36 +179,63 @@ def write(u, path):
         f.close()
 
 
-def make_temp_path(path):
+def make_temp_path(path, new_ext=None):
+    """
+    Arguments:
+
+      new_ext: the new file extension, including the leading dot.
+        Defaults to preserving the existing file extension.
+
+    """
     root, ext = os.path.splitext(path)
-    temp_path = root + TEMP_REST_EXTENSION
+    if new_ext is None:
+        new_ext = ext
+    temp_path = root + TEMP_EXTENSION + new_ext
     return temp_path
 
 
-def convert_md_to_rst(md_path):
-    """
-    Convert the given file from markdown to reStructuredText.
+def strip_html_comments(text):
+    """Strip HTML comments from a unicode string."""
+    lines = text.splitlines(True)  # preserve line endings.
+
+    # Remove HTML comments (which we only allow to take a special form).
+    new_lines = filter(lambda line: not line.startswith("<!--"), lines)
+
+    return "".join(new_lines)
 
-    Returns the converted text as a unicode string.
 
+# We write the converted file to a temp file to simplify debugging and
+# to avoid removing a valid pre-existing file on failure.
+def convert_md_to_rst(md_path, rst_temp_path):
     """
-    # We write the converted files to temp files to simplify debugging.
-    temp_path = make_temp_path(md_path)
-    print("Converting: %s to %s" % (md_path, temp_path))
+    Convert the contents of a file from Markdown to reStructuredText.
+
+    Returns the converted text as a Unicode string.
+
+    Arguments:
 
-    if os.path.exists(temp_path):
-        os.remove(temp_path)
+      md_path: a path to a UTF-8 encoded Markdown file to convert.
 
+      rst_temp_path: a temporary path to which to write the converted contents.
+
+    """
     # Pandoc uses the UTF-8 character encoding for both input and output.
-    command = "pandoc --write=rst --output=%s %s" % (temp_path, md_path)
+    command = "pandoc --write=rst --output=%s %s" % (rst_temp_path, md_path)
+    print("converting with pandoc: %s to %s\n-->%s" % (md_path, rst_temp_path,
+                                                       command))
+
+    if os.path.exists(rst_temp_path):
+        os.remove(rst_temp_path)
+
     os.system(command)
 
-    if not os.path.exists(temp_path):
+    if not os.path.exists(rst_temp_path):
         s = ("Error running: %s\n"
-             "  Did you install pandoc per the %s docstring?" % (command, __file__))
+             "  Did you install pandoc per the %s docstring?" % (command,
+                                                                 __file__))
         sys.exit(s)
 
-    return read(temp_path)
+    return read(rst_temp_path)
 
 
 # The long_description needs to be formatted as reStructuredText.
@@ -218,50 +246,58 @@ def convert_md_to_rst(md_path):
 #
 def make_long_description():
     """
-    Generate the long_description for setup() from source files.
+    Generate the reST long_description for setup() from source files.
 
-    Returns the long_description as a unicode string.
+    Returns the generated long_description as a unicode string.
 
     """
-    readme_section = convert_md_to_rst(README_PATH)
-    history_section = convert_md_to_rst(HISTORY_PATH)
+    readme_path = README_PATH
 
-    license_section = """\
+    # Remove our HTML comments because PyPI does not allow it.
+    # See the setup.py docstring for more info on this.
+    readme_md = strip_html_comments(read(readme_path))
+    history_md = strip_html_comments(read(HISTORY_PATH))
+    license_md = """\
 License
 =======
 
 """ + read(LICENSE_PATH)
 
-    sections = [LONG_DESCRIPTION_INTRO,
-                readme_section,
-                history_section,
-                license_section]
+    sections = [readme_md, history_md, license_md]
+    md_description = '\n\n'.join(sections)
+
+    # Write the combined Markdown file to a temp path.
+    md_ext = os.path.splitext(readme_path)[1]
+    md_description_path = make_temp_path(RST_DESCRIPTION_PATH, new_ext=md_ext)
+    write(md_description, md_description_path)
 
-    return '\n'.join(sections)
+    rst_temp_path = make_temp_path(RST_DESCRIPTION_PATH)
+    long_description = convert_md_to_rst(md_path=md_description_path,
+                                         rst_temp_path=rst_temp_path)
+
+    return "\n".join([RST_LONG_DESCRIPTION_INTRO, long_description])
 
 
 def prep():
+    """Update the reST long_description file."""
     long_description = make_long_description()
-    write(long_description, DESCRIPTION_PATH)
+    write(long_description, RST_DESCRIPTION_PATH)
 
 
 def publish():
-    """
-    Publish this package to PyPI (aka "the Cheeseshop").
-
-    """
+    """Publish this package to PyPI (aka "the Cheeseshop")."""
     long_description = make_long_description()
 
-    if long_description != read(DESCRIPTION_PATH):
+    if long_description != read(RST_DESCRIPTION_PATH):
         print("""\
 Description file not up-to-date: %s
 Run the following command and commit the changes--
 
     python setup.py %s
-""" % (DESCRIPTION_PATH, PREP_COMMAND))
+""" % (RST_DESCRIPTION_PATH, PREP_COMMAND))
         sys.exit()
 
-    print("Description up-to-date: %s" % DESCRIPTION_PATH)
+    print("Description up-to-date: %s" % RST_DESCRIPTION_PATH)
 
     answer = raw_input("Are you sure you want to publish to PyPI (yes/no)?")
 
@@ -339,7 +375,7 @@ def main(sys_argv):
         prep()
         sys.exit()
 
-    long_description = read(DESCRIPTION_PATH)
+    long_description = read(RST_DESCRIPTION_PATH)
     template_files = ['*.mustache', '*.txt']
     extra_args = get_extra_args()