A place to cache linked articles (think custom and personal wayback machine)
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

index.html 50KB

  1. <!doctype html><!-- This is a valid HTML5 document. -->
  2. <!-- Screen readers, SEO, extensions and so on. -->
  3. <html lang="fr">
  4. <!-- Has to be within the first 1024 bytes, hence before the <title>
  5. See: https://www.w3.org/TR/2012/CR-html5-20121217/document-metadata.html#charset -->
  6. <meta charset="utf-8">
  7. <!-- Why no `X-UA-Compatible` meta: https://stackoverflow.com/a/6771584 -->
  8. <!-- The viewport meta is quite crowded and we are responsible for that.
  9. See: https://codepen.io/tigt/post/meta-viewport-for-2015 -->
  10. <meta name="viewport" content="width=device-width,initial-scale=1">
  11. <!-- Required to make a valid HTML5 document. -->
  12. <title>Mercurial's Journey to and Reflections on Python 3 (archive) — David Larlet</title>
  13. <meta name="description" content="Publication mise en cache pour en conserver une trace.">
  14. <!-- That good ol' feed, subscribe :). -->
  15. <link rel="alternate" type="application/atom+xml" title="Feed" href="/david/log/">
  16. <!-- Generated from https://realfavicongenerator.net/ such a mess. -->
  17. <link rel="apple-touch-icon" sizes="180x180" href="/static/david/icons2/apple-touch-icon.png">
  18. <link rel="icon" type="image/png" sizes="32x32" href="/static/david/icons2/favicon-32x32.png">
  19. <link rel="icon" type="image/png" sizes="16x16" href="/static/david/icons2/favicon-16x16.png">
  20. <link rel="manifest" href="/static/david/icons2/site.webmanifest">
  21. <link rel="mask-icon" href="/static/david/icons2/safari-pinned-tab.svg" color="#07486c">
  22. <link rel="shortcut icon" href="/static/david/icons2/favicon.ico">
  23. <meta name="msapplication-TileColor" content="#f0f0ea">
  24. <meta name="msapplication-config" content="/static/david/icons2/browserconfig.xml">
  25. <meta name="theme-color" content="#f0f0ea">
  26. <!-- Documented, feel free to shoot an email. -->
  27. <link rel="stylesheet" href="/static/david/css/style_2020-06-19.css">
  28. <!-- See https://www.zachleat.com/web/comprehensive-webfonts/ for the trade-off. -->
  29. <link rel="preload" href="/static/david/css/fonts/triplicate_t4_poly_regular.woff2" as="font" type="font/woff2" media="(prefers-color-scheme: light), (prefers-color-scheme: no-preference)" crossorigin>
  30. <link rel="preload" href="/static/david/css/fonts/triplicate_t4_poly_bold.woff2" as="font" type="font/woff2" media="(prefers-color-scheme: light), (prefers-color-scheme: no-preference)" crossorigin>
  31. <link rel="preload" href="/static/david/css/fonts/triplicate_t4_poly_italic.woff2" as="font" type="font/woff2" media="(prefers-color-scheme: light), (prefers-color-scheme: no-preference)" crossorigin>
  32. <link rel="preload" href="/static/david/css/fonts/triplicate_t3_regular.woff2" as="font" type="font/woff2" media="(prefers-color-scheme: dark)" crossorigin>
  33. <link rel="preload" href="/static/david/css/fonts/triplicate_t3_bold.woff2" as="font" type="font/woff2" media="(prefers-color-scheme: dark)" crossorigin>
  34. <link rel="preload" href="/static/david/css/fonts/triplicate_t3_italic.woff2" as="font" type="font/woff2" media="(prefers-color-scheme: dark)" crossorigin>
  35. <script type="text/javascript">
  36. function toggleTheme(themeName) {
  37. document.documentElement.classList.toggle(
  38. 'forced-dark',
  39. themeName === 'dark'
  40. )
  41. document.documentElement.classList.toggle(
  42. 'forced-light',
  43. themeName === 'light'
  44. )
  45. }
  46. const selectedTheme = localStorage.getItem('theme')
  47. if (selectedTheme !== 'undefined') {
  48. toggleTheme(selectedTheme)
  49. }
  50. </script>
  51. <meta name="robots" content="noindex, nofollow">
  52. <meta content="origin-when-cross-origin" name="referrer">
  53. <!-- Canonical URL for SEO purposes -->
  54. <link rel="canonical" href="https://gregoryszorc.com/blog/2020/01/13/mercurial%27s-journey-to-and-reflections-on-python-3/">
  55. <body class="remarkdown h1-underline h2-underline h3-underline hr-center ul-star pre-tick">
  56. <article>
  57. <header>
  58. <h1>Mercurial's Journey to and Reflections on Python 3</h1>
  59. </header>
  60. <nav>
  61. <p class="center">
  62. <a href="/david/" title="Aller à l’accueil">🏠</a> •
  63. <a href="https://gregoryszorc.com/blog/2020/01/13/mercurial%27s-journey-to-and-reflections-on-python-3/" title="Lien vers le contenu original">Source originale</a>
  64. </p>
  65. </nav>
  66. <hr>
  67. <main>
  68. <p>Mercurial 5.2 was released on November 5, 2019. It is the first version
  69. of Mercurial that supports Python 3. This milestone comes nearly 11 years
  70. after Python 3.0 was first released on December 3, 2008.</p>
  71. <p>Speaking as a maintainer of Mercurial and an avid user of Python, I
  72. feel like the experience of making Mercurial work with Python 3 is
  73. worth sharing because there are a number of lessons to be learned.</p>
  74. <p>This post is logically divided into two sections: a mostly factual recount
  75. of Mercurial's Python 3 porting effort and a more opinionated commentary
  76. of the transition to Python 3 and the Python language ecosystem as a whole.
  77. Those who don't care about the mechanics of porting a large Python project
  78. to Python 3 may want to skip the next section or two.</p>
  79. <h2>Porting Mercurial to Python 3</h2>
  80. <p>Let's start with a brief history lesson of Mercurial's support for
  81. Python 3 as told by its own commit history.</p>
  82. <p>The Mercurial version control tool was first released in April 2005
  83. (the same month that Git was initially released). Version 1.0 came out
  84. in March 2008. The first reference to Python 3 I found in the code base
  85. was in <a href="https://www.mercurial-scm.org/repo/hg/rev/8fee8ff13d37">September 2008</a>.
  86. Then not much happens for a while until
  87. <a href="https://www.mercurial-scm.org/repo/hg/rev/4494fb02d549">June 2010</a>, when
  88. someone authors a bunch of changes to make the Python C extensions
  89. start to recognize Python 3. Then things were again quiet for a while
  90. until <a href="https://www.mercurial-scm.org/repo/hg/rev/56ef99fbd6f2">January 2013</a>,
  91. when a handful of changes landed to remove 2 argument <code>raise</code>. There were
  92. a handful of commits in 2014 but nothing worth calling out.</p>
  93. <p>Mercurial's meaningful journey to Python 3 started in 2015. In code,
  94. the work started in
  95. <a href="https://www.mercurial-scm.org/repo/hg/rev/af6e6a0781d7">April 2015</a>, with
  96. effort to make Mercurial's test harness run with Python 3. Part of
  97. this was a <a href="https://www.mercurial-scm.org/repo/hg/rev/fefc72523491">decision</a>
  98. that Python 3.5 (to be released several months later in September 2015)
  99. would be the minimum Python 3 version that Mercurial would support.</p>
  100. <p>Once the Mercurial Project decided it wanted to port to Python 3 (as opposed
  101. to another language), one of the earliest decisions was how to perform that
  102. port. <strong>Mercurial's code base was too large to attempt a flag day conversion</strong>
  103. where there would be a Python 2 version and a Python 3 version and one day
  104. everyone would switch from Python 2 to 3. <strong>Mercurial needed a way to run the
  105. same code (or as much of the same code) on both Python 2 and 3.</strong> We would
  106. maintain a single code base and users would gradually switch from running with
  107. Python 2 to Python 3.</p>
  108. <p>In <a href="https://www.mercurial-scm.org/repo/hg/rev/e1fb276d4619">May 2015</a>,
  109. Mercurial dropped support for Python 2.4 and 2.5. Dropping support for
  110. these older Python versions was critical, as it was effectively impossible to
  111. write Python code that ran on this wide gamut of versions because of
  112. incompatibilities in syntax and language features. For example, you needed
  113. Python 2.6 to get <code>print()</code> via <code>from __future__ import print_function</code>.
  114. The project's late start at a Python 3 port can be significantly attributed
  115. to Python 2.4 and 2.5 compatibility holding us back.</p>
  116. <p>The main goal with Mercurial's early porting work was just getting the code base
  117. to a point where <code>import mercurial</code> would work. There were a myriad of places
  118. where Mercurial used syntax that was invalid on Python 3 and Python 3
  119. couldn't even parse the source code, let alone compile it to bytecode and
  120. execute it.</p>
  121. <p>This effort began in earnest in
  122. <a href="https://www.mercurial-scm.org/repo/hg/rev/e93036747902">June 2015</a>
  123. with global source code rewrites like using modern octal syntax,
  124. modern exception catching syntax (<code>except Exception as e</code> instead of
  125. <code>except Exception, e</code>), <code>print()</code> instead of <code>print</code>, and a
  126. <a href="https://www.mercurial-scm.org/repo/hg/rev/1a6a117d0b95">modern import convention</a>
  127. along with the use of <code>from __future__ import absolute_import</code>.</p>
  128. <p>In the early days of the port, our first goal was to get all source code
  129. parsing as valid Python 3. The next step was to get all the modules <code>import</code>ing
  130. cleanly. This entailed fixing code that ran at <code>import</code> time to work on
  131. Python 3. Our thinking was that we would need the code base to be <code>import</code>
  132. clean on Python 3 before seriously thinking about run-time behavior. In reality,
  133. we quickly ported a lot of modules to <code>import</code> cleanly and then moved on
  134. to higher-level porting, leaving a long-tail of modules with <code>import</code> failures.</p>
  135. <p>This initial porting effort played out over months. There weren't many
  136. people working on it in the early days: a few people would basically hack on
  137. Python 3 as a form of itch scratching and most of the project's energy was
  138. focused on improving the existing Python 2 based product. You can get a rough
  139. idea of the timeline and participation in the early porting effort through the
  140. <a href="https://www.mercurial-scm.org/repo/hg/log/081a77df7bc6/tests/test-check-py3-compat.t?revcount=960">history of test-check-py3-compat.t</a>.
  141. We see the test being added in <a href="https://www.mercurial-scm.org/repo/hg/rev/40eb385f798f">December 2015</a>,
  142. By June 2016, most of the code base was ported to our modern import convention
  143. and we were ready to move on to more meaningful porting.</p>
  144. <p>One of the biggest early hurdles in our porting effort was how to overcome
  145. the string literals type mismatch between Python 2 and 3. In Python 2, a
  146. <code>''</code> string literal is a sequence of bytes. In Python 3, a <code>''</code> string literal
  147. is a sequence of Unicode code points. These are fundamentally different types.
  148. And in Mercurial's code base, <strong>most of our <em>string</em> types are binary by design:
  149. use of a Unicode based <code>str</code> for representing data is flat out wrong for our use
  150. case</strong>. We knew that Mercurial would need to eventually switch many string
  151. literals from <code>''</code> to <code>b''</code> to preserve type compatibility. But doing so would
  152. be problematic.</p>
  153. <p>In the early days of Mercurial's Python 3 port in 2015, Mercurial's project
  154. maintainer (Matt Mackall) set a ground rule that the Python 3 port shouldn't overly
  155. disrupt others: he wanted the Python 3 port to more or less happen in the background
  156. and not require every developer to be aware of Python 3's low-level behavior in order
  157. to get work done on the existing Python 2 code base. This may seem like a questionable
  158. decision (and I probably disagreed with him to some extent at the time because I was
  159. doing Python 3 porting work and the decision constrained this work). But it was the
  160. correct decision. Matt knew that it would be years before the Python 3 port was either
  161. necessary or resulted in a meaningful return on investment (the value proposition of
  162. Python 3 has always been weak to Mercurial because Python 3 doesn't demonstrate a
  163. compelling advantage over Python 2 for our use case). What Matt was trying to do was
  164. minimize the externalized costs that a Python 3 port would inflict on the project.
  165. He correctly recognized that maintaining the existing product and supporting
  166. existing users was more important than a long-term bet in its infancy.</p>
  167. <p>This ground rule meant that a mass insertion of <code>b''</code> prefixes everywhere
  168. was not desirable, as that would require developers to think about whether
  169. a type was a <code>bytes</code> or <code>str</code>, a distinction they didn't have to worry about
  170. on Python 2 because we practically never used the Unicode-based string type in
  171. Mercurial.</p>
  172. <p>In addition, there were some other practical issues with doing a bulk <code>b''</code>
  173. prefix insertion. One was that the added <code>b</code> characters would cause a lot of lines
  174. to grow beyond our length limits and we'd have to reformat code. That would
  175. require manual intervention and would significantly slow down porting. And
  176. a sub-issue of adding all the <code>b</code> prefixes and reformatting code is that it would
  177. <em>break</em> annotate/blame more than was tolerable. The latter issue was addressed
  178. by teaching Mercurial's annotate/blame feature to <em>skip</em> revisions. The project
  179. now has a convention of annotating commit messages with <code># skip-blame &lt;reason&gt;</code>
  180. so structural only changes can easily be ignored when performing an
  181. annotate/blame.</p>
  182. <p>A stop-gap solution to the <code>b''</code> everywhere issue came in
  183. <a href="https://www.mercurial-scm.org/repo/hg/rev/1c22400db72d">July 2016</a>, when I
  184. introduced a custom Python module importer that rewrote source code as part
  185. of <code>import</code> when running on Python 3. (I have
  186. <a href="/blog/2017/03/13/from-__past__-import-bytes_literals/">previously blogged</a>
  187. about this hack.) What this did was transparently add <code>b''</code> prefixes to all
  188. un-prefixed string literals as well as modify how a few common functions were
  189. called so that we wouldn't need to modify source code so things would run natively
  190. on Python 3. The source transformer allowed us to have the benefits of progressing
  191. in our Python 3 port without having to rewrite tens of thousands of lines of
  192. source code. The solution was hacky. But it enabled us to make significant
  193. progress on the Python 3 port without externalizing a lot of cost onto others.</p>
  194. <p>I thought the source transformer would be relatively short-lived and would be
  195. removed shortly after the project inevitably decided to go all in on Python 3.
  196. To my surprise, others built additional transforms over the years and the source
  197. transformer persisted all the way until
  198. <a href="https://www.mercurial-scm.org/repo/hg/rev/d783f945a701">October 2019</a>, when
  199. I removed it just before the first non-alpha Python 3 compatible version
  200. of Mercurial was released.</p>
  201. <p>A common problem Mercurial faced with making the code base dual Python 2/3 native
  202. was dealing with standard library differences. Most of the problems stemmed
  203. from changes between Python 2.7 and 3.5+. But there are changes within the
  204. versions of Python 3 that we had to wallpaper over as well. In
  205. <a href="https://www.mercurial-scm.org/repo/hg/rev/6041fb8f2da8">April 2016</a>, the
  206. <code>mercurial.pycompat</code> module was introduced to export aliases or wrappers around
  207. standard library functionality to abstract the differences between Python
  208. versions. This file <a href="https://www.mercurial-scm.org/repo/hg/log/66af68d4c751/mercurial/pycompat.py?revcount=240">grew over time</a>
  209. and <a href="https://www.mercurial-scm.org/repo/hg/file/66af68d4c751/mercurial/pycompat.py">eventually became</a>
  210. Mercurial's version of <a href="https://six.readthedocs.io/">six</a>. To be honest, I'm
  211. not sure if we should have used <code>six</code> from the beginning. <code>six</code> probably would
  212. have saved some work. But we had to eventually write a lot of shims for
  213. converting between <code>str</code> and <code>bytes</code> and would have needed to invent a
  214. <code>pycompat</code> layer in some form anyway. So I'm not sure <code>six</code> would have saved
  215. enough effort to justify the baggage of integrating a 3rd party package into
  216. Mercurial. (When Mercurial accepts a 3rd party package, downstream packagers
  217. like Debian get all hot and bothered and end up making questionable patches
  218. to our source code. So we prefer to minimize the surface area for
  219. problems by minimizing dependencies on 3rd party packages.)</p>
  220. <p>Once we had a source transforming module importer and the <code>pycompat</code>
  221. compatibility shim, we started to focus in earnest on making core
  222. functionality actually work on Python 3. We established a convention of
  223. annotating changesets needed for Python 3 with <code>py3</code>, so a
  224. <a href="https://www.mercurial-scm.org/repo/hg/log?rev=desc(py3)&amp;revcount=4000">commit message search</a>
  225. yields a lot of the history. (But it isn't a full history since not every Python 3
  226. oriented change used this convention). We see from that history that after
  227. the source importer landed, a lot of porting effort was spent on things
  228. very early in the <code>hg</code> process lifetime. This included handling environment
  229. variables, loading config files, and argument parsing. We introduced a
  230. <a href="https://www.mercurial-scm.org/repo/hg/log/@/tests/test-check-py3-commands.t">test-check-py3-commands.t</a>
  231. test to track the progress of <code>hg</code> commands working in Python 3. The very early
  232. history of that file shows the various error messages changing, as underlying
  233. early process functionality was slowly ported to work on Python 3. By
  234. <a href="https://www.mercurial-scm.org/repo/hg/rev/2d555d753f0e">December 2016</a>, we
  235. had <code>hg version</code> working on Python 3!</p>
  236. <p>With basic <code>hg</code> command dispatch ported to Python 3 at the end of 2016,
  237. 2017 represented an inflection point in the Python 3 porting effort. With the
  238. early process functionality working, different people could pick up different
  239. commands and code paths and start making code work with Python 3. By
  240. <a href="https://www.mercurial-scm.org/repo/hg/rev/52ee1b5ac277">March 2017</a>, basic
  241. repository opening and <code>hg files</code> worked. Shortly thereafter,
  242. <a href="https://www.mercurial-scm.org/repo/hg/rev/ed23f929af38">hg init started working as well</a>.
  243. And <a href="https://www.mercurial-scm.org/repo/hg/rev/935a1b1117c7">hg status</a> and
  244. <a href="https://www.mercurial-scm.org/repo/hg/rev/aea8ec3f7dd1">hg commit</a> did as well.</p>
  245. <p>Within a few months, enough of Mercurial's functionality was working with Python
  246. 3 that we started to <a href="https://www.mercurial-scm.org/repo/hg/rev/7a877e569ed6">track which tests passed on Python 3</a>.
  247. The <a href="https://www.mercurial-scm.org/repo/hg/log/@/contrib/python3-whitelist?revcount=480">evolution of this file</a>
  248. shows a reasonable history of the porting velocity.</p>
  249. <p>In <a href="https://www.mercurial-scm.org/repo/hg/rev/feb910d2f59b">May 2017</a>, we dropped
  250. support for Python 2.6. This significantly reduced the complexity of supporting
  251. Python 3, as there was tons of functionality in Python 2.7 that made it easier
  252. to target both Python 2 and 3 and now our hands were untied to utilize it.</p>
  253. <p>In <a href="https://www.mercurial-scm.org/repo/hg/rev/bd8875b6473c">November 2017</a>, I
  254. landed a test harness feature to report exceptions seen during test runs. I
  255. later <a href="https://www.mercurial-scm.org/repo/hg/rev/8de90e006c78">refined the output</a>
  256. so the most frequent failures were reported more prominently. This feature
  257. greatly enabled our ability to target the most common exceptions, allowing
  258. us to write patches to fix the most prevalent issues on Python 3 and uncover
  259. previously unknown failures.</p>
  260. <p>By the end of 2017, we had most of the structural pieces in place to complete
  261. the port. Essentially all that was required at that point was time and labor.
  262. We didn't have a formal mechanism in place to target porting efforts. Instead,
  263. people would pick up a component or test that they wanted to hack on and then
  264. make incremental changes towards making that work. All the while, we didn't
  265. have a strict policy on not regressing Python 3 and regressions in Python 3
  266. porting progress were semi-frequent. Although we did tend to correct
  267. regressions quickly. And over time, developers saw a flurry of Python 3
  268. patches and slowly grew awareness of how to accommodate Python 3, and the
  269. number of Python 3 regressions became less frequent.</p>
  270. <p>As useful as the source-transforming module importer was, it incurred some
  271. additional burden for the porting effort. The source transformer effectively
  272. converted all un-prefixed string literals (<code>''</code>) to bytes literals (<code>b''</code>)
  273. to preserve string type behavior with Python 2. But various aspects of Python
  274. 3 didn't like the existence of <code>bytes</code>. Various standard library functionality
  275. now wanted unicode <code>str</code> and didn't accept <code>bytes</code>, even though the Python
  276. 2 implementation used the equivalent of <code>bytes</code>. So our <code>pycompat</code> layer
  277. grew pretty large to accommodate calling into various standard library
  278. functionality. Another side-effect which we didn't initially anticipate
  279. was the <code>**kwargs</code> calling convention. Python allows you to use <code>**</code>
  280. with a dict with string keys to turn those keys into named arguments
  281. in a function call. But Python 3 requires these <code>dict</code> keys to be
  282. <code>str</code> and outright rejects <code>bytes</code> keys, even if the <code>bytes</code> instance
  283. is ASCII safe and has the same underlying byte representation of the
  284. string data as the <code>str</code> instance would. So we had to invent support
  285. functions that would convert <code>dict</code> keys from <code>bytes</code> to <code>str</code> for
  286. use with <code>**kwargs</code> and another to convert a <code>**kwargs</code> dict from
  287. <code>str</code> keys to <code>bytes</code> keys so we could use <code>''</code> syntax to access keys
  288. in our source code! Also on the string type front, we had to sprinkle
  289. the codebase with raw string literals (<code>r''</code>) to force the use of
  290. <code>str</code> irregardless of which Python version you were running on (our
  291. source transformer only changed unprefixed string literals, so existing
  292. <code>r''</code> strings would be preserved as <code>str</code>).</p>
  293. <p>Blind transformation of all string literals to <code>bytes</code> was less than ideal
  294. and it did impose some unwanted side-effects. But, again, most <em>strings</em>
  295. in Mercurial are bytes by design, so we thought it would be easier to
  296. <em>byteify</em> all strings then selectively undo that where native strings
  297. were actually warranted (like keys in most <code>dict</code>s) than to take the
  298. up-front cost to examine every string and make an intelligent determination
  299. as to what type it should be. I go back and forth as to whether this was the
  300. correct call. But when you factor in that the source transforming
  301. module importer unblocked Python 3 porting at a time in the project's
  302. history when there was so much focus on improving the core product and it
  303. did so without externalizing many costs onto the people doing the critical
  304. core product work, I think it was the right call.</p>
  305. <p>By mid 2019, the number of test failures in Python 3 had been whittled
  306. down to a reasonable, less daunting number. It felt like victory was
  307. in grasp and inevitable. But a few significant issues lingered.</p>
  308. <p>One remaining question was around addressing differences between Python
  309. 3 versions. At the time, Python 3.5, 3.6, and 3.7 were released and 3.8
  310. was scheduled for release by the end of the year. We had a surprising
  311. number of issues with differences in Python 3 versions. Many of us
  312. were running Python 3.7, so it had the fewest failures. We had to spend
  313. extra effort to get Python 3.5 and 3.6 working as well as 3.7. Same for
  314. 3.8.</p>
  315. <p>Another task we deferred until the second half of 2019 was standing up
  316. robust CI for Python 3. We had some coverage, but it was minimal. Wanting
  317. a distraction from PyOxidizer for a bit and wanting to overhaul Mercurial's
  318. CI system (which is officially built on Buildbot), I cobbled together a
  319. <em>serverless</em> CI system built on top of AWS DynamoDB and S3 for storage,
  320. Lambda functions and CloudWatch events for all business logic, and EC2 spot
  321. instances for job execution. This CI system executed Python 3.5, 3.6, 3.7,
  322. and 3.8 variants of our test harness on Linux and Python 3.7 on Windows.
  323. This gave developers insight into version-specific failures. More
  324. importantly, it also gave insight into Windows failures, which was
  325. previously not well tested. It was discovered that Python 3 on Windows was
  326. lagging significantly behind POSIX.</p>
  327. <p>By the time of the Mercurial developer meetup in October 2019, nearly
  328. all tests were passing on POSIX platforms and we were confident that
  329. we could declare Python 3 support as at least beta quality for the
  330. Mercurial 5.2 release, planned for early November.</p>
  331. <p>One of our blockers for ripping off the alpha label on Python 3 support
  332. was removing our source-transforming module importer. It had performance
  333. implications and it wasn't something we wanted to ship because it felt
  334. too hacky. A blocker for this was we wanted to automatically format
  335. our source tree with <a href="https://black.readthedocs.io/en/stable/">black</a>
  336. because if we removed the source transformer, we'd have to rewrite
  337. a lot of source code to apply changes the transformer was performing,
  338. which would necessitate wrapping a lot of lines, which would involve a lot
  339. of manual effort. We wanted to <em>blacken</em> our code base first so that
  340. mass rewriting source code wouldn't involve a lot of tedious reformatting
  341. since <code>black</code> would handle that for us automatically. And rewriting the
  342. source tree with <code>black</code> was blocked on a specific feature landing in
  343. <code>black</code>! (We did not agree with <code>black</code>'s behavior of
  344. unwrapping comma-delimited lists of items if they could fit on a single
  345. line. So one of our core contributors wrote a patch to <code>black</code> that
  346. changed its behavior so a trailing <code>,</code> in a list of items will force
  347. items to be formatted on multiple lines. I personally find the multiple line
  348. formatting much easier to read. And the behavior is arguably better for
  349. code review and <em>annotation</em>, which is line based.) Once this feature
  350. landed in <code>black</code>, we reformatted our source tree and started ripping
  351. out the source transformations, starting by inserting <code>b''</code> literals
  352. everywhere. By late October, the source transformer was no more and
  353. we were ready to release beta quality support for Python 3 (at least
  354. on UNIX-like platforms).</p>
  355. <p>Having described a mostly factual overview of Mercurial's port to Python
  356. 3, it is now time to shift gears to the speculative and opinionated
  357. parts of this post. <strong>I want to underscore that the opinions reflected
  358. here are my own and do not reflect the overall Mercurial Project or even
  359. a consensus within it.</strong></p>
  360. <h2>The Future of Python 3 and Mercurial</h2>
  361. <p>Mercurial's port to Python 3 is still ongoing. While we've shipped
  362. Python 3 support and the test harness is clean on Python 3, I view shipping
  363. as only a milestone - arguably <em>the</em> most important one - in a longer
  364. journey. There's still a lot of work to do.</p>
  365. <p>It is now 2020 and Python 2 support is now officially dead from the
  366. perspective of the Python language maintainers. Linux distributions are
  367. starting to rip out Python 2. Packages are dropping Python 2 support in
  368. new versions. The world is moving to Python 3 only. But <strong>Mercurial still
  369. officially supports Python 2</strong>. And it is still yet to be determined how
  370. long we will retain support for Python 2 in the code base. We've only had
  371. one release supporting Python 3. Our users still need to port their
  372. extensions (implemented in Python). Our users still need to start widely
  373. using Mercurial with Python 3. Even our own developers need to switch to
  374. Python 3 (old habits are hard to break).</p>
  375. <p>I anticipate a long tail of random bugs in Mercurial on Python 3. While
  376. the tests may pass, our code coverage is not 100%. And even if it were,
  377. Python is a dynamic language and there are tons of invariants that aren't
  378. caught at compile time and can only be discovered at run time. <strong>These
  379. invariants cannot all be detected by tests, no matter how good your test
  380. coverage is.</strong> This is a <em>feature</em>/<em>limitation</em> of dynamic languages. Our
  381. users will likely be finding a long tail of miscellaneous bugs on Python
  382. 3 for <em>years</em>.</p>
  383. <p>At present, our code base is littered with tons of random hacks to bridge
  384. the gap between Python 2 and 3. Once Python 2 support is dropped, we'll
  385. need to remove these hacks and make the source tree Python 3 native, with
  386. minimal shims to wallpaper over differences in Python 3 versions. <strong>Removing
  387. this Python version bridge code will likely require hundreds of commits and
  388. will be a non-trivial effort.</strong> It's likely to be deemed a low priority (it
  389. is glorified busy work after all), and code for the express purpose of
  390. supporting Python 2 will likely linger for years.</p>
  391. <p>We are also still shoring up our packaging and distribution story on
  392. Python 3. This is easier on some platforms than others. I created
  393. <a href="https://github.com/indygreg/PyOxidizer">PyOxidizer</a> partially because
  394. of the poor experience I had with Python application packaging and
  395. distribution through the Mercurial Project. The Mercurial Project has
  396. already signed off on using PyOxidizer for distributing Mercurial in
  397. the future. So look for an <em>oxidized</em> Mercurial distribution in the
  398. near future! (You could argue PyOxidizer is an epic yak shave to better
  399. support Mercurial. But that's for another post.)</p>
  400. <p>Then there's Windows support. A Python 3 powered Mercurial on Windows
  401. still has a handful of known issues. It may require a few more releases
  402. before we consider Python 3 on Windows to be stable.</p>
  403. <p>Because we're still on a code base that must support Python 2, our
  404. adoption of Python 3 features is very limited. The only Python 3
  405. feature that Mercurial developers seem to almost universally get excited
  406. about is type annotations. We already have some people playing around
  407. with <code>pytype</code> using comment-based annotations and <code>pytype</code> has already
  408. caught a few bugs. We're eager to go all in on type annotations and
  409. uncover lots of dynamic typing bugs and poorly implemented APIs.
  410. Beyond type annotations, I can't name any feature that people are screaming
  411. to adopt and which makes a lot of sense for Mercurial. There's a long
  412. tail of minor features I'm sure will get utilized. But none of the
  413. marquee features that define major language releases seem that interesting
  414. to us. Time will tell.</p>
  415. <h2>Commentary on Python 3</h2>
  416. <p>Having described Mercurial's ongoing journey to Python 3, I now want to
  417. focus more on Python itself. Again, the opinions here are my own and
  418. don't reflect those of the Mercurial Project.</p>
  419. <p><strong>Succinctly, my experience porting Mercurial and other projects to
  420. Python 3 has significantly soured my perceptions of Python. As much as
  421. I have historically loved Python - from the language to the welcoming
  422. community - I am still struggling to understand how Python could manage
  423. to inflict so much hardship on the community by choosing the transition
  424. plan that they did.</strong> I believe Python's choices represent a terrific
  425. example of what not to do when managing a large project or ecosystem.
  426. Maintainers of other largely-deployed systems would benefit from taking
  427. the time to understand and reflect on Python's missteps.</p>
  428. <p>Python 3.0 was released on December 3, 2008. And it took the better part of
  429. a decade for the community to embrace it. <strong>This should be universally
  430. recognized as a failure.</strong> While hindsight is 20/20, many of the issues
  431. with Python 3 were obvious at the time and could have been mitigated had
  432. the language maintainers been more accommodating - and dare I say
  433. empathetic - to its users.</p>
  434. <p>Initially, Python 3 had a rather cavalier attitude towards backwards and
  435. forwards compatibility. In the early years of Python 3, the attitude of
  436. Python's maintainers was <em>Python 3 is a new, better language: you should
  437. target it explicitly</em>. There were some tools and methods to ease the
  438. transition. But nothing super polished, especially in the early years.
  439. Adoption of Python 3 in the overall community was slow. Python developers
  440. in the wild justifiably complained that the value proposition of Python 3
  441. was too weak to justify porting effort. Not helping was that the early
  442. advice for targeting Python 3 was to rewrite the source code to become
  443. Python 3 native. This is in contrast with using the same source to run on both
  444. Python 2 and 3. For library and application maintainers, this potentially
  445. meant maintaining separate versions of your code or forcing end-users to
  446. make a giant leap, which would realistically orphan users on an old version,
  447. fragmenting your user base. Neither of those were great alternatives, so
  448. you can understand why many projects didn't bite.</p>
  449. <p>For many projects of non-trivial size, flag day transitions from Python 2 to
  450. 3 were simply not viable: the pathway to Python 3 was to make code dual
  451. Python 2/3 compatible and gradually switch over the runtime to Python 3.
  452. But initial versions of Python 3 made this effectively impossible! Let me
  453. give a few specific examples.</p>
  454. <p>In Python 2, a string literal <code>''</code> is effectively an array of bytes. In
  455. Python 3, it is a series of Unicode code points - a fundamentally different
  456. type! In Python 2, you could write <code>b''</code> to be explicit that a string literal
  457. was bytes or you could write <code>u''</code> to indicate a Unicode literal, mimicking
  458. Python 3's behavior. In Python 3, you could write <code>b''</code> to create a <code>bytes</code>
  459. instance. But for whatever reason, Python 3 initially removed the <code>u''</code> syntax,
  460. meaning there wasn't as easy way to explicitly denote the type of each
  461. string literal so that it was consistent between Python 2 and 3! Python 3.3
  462. (released September 2012) restored <code>u''</code> support, making it more viable to
  463. write Python source code that worked on both Python 2 and 3. <strong>For nearly 4
  464. years, Python 3 took away the consistent syntax for denoting bytes/Unicode
  465. string literals.</strong></p>
  466. <p>Another feature was <code>%</code> formatting of strings. Python 2 allowed use of the
  467. <code>%</code> formatting operator on both its string types. But Python 3 initially
  468. removed the implementation of <code>%</code> from <code>bytes</code>. Why, I have no clue. It
  469. is perfectly reasonable to splice byte sequences into a buffer via use of
  470. a formatting string. But the Python language maintainers insisted otherwise.
  471. And it wasn't until the community complained about its absence loudly enough
  472. that this feature was
  473. <a href="https://docs.python.org/3/whatsnew/3.5.html#whatsnew-pep-461">restored in Python 3.5</a>,
  474. which was released in September 2015. Fun fact: the lack of this feature was
  475. once considered a blocker for Mercurial moving to Python 3 because
  476. Mercurial uses <code>bytes</code> almost universally, which meant that nearly every use
  477. of <code>%</code> would have to be changed to something else. And to this day, Python
  478. 3's <code>bytes</code> still doesn't have a <code>format()</code> method, so the alternative was
  479. effectively string concatenation, which is a massive step backwards from the
  480. expressiveness of <code>%</code> formatting.</p>
  481. <p><strong>The initial approach of Python 3 mirrors a folly that many developers
  482. and projects make: attempting a rewrite instead of performing incremental
  483. evolution.</strong> For established projects, large scale rewrites often go poorly.
  484. And Python 3 is no exception. Yes, from a code level, CPython (and likely
  485. other Python implementations) were incremental changes over Python 2 using
  486. the same code base. But from a language and standard library level, the
  487. differences in Python 3 were significant enough that I - and even Python's
  488. core maintainers - considered it a new language, and therefore a rewrite.
  489. When your random project attempts a rewrite and fails, the blast radius of that is
  490. often contained to that project. Maybe you don't publish a new release
  491. as soon as you otherwise would. <strong>But when you are powering an ecosystem,
  492. the ripple effects from a failed rewrite percolate throughout that ecosystem
  493. and last for years and have many second order effects. We see this with
  494. Python 3, where poor choices made in the late 2000s are inflicting significant
  495. hardship still in 2020.</strong></p>
  496. <p>From the initial restrained adoption of Python 3, it is obvious that the
  497. Python ecosystem overwhelmingly rejected the initial boil the oceans approach
  498. of Python 3. Python's maintainers eventually got the message and started
  499. restoring features like <code>u''</code> and <code>bytes</code> <code>%</code> formatting back into the
  500. language to placate the community. All the while Python 3 had been accumulating
  501. new features and the cumulative sum of those features was compelling enough
  502. to win over users.</p>
  503. <p>For many projects (including Mercurial), Python 3.4/3.5 was the first viable
  504. porting target for Python 3. Python 3.5 was released in September 2015, almost
  505. 7 years after Python 3.0 was released in December 2008. <strong>Seven. Years.</strong>
  506. An ecosystem that falters for that long is generally not healthy. What may have
  507. saved Python from total collapse here is that Python 2 was still going strong and
  508. people were generally happy with it. I really do think Python dodged a bullet
  509. here, because there was a massive window where the language could have
  510. hemorrhaged a critical amount of its user base and been relegated to an
  511. afterthought. One could draw an analogy to Perl, which lost out to PHP,
  512. Python, and Ruby, and whose fall from grace aligned with a lengthy
  513. transition from Perl 5 to 6.</p>
  514. <p>If you look back at the early history of Python 3, <strong>I think you are forced
  515. to conclude that Python effectively kneecapped itself for 5-7 years
  516. through questionable implementation choices that prevented users from
  517. incurring incremental transitions between the major language versions. 2008
  518. to 2013-2015 should be known as the <em>lost years of Python</em> because so much
  519. opportunity and energy was squandered.</strong> Yes, Python is still healthy today
  520. and Python 3 is (finally) being adopted at scale. But had earlier versions
  521. of Python 3 been more <em>empathetic</em> towards Python 2 users porting to it,
  522. Python and Python 3 in 2020 would be even stronger than it is. The community
  523. was artificially hindered for years. And we won't know until 2023-2025 what
  524. things could have looked like in 2020 had the Python core language team
  525. spent more time paving a smoother road between the major language versions.</p>
  526. <p>To be clear, I do think Python 3 is generally a better language than Python 2.
  527. It has fewer warts, more compelling features, and better performance (except
  528. for startup time, which is still slower than Python 2). I am ecstatic the
  529. community is finally rallying around Python 3! For my Python coding, it has
  530. reached the point where I curse under my breath when I need to support
  531. Python 2 or even older versions of Python 3, like 3.5 or 3.6: I just wish
  532. the world would move on and adopt the future already!</p>
  533. <p>But I would be remiss if I failed to mention some of my gripes with Python
  534. 3 beyond the transition shenanigans.</p>
  535. <p>Perhaps my least favorite <em>feature</em> of Python 3 is its insistence that the
  536. world is Unicode. In Python 2, the default string type was backed by
  537. bytes. In Python 3, the default string type is backed by Unicode code
  538. points. As part of that transition, large parts of the standard library
  539. now operate in the Unicode space instead of the domain of bytes. I understand
  540. why Python does this: they want <em>strings</em> to be Unicode and don't want
  541. users to have to spend that much energy thinking about when to use
  542. <code>str</code> versus <code>bytes</code>. This approach is admirable and somewhat defensible
  543. because it takes a stand on a solution that is arguably <em>good enough</em> for
  544. most users. However, <strong>the approach of assuming the world is Unicode is
  545. flat out wrong and has significant implications for systems level
  546. applications</strong> (like version control tools).</p>
  547. <p>There are a myriad of places in Python's standard library where Python
  548. insists on using the Unicode-backed <code>str</code> type and rejects <code>bytes</code>. For
  549. example, various networking modules refuse to accept <code>bytes</code> for hostnames
  550. or URLs. HTTP libraries won't accept <code>bytes</code> for HTTP header names or values.
  551. Functions that are proxies to POSIX-defined functions won't accept <code>bytes</code>
  552. even though the POSIX function it calls into is using <code>char *</code> and isn't
  553. Unicode aware. Then there's filename handling, where Python assumes the
  554. existence of a global encoding for filenames and uses this encoding to convert
  555. between <code>str</code> and <code>bytes</code>. And it does this despite POSIX filesystem paths
  556. being a bag of bytes where the only rules are that <code>\0</code> terminates the
  557. filename and <code>/</code> is special.</p>
  558. <p>In cases like Python refusing to accept <code>bytes</code> for things like HTTP
  559. header names (which will just be spit out over the wire as bytes), Python's
  560. pendulum has swung too far towards Unicode only. In my opinion, Python needs
  561. to be more accommodating and allow <code>bytes</code> when it makes sense. I hope the
  562. pendulum knocks some sense into people when it swings back towards a more
  563. reasonable solution that better acknowledges the realities of the world we
  564. live in.</p>
  565. <p>For areas like filename handling, the world is more complicated. Python
  566. is effectively an abstraction layer over the operating system APIs exposing
  567. this functionality. And there is often an impedance mismatch between operating
  568. systems. For example, POSIX (Linux) tends to use <code>char *</code> for everything
  569. and doesn't care about encoding and Windows tends to use 16 bit character
  570. types where the encoding is... a can of worms.</p>
  571. <p><strong>The reality here is that it is impossible to abstract over differences
  572. between operating system behavior without compromises that can result in data
  573. loss, outright wrong behavior, or loss of functionality. But Python 3 attempts
  574. to do it anyway, making Python 3 unsuitable (or at least highly undesirable) for
  575. certain systems level applications that rely on it</strong> (like a version control
  576. tool).</p>
  577. <p>In fairness to Python, it isn't the only programming language that gets
  578. this wrong. The only language I've seen <em>properly</em> implement higher-order
  579. abstractions on top of operating system facilities is Rust, whose approach can
  580. be generalized as <em>use Python 3's solution of normalizing to Unicode/UTF-8 by
  581. default</em>, but expose <em>escape hatches</em> which allow access to the raw underlying
  582. types and APIs used by the operating system for the advanced consumers who
  583. require it. For example, Rust's <code>Path</code> type which represents a filesystem path
  584. <a href="https://doc.rust-lang.org/std/path/struct.Path.html#method.as_os_str">allows access</a>
  585. to the raw <a href="https://doc.rust-lang.org/std/ffi/struct.OsStr.html">OsStr</a> value
  586. used by the operating system, not a normalization of it to bytes or Unicode,
  587. which may be lossy. This allows consumers to e.g. create and retrieve
  588. OS-native filesystem paths without data loss. This functionality is critical
  589. in some domains. Python 3's awareness/insistence that the world is
  590. Unicode (which it isn't universally) reduces Python's applicability in these
  591. domains.</p>
  592. <p>Speaking of Rust, at the Mercurial developer meetup in October 2019, we were
  593. discussing the use of Rust in Mercurial and one of the core maintainers blurted
  594. out something along the lines of <em>if Rust were at its current state 5 years ago,
  595. Mercurial would have likely ported from Python 2 to Rust instead of Python 3</em>.
  596. As crazy as it initially sounded, I think I agree with that assessment. With the
  597. benefit of hindsight, having been a key player in the Python 3 porting effort,
  598. seeing all the complications and headaches Python 3 is introducing, and
  599. having learned Rust and witnessed its benefits for performance, control,
  600. and correctness firsthand, porting to Rust would likely have been the correct
  601. move for the project at that point in time. 2020 is not 2014, however, and I'm
  602. not sure if I would opt for a rewrite in Rust today. (Most rewrites are follies
  603. after all.) But I know one thing: I certainly wouldn't implement a new version
  604. control tool in Python 3 and I would probably choose Rust as an implementation
  605. language for most new projects in the systems level space or with an expected
  606. shelf life of 10+ years. (I really should blog about how awesome Rust is.)</p>
  607. <p>Back to the topic of Python itself, <strong>I'm really soured on Python at this
  608. point in time. The effort required to port to Python 3 was staggering. For
  609. Mercurial, Python 3 introduces a ton of problems and doesn't really solve
  610. many. We effectively sludged through mud for several years only to wind
  611. up in a state that feels strictly worse than where we started. I'm sure it will
  612. be strictly better in a few years. But at that point, we're talking about a
  613. 5+ year transition. To call the Python 3 transition disruptive and
  614. distracting for the project would be an understatement. As a project maintainer,
  615. it's natural to ask what we could have accomplished if we weren't forced
  616. to carry out this sideshow.</strong></p>
  617. <p>I can't shake the feeling that a lot of the pain afflicted by the Python 3
  618. transition could have been avoided had Python's language leadership made
  619. a different set of decisions and more highly prioritized the transition
  620. experience. (Like not initially removing features like <code>u''</code> and <code>bytes %</code>
  621. and not introducing gratuitous backwards compatibility breaks, like with
  622. <code>items()/iteritems()</code>. I would have also liked to see a feature like
  623. <code>from __future__</code> - maybe <code>from __past__</code> - that would make it easier for
  624. Python 3 code to target semantics in earlier versions in order to provide
  625. a more turnkey on-ramp onto new versions.) I simultaneously see Python 3
  626. losing its position as a justifiable tool in some domains (like systems
  627. level tooling) due to ongoing design decisions and poor implementation (like
  628. startup overhead problems). (In contrast, I see Rust excelling where Python
  629. is faltering and find Rust code surprisingly expressive to write and maintain
  630. given how low-level it is and therefore feel that Rust is a compelling
  631. alternative to Python in a surprisingly large number of domains.)</p>
  632. <p>Look, I know it is easy for me to armchair quarterback and critique with the
  633. benefit of hindsight/ignorance. I'm sure there is a lot of nuance here. I'm
  634. sure there was disagreement within the Python community over a lot of these
  635. issues. Maintaining a large and successful programming language and community
  636. like Python's is hard and you aren't going to please all the people all the
  637. time. And speaking as a maintainer, I have mad respect for the people leading
  638. such a large community. But niceties aside, everyone knows the Python 3
  639. transition was rough and could have gone better. It should not have taken 11
  640. years to get to where we are today.</p>
  641. <p><strong>I'd like to encourage the Python Project to conduct a thorough postmortem on
  642. the transition to Python 3.</strong> Identify what went well, what could have gone
  643. better, and what should be done next time such a large language change is wanted.
  644. Speaking as a Python user, a maintainer of a Python project, and as someone in
  645. industry who is now skeptical about use of Python at work due to risks of
  646. potentially company crippling high-effort migrations in the future, a postmortem
  647. would help restore my confidence that Python's maintainers learned from the
  648. various missteps on the road to Python 3 and these potentially ecosystem
  649. crippling mistakes won't be made again.</p>
  650. <p>Python had a wildly successful past few decades. And it can continue to
  651. thrive for several more. But the Python 3 migration was painful for all
  652. involved. And as much as we need to move on and leave Python 2 behind us,
  653. there are some important lessons to be learned. I hope the Python community
  654. takes the opportunity to reflect and am confident it will grow stronger by
  655. taking the time to do so.</p>
  656. </main>
  657. </article>
  658. <hr>
  659. <footer>
  660. <p>
  661. <a href="/david/" title="Aller à l’accueil">🏠</a> •
  662. <a href="/david/log/" title="Accès au flux RSS">🤖</a> •
  663. <a href="http://larlet.com" title="Go to my English profile" data-instant>🇨🇦</a> •
  664. <a href="mailto:david%40larlet.fr" title="Envoyer un courriel">📮</a> •
  665. <abbr title="Hébergeur : Alwaysdata, 62 rue Tiquetonne 75002 Paris, +33184162340">🧚</abbr>
  666. </p>
  667. <template id="theme-selector">
  668. <form>
  669. <fieldset>
  670. <legend>Thème</legend>
  671. <label>
  672. <input type="radio" value="auto" name="chosen-color-scheme" checked> Auto
  673. </label>
  674. <label>
  675. <input type="radio" value="dark" name="chosen-color-scheme"> Foncé
  676. </label>
  677. <label>
  678. <input type="radio" value="light" name="chosen-color-scheme"> Clair
  679. </label>
  680. </fieldset>
  681. </form>
  682. </template>
  683. </footer>
  684. <script type="text/javascript">
  685. function loadThemeForm(templateName) {
  686. const themeSelectorTemplate = document.querySelector(templateName)
  687. const form = themeSelectorTemplate.content.firstElementChild
  688. themeSelectorTemplate.replaceWith(form)
  689. form.addEventListener('change', (e) => {
  690. const chosenColorScheme = e.target.value
  691. localStorage.setItem('theme', chosenColorScheme)
  692. toggleTheme(chosenColorScheme)
  693. })
  694. const selectedTheme = localStorage.getItem('theme')
  695. if (selectedTheme && selectedTheme !== 'undefined') {
  696. form.querySelector(`[value="${selectedTheme}"]`).checked = true
  697. }
  698. }
  699. const prefersColorSchemeDark = '(prefers-color-scheme: dark)'
  700. window.addEventListener('load', () => {
  701. let hasDarkRules = false
  702. for (const styleSheet of Array.from(document.styleSheets)) {
  703. let mediaRules = []
  704. for (const cssRule of styleSheet.cssRules) {
  705. if (cssRule.type !== CSSRule.MEDIA_RULE) {
  706. continue
  707. }
  708. // WARNING: Safari does not have/supports `conditionText`.
  709. if (cssRule.conditionText) {
  710. if (cssRule.conditionText !== prefersColorSchemeDark) {
  711. continue
  712. }
  713. } else {
  714. if (cssRule.cssText.startsWith(prefersColorSchemeDark)) {
  715. continue
  716. }
  717. }
  718. mediaRules = mediaRules.concat(Array.from(cssRule.cssRules))
  719. }
  720. // WARNING: do not try to insert a Rule to a styleSheet you are
  721. // currently iterating on, otherwise the browser will be stuck
  722. // in a infinite loop…
  723. for (const mediaRule of mediaRules) {
  724. styleSheet.insertRule(mediaRule.cssText)
  725. hasDarkRules = true
  726. }
  727. }
  728. if (hasDarkRules) {
  729. loadThemeForm('#theme-selector')
  730. }
  731. })
  732. </script>
  733. </body>
  734. </html>