davidbgk
/
larlet-fr-david-cache


			
							123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238
							<!doctype html><!-- This is a valid HTML5 document. -->
<!-- Screen readers, SEO, extensions and so on. -->
<html lang="en">
<!-- Has to be within the first 1024 bytes, hence before the `title` element
  See: https://www.w3.org/TR/2012/CR-html5-20121217/document-metadata.html#charset -->
<meta charset="utf-8">
<!-- Why no `X-UA-Compatible` meta: https://stackoverflow.com/a/6771584 -->
<!-- The viewport meta is quite crowded and we are responsible for that.
  See: https://codepen.io/tigt/post/meta-viewport-for-2015 -->
<meta name="viewport" content="width=device-width,initial-scale=1">
<!-- Required to make a valid HTML5 document. -->
<title>Everything I know about the XZ backdoor (archive) — David Larlet</title>
<meta name="description" content="Publication mise en cache pour en conserver une trace.">
<!-- That good ol' feed, subscribe :). -->
<link rel="alternate" type="application/atom+xml" title="Feed" href="/david/log/">
<!-- Generated from https://realfavicongenerator.net/ such a mess. -->
<link rel="apple-touch-icon" sizes="180x180" href="/static/david/icons2/apple-touch-icon.png">
<link rel="icon" type="image/png" sizes="32x32" href="/static/david/icons2/favicon-32x32.png">
<link rel="icon" type="image/png" sizes="16x16" href="/static/david/icons2/favicon-16x16.png">
<link rel="manifest" href="/static/david/icons2/site.webmanifest">
<link rel="mask-icon" href="/static/david/icons2/safari-pinned-tab.svg" color="#07486c">
<link rel="shortcut icon" href="/static/david/icons2/favicon.ico">
<meta name="msapplication-TileColor" content="#f7f7f7">
<meta name="msapplication-config" content="/static/david/icons2/browserconfig.xml">
<meta name="theme-color" content="#f7f7f7" media="(prefers-color-scheme: light)">
<meta name="theme-color" content="#272727" media="(prefers-color-scheme: dark)">
<!-- Is that even respected? Retrospectively? What a shAItshow…
  https://neil-clarke.com/block-the-bots-that-feed-ai-models-by-scraping-your-website/ -->
<meta name="robots" content="noai, noimageai">
<!-- Documented, feel free to shoot an email. -->
<link rel="stylesheet" href="/static/david/css/style_2021-01-20.css">
<!-- See https://www.zachleat.com/web/comprehensive-webfonts/ for the trade-off. -->
<link rel="preload" href="/static/david/css/fonts/triplicate_t4_poly_regular.woff2" as="font" type="font/woff2" media="(prefers-color-scheme: light), (prefers-color-scheme: no-preference)" crossorigin>
<link rel="preload" href="/static/david/css/fonts/triplicate_t4_poly_bold.woff2" as="font" type="font/woff2" media="(prefers-color-scheme: light), (prefers-color-scheme: no-preference)" crossorigin>
<link rel="preload" href="/static/david/css/fonts/triplicate_t4_poly_italic.woff2" as="font" type="font/woff2" media="(prefers-color-scheme: light), (prefers-color-scheme: no-preference)" crossorigin>
<link rel="preload" href="/static/david/css/fonts/triplicate_t3_regular.woff2" as="font" type="font/woff2" media="(prefers-color-scheme: dark)" crossorigin>
<link rel="preload" href="/static/david/css/fonts/triplicate_t3_bold.woff2" as="font" type="font/woff2" media="(prefers-color-scheme: dark)" crossorigin>
<link rel="preload" href="/static/david/css/fonts/triplicate_t3_italic.woff2" as="font" type="font/woff2" media="(prefers-color-scheme: dark)" crossorigin>
<script>
    function toggleTheme(themeName) {
        document.documentElement.classList.toggle(
            'forced-dark',
            themeName === 'dark'
        )
        document.documentElement.classList.toggle(
            'forced-light',
            themeName === 'light'
        )
    }
    const selectedTheme = localStorage.getItem('theme')
    if (selectedTheme !== 'undefined') {
        toggleTheme(selectedTheme)
    }
</script>

  <meta name="robots" content="noindex, nofollow">
  <meta content="origin-when-cross-origin" name="referrer">
  <!-- Canonical URL for SEO purposes -->
  <link rel="canonical" href="https://boehs.org/node/everything-i-know-about-the-xz-backdoor">

<body class="remarkdown h1-underline h2-underline h3-underline em-underscore hr-center ul-star pre-tick" data-instant-intensity="viewport-all">


<article>
  <header>
    <h1>Everything I know about the XZ backdoor</h1>
  </header>
  <nav>
    <p class="center">
      <a href="/david/" title="Aller à l’accueil"><svg class="icon icon-home">
        <use xlink:href="/static/david/icons2/symbol-defs-2021-12.svg#icon-home"></use>
      </svg> Accueil</a> •
      <a href="https://boehs.org/node/everything-i-know-about-the-xz-backdoor" title="Lien vers le contenu original">Source originale</a>
      <br>
      Mis en cache le 2024-04-04
    </p>
  </nav>
  <hr>
  <p><em>Please note: This is being updated in real-time. The intent is to make sense of lots of simultaneous discoveries regarding this backdoor. last updated: 5:30 EST, on April 2nd</em></p>
<p><em>Update: The GitHub page for xz has been suspended.</em></p>
<h3 id="2021" tabindex="-1">2021</h3>
<p>JiaT75 (Jia Tan) creates their GitHub account.</p>
<p>The first commits they make are not to xz, but they are deeply suspicious. Specifically, they open a PR in libarchive: <a href="https://github.com/libarchive/libarchive/pull/1609">Added error text to warning when untaring with bsdtar</a>. This commit does a little more than it says. It replaces <code>safe_fprint</code> with an unsafe variant, potentially introducing another vulnerability. The code was merged without any discussion, and <s><a href="https://github.com/libarchive/libarchive/blob/master/tar/read.c#L374-L375">lives on to this day</a></s> (<a href="https://github.com/libarchive/libarchive/pull/2101">patched</a>). libarchive should also be considered compromised until proven otherwise.</p>
<h3 id="2022" tabindex="-1">2022</h3>
<p>In April 2022, Jia Tan submitted a patch via a mailing list. The contents of the patch are not relevant, but the events that follow are. A new persona — <em>Jigar Kumar</em> — enters, and begins <a href="https://www.mail-archive.com/xz-devel@tukaani.org/msg00565.html">pressuring</a> for this patch to be merged.</p>
<p>Soon after, <em>Jigar Kumar</em> <a href="https://www.mail-archive.com/xz-devel@tukaani.org/msg00566.html">begins</a> pressuring <em>Lasse Collin</em> to add another maintainer to XZ. In the fallout, there is much to learn about mental health in open source.</p>
<p>Three days after the emails pressuring <em>Lasse Collin</em> to add another maintainer, JiaT75 makes their first commit to xz: <a href="https://git.tukaani.org/?p=xz.git;a=commitdiff;h=aa75c5563a760aea3aa23d997d519e702e82726b">Tests: Created tests for hardware functions.</a>. Since this commit, they become a regular contributor to xz (they are currently the second most active). It’s unclear exactly when they became trusted in this repository.</p>
<p><em>Jigar Kumar</em> is <a href="https://www.mail-archive.com/search?l=xz-devel@tukaani.org&amp;q=Kumar&amp;x=0&amp;y=0">never seen again</a>. Another account — <a href="https://www.mail-archive.com/search?l=xz-devel@tukaani.org&amp;q=from:%22Dennis+Ens%22">Dennis Ens</a> also participates in pressure, with a similar name+number formatted email. This account is also never seen outside of xz discussion, and neither have any associated accounts that have been discovered.</p>
<article class="mastodon-embed"><div><img src="https://files.mastodon.social/accounts/avatars/000/023/457/original/ad501ceca43dd473.png"><div><strong><a href="https://mastodon.social/@glyph/112180922900094371">Glyph</a></strong> <sup>@glyph@mastodon.social</sup></div></div><p></p><p><span class="h-card" translate="no"><a href="https://social.coop/@eb" class="u-url mention">@<span>eb</span></a></span> I really hope that this causes an industry-wide reckoning with the common practice of letting your entire goddamn product rest on the shoulders of one overworked person having a slow mental health crisis without financially or operationally supporting them whatsoever. I want everyone who has an open source dependency to read this message <a href="https://www.mail-archive.com/xz-devel@tukaani.org/msg00567.html" target="_blank" rel="nofollow noopener noreferrer" translate="no"><span class="invisible">https://www.</span><span class="ellipsis">mail-archive.com/xz-devel@tuka</span><span class="invisible">ani.org/msg00567.html</span></a></p><p></p><p><sup>Mar 29, 2024, 20:43</sup> <sup>624 retoots</sup></p></article>
<h3 id="2023" tabindex="-1">2023</h3>
<p>JiaT75 merges their first commit <a href="https://github.com/tukaani-project/xz/pull/7">on Jan 7, 2023</a>, which gives us a good indication of when they fully gain trust.</p>
<p>In March, the primary contact email in Google’s oss-fuzz is <a href="https://github.com/JiaT75/oss-fuzz/commit/6403e93344476972e908ce17e8244f5c2b957dfd">updated</a> to be Jia’s, instead of <em>Lasse Collin</em>.</p>
<p>Testing infrastructure that will be used in this exploit is committed. Despite <em>Lasse Collin</em> being attributed as the author for this, <em>Jia Tan</em> committed it, and it was originally written by <em>Hans Jansen</em> in June:</p>
<p><em>Hans Jansen</em>’s account was seemingly made specifically to create this pull request. There is very little activity before and after. They will later push for the compromised version of XZ to be included in Debian.</p>
<p>In July, <a href="https://github.com/google/oss-fuzz/pull/10667">a PR</a> was opened in oss-fuzz to disable ifunc for fuzzing builds, due to issues introduced by the changes above. This appears to be deliberate to mask the malicious changes that will be introduced soon. Also, JiaT75 opened an <a href="https://github.com/llvm/llvm-project/issues/63957">issue</a> about a warning in clang that, while indeed incorrect, drew attention to ifuncs.</p>
<h3 id="2024" tabindex="-1">2024</h3>
<p>A pull request for Google’s <a href="https://github.com/google/oss-fuzz/pull/11587">oss-fuzz is opened</a> that changes the URL for the project from <a href="http://tukaani.org/xz/">tukaani.org/xz/</a> to <a href="http://xz.tukaani.org/xz-utils/">xz.tukaani.org/xz-utils/</a>. <a href="http://tukaani.org">tukaani.org</a> is hosted at <code>5.44.245.25</code> in Finland, at <a href="https://www.zoner.fi/">this</a> hosting company. The <code>xz</code> subdomain, meanwhile, points to GitHub pages. This furthers the amount of control Jia has over the project.</p>
<p>A commit containing the final steps required to execute this backdoor is added to the repository:</p>
<h4 id="the-discovery" tabindex="-1">The discovery</h4>
<p>An email is sent to the oss-security mailing list: <a href="https://www.openwall.com/lists/oss-security/2024/03/29/4">backdoor in upstream xz/liblzma leading to ssh server compromise</a>, announcing this discovery, and doing it’s best to explain the exploit chain.</p>
<article class="mastodon-embed"><p></p><p>I was doing some micro-benchmarking at the time, needed to quiesce the system to reduce noise. Saw sshd processes were using a surprising amount of CPU, despite immediately failing because of wrong usernames etc. Profiled sshd, showing lots of cpu time in liblzma, with perf unable to attribute it to a symbol. Got suspicious. Recalled that I had seen an odd valgrind complaint in automated testing of postgres, a few weeks earlier, after package updates.</p><p>Really required a lot of coincidences.</p><p></p><p><sup>Mar 29, 2024, 18:32</sup> <sup>858 retoots</sup></p></article>
<p>A <a href="https://gist.github.com/thesamesam/223949d5a074ebc3dce9ee78baad9e27">gist</a> has been published with a great high-level technical overview and a “what you need to know”</p>
<p>In addition to the gist and the email above, several analysis attempts have begun emerging:</p>
<h4 id="a-sudden-push-for-inclusion" tabindex="-1">A sudden push for inclusion</h4>
<p>A request for the vulnerable version to be included in Debian is opened by Hans:</p>
<p>This request was opened the <a href="https://salsa.debian.org/hjansen">same week</a> Hans’ Debian GitLab account was created. The account created a few similar “update” requests in various low-traffic repositories to build credibility, after asking for this one.</p>
<p>Several other, suspicious, anonymous name+number accounts with little former activity also push for its inclusion, including <em>misoeater91</em> and <em>krygorin4545</em>. <em>krygorin4545</em>’s PGP key was made 2 days before joining the discussion.</p>
<blockquote><p>Also seeing this bug. Extra valgrind output causes some failed tests for me. Looks like the new version will resolve it. Would like this new version so I can continue work.</p></blockquote>
<blockquote><p>I noticed this last week and almost made a Valgrind bug. Glad to see it being fixed.<br>Thanks Hans!</p></blockquote>
<p>The Valgrind bugs mentioned were <em>introduced</em> by this malicious injection, as noted in the email to OSS-Security:</p>
<blockquote><p>Subsequently the injected code (more about that below) caused valgrind errors and crashes in some configurations, due to the stack layout differing from what the backdoor was expecting. These issues were attempted to be worked around in 5.6.1:</p></blockquote>
<p>A <a href="https://github.com/jamespfennell/xz/pull/2">pull request</a> to a go library by a 1Password employee is opened asking to upgrade the library to the vulnerable version, however, it was all unfortunate timing. 1Password reached out by email referring me to this <a href="https://github.com/jamespfennell/xz/pull/2#issuecomment-2027836356">comment</a>, and everything seems to check out.</p>
<p>A Fedora contributor <a href="https://news.ycombinator.com/item?id=39866275">states</a> that <em>Jia</em> was pushing for its inclusion in Fedora as it contains “great new features”</p>
<p><em>Jia Tan</em> also <a href="https://bugs.launchpad.net/ubuntu/+source/xz-utils/+bug/2059417">attempted</a> to get it into Ubuntu days before the beta freeze.</p>
<p>A few hours after all this came out, GitHub suspended JiaT75’s account. Thanks? They also banned the repository, meaning people can no longer audit the changes made to it without resorting to mirrors. Immensely helpful, GitHub. They also <a href="https://github.com/JiaT75?tab=following">suspended</a> <em>Lasse Collin</em>’s account, which is completely disgraceful.</p>
<p>Lasse has begun reverting changes introduced by Jia, <a href="https://git.tukaani.org/?p=xz.git;a=commitdiff;h=f9cf4c05edd14dedfe63833f8ccbe41b55823b00">including</a> one that <a href="https://git.tukaani.org/?p=xz.git;a=commitdiff;h=328c52da8a2bbb81307644efdb58db2c422d9ba7">added</a> a sneaky period to disable the sandbox. They also have published a FAQ that begins to explain the situation: <a href="https://tukaani.org/xz-backdoor/">XZ Utils backdoor</a></p>
<h3 id="osint" tabindex="-1">OSINT</h3>
<p>Various people have reached out to me regarding discoveries about the identity of Jia. Some of this has been incorporated in the timeline, but other stuff is “timeless” so I’m putting it here:</p>
<h4 id="irc" tabindex="-1">IRC</h4>
<p>I received an email that clarified a few points and provided new insight into the situation.</p>
<p>“Jia Tan” was present on the #tukaani IRC channel on Libera.Chat. A /whois revealed their connecting IP and activity on March 29th.</p>
<pre><code>[libera] -!- jiatan [~jiatan@185.128.24.163]
[libera] -!-  was      : Jia Tan
[libera] -!-  hostname : 185.128.24.163
[libera] -!-  account  : jiatan
[libera] -!-  server   : tungsten.libera.chat [Fri Mar 29 14:47:40 2024]
[libera] -!- End of WHOWAS
</code></pre>
<p>Running a Nmap on the IP shows a lot of open ports, which probably indicates a proxy, hosting provider, or something of the sort. The IP is from Singapore.</p>
<p>Further research shows that this IP belongs to Witopia VPN, so it’s not entirely indicative of a region. Given the timezone, however, I feel like proximity becomes plausible.</p>
<h4 id="important-notes-on-linkedin" tabindex="-1">Important notes on LinkedIn</h4>
<p>I have received a few emails alerting me to a LinkedIn of somebody named <em>Jia Tan</em>. Their bio boasts of <em>large-scale vulnerability management</em>. They claim to live in California. Is this our man? The commits on JiaT75’s GitHub are set to +0800, which would not indicate presence in California. UTC-0800 would be California. Most of the commits <a href="https://play.clickhouse.com/play?user=play#U0VMRUNUIHRvSG91cihjcmVhdGVkX2F0KSBBUyBob3VyLCBjb3VudCgqKSBGUk9NIGdpdGh1Yl9ldmVudHMgV0hFUkUgYWN0b3JfbG9naW49J0ppYVQ3NScgR1JPVVAgQlkgaG91ciBPUkRFUiBCWSBob3Vy">were made</a> between UTC 12-17, which is awfully early for California. In my opinion, there is no sufficient evidence that the LinkedIn being discussed is our man. I think identity theft is more likely, but I am of course open to more evidence.</p>
<h4 id="discoveries-in-the-git-logs" tabindex="-1">Discoveries in the Git logs</h4>
<p>I received an email from <a href="https://github.com/minhuw">Minhu Wang</a> who investigated the Git log, and found one instance where Jia’s username was different:</p>
<pre class="language-bash"><code class="language-bash">$ <span class="token function">git</span> shortlog <span class="token parameter variable">--summary</span> <span class="token parameter variable">--numbered</span> <span class="token parameter variable">--email</span> <span class="token operator">|</span> <span class="token function">grep</span> jiat0218@gmail.com
<span class="token number">273</span> Jia Tan <span class="token operator">&lt;</span>jiat0218@gmail.com<span class="token operator">&gt;</span>
<span class="token number">2</span> jiat75 <span class="token operator">&lt;</span>jiat0218@gmail.com<span class="token operator">&gt;</span>
<span class="token number">1</span> Jia Cheong Tan <span class="token operator">&lt;</span>jiat0218@gmail.com<span class="token operator">&gt;</span></code></pre>
<p>They found this particularly interesting as <code>Cheong</code> is new information. I’ve now learned from another source that <em>Cheong</em> isn’t Mandarin, it’s Cantonese. This source theorizes that Cheong is a variant of the 張 surname, as “eong” matches Jyutping (a Cantonese romanisation standard), and “Cheung” is pretty common in Hong Kong as an official surname romanisation. A third source has alerted me that “Jia” is Mandarin (as Cantonese rarely uses <code>J</code> and especially not <code>Ji</code>). The <code>Tan</code> last name is <em>possible</em> in Mandarin but is most common for the Hokkien Chinese dialect pronunciation of the character 陳 (Cantonese: Chan, Mandarin: Chen). It’s most likely our actor just mashed plausible-sounding Chinese names together.</p>
<p>Furthermore, an <a href="https://rheaeve.substack.com/p/xz-backdoor-times-damned-times-and">independent analysis</a> of commit timings concludes that the perpetrator worked “Office Hours” in a UTC+02/03 timezone. It’s particularly notable that they worked through the Lunar New Year, and did not work on some notable Eastern European holidays, including Christmas and New Year. I have, however, been presented with a differing view, which you can read <a href="https://lunduke.locals.com/post/5467061/xz-backdoor-i-did-a-more-thorough-analysis-and-i-changed-my-mind-again-specifically-i-compar">here</a>.</p>
</article>


<hr>

<footer>
  <p>
    <a href="/david/" title="Aller à l’accueil"><svg class="icon icon-home">
        <use xlink:href="/static/david/icons2/symbol-defs-2021-12.svg#icon-home"></use>
      </svg> Accueil</a> •
    <a href="/david/log/" title="Accès au flux RSS"><svg class="icon icon-rss2">
        <use xlink:href="/static/david/icons2/symbol-defs-2021-12.svg#icon-rss2"></use>
      </svg> Suivre</a> •
    <a href="http://larlet.com" title="Go to my English profile" data-instant><svg class="icon icon-user-tie">
        <use xlink:href="/static/david/icons2/symbol-defs-2021-12.svg#icon-user-tie"></use>
      </svg> Pro</a> •
    <a href="mailto:david%40larlet.fr" title="Envoyer un courriel"><svg class="icon icon-mail">
        <use xlink:href="/static/david/icons2/symbol-defs-2021-12.svg#icon-mail"></use>
      </svg> Email</a> •
    <abbr class="nowrap" title="Hébergeur : Alwaysdata, 62 rue Tiquetonne 75002 Paris, +33184162340"><svg class="icon icon-hammer2">
        <use xlink:href="/static/david/icons2/symbol-defs-2021-12.svg#icon-hammer2"></use>
      </svg> Légal</abbr>
  </p>
  <template id="theme-selector">
      <form>
          <fieldset>
              <legend><svg class="icon icon-brightness-contrast">
                <use xlink:href="/static/david/icons2/symbol-defs-2021-12.svg#icon-brightness-contrast"></use>
              </svg> Thème</legend>
              <label>
                  <input type="radio" value="auto" name="chosen-color-scheme" checked> Auto
              </label>
              <label>
                  <input type="radio" value="dark" name="chosen-color-scheme"> Foncé
              </label>
              <label>
                  <input type="radio" value="light" name="chosen-color-scheme"> Clair
              </label>
          </fieldset>
      </form>
  </template>
</footer>
<script src="/static/david/js/instantpage-5.1.0.min.js" type="module"></script>
<script>
    function loadThemeForm(templateName) {
        const themeSelectorTemplate = document.querySelector(templateName)
        const form = themeSelectorTemplate.content.firstElementChild
        themeSelectorTemplate.replaceWith(form)

        form.addEventListener('change', (e) => {
            const chosenColorScheme = e.target.value
            localStorage.setItem('theme', chosenColorScheme)
            toggleTheme(chosenColorScheme)
        })

        const selectedTheme = localStorage.getItem('theme')
        if (selectedTheme && selectedTheme !== 'undefined') {
            form.querySelector(`[value="${selectedTheme}"]`).checked = true
        }
    }

    const prefersColorSchemeDark = '(prefers-color-scheme: dark)'
    window.addEventListener('load', () => {
        let hasDarkRules = false
        for (const styleSheet of Array.from(document.styleSheets)) {
            let mediaRules = []
            for (const cssRule of styleSheet.cssRules) {
                if (cssRule.type !== CSSRule.MEDIA_RULE) {
                    continue
                }
                // WARNING: Safari does not have/supports `conditionText`.
                if (cssRule.conditionText) {
                    if (cssRule.conditionText !== prefersColorSchemeDark) {
                        continue
                    }
                } else {
                    if (cssRule.cssText.startsWith(prefersColorSchemeDark)) {
                        continue
                    }
                }
                mediaRules = mediaRules.concat(Array.from(cssRule.cssRules))
            }

            // WARNING: do not try to insert a Rule to a styleSheet you are
            // currently iterating on, otherwise the browser will be stuck
            // in a infinite loop…
            for (const mediaRule of mediaRules) {
                styleSheet.insertRule(mediaRule.cssText)
                hasDarkRules = true
            }
        }
        if (hasDarkRules) {
            loadThemeForm('#theme-selector')
        }
    })
</script>
</body>
</html>