|
|
|
@@ -0,0 +1,214 @@ |
|
|
|
<!doctype html><!-- This is a valid HTML5 document. --> |
|
|
|
<!-- Screen readers, SEO, extensions and so on. --> |
|
|
|
<html lang="fr"> |
|
|
|
<!-- Has to be within the first 1024 bytes, hence before the `title` element |
|
|
|
See: https://www.w3.org/TR/2012/CR-html5-20121217/document-metadata.html#charset --> |
|
|
|
<meta charset="utf-8"> |
|
|
|
<!-- Why no `X-UA-Compatible` meta: https://stackoverflow.com/a/6771584 --> |
|
|
|
<!-- The viewport meta is quite crowded and we are responsible for that. |
|
|
|
See: https://codepen.io/tigt/post/meta-viewport-for-2015 --> |
|
|
|
<meta name="viewport" content="width=device-width,initial-scale=1"> |
|
|
|
<!-- Required to make a valid HTML5 document. --> |
|
|
|
<title>The Illusion Of Developer “Productivity” Opens The Door To Snake Oil (archive) — David Larlet</title> |
|
|
|
<meta name="description" content="Publication mise en cache pour en conserver une trace."> |
|
|
|
<!-- That good ol' feed, subscribe :). --> |
|
|
|
<link rel="alternate" type="application/atom+xml" title="Feed" href="/david/log/"> |
|
|
|
<!-- Generated from https://realfavicongenerator.net/ such a mess. --> |
|
|
|
<link rel="apple-touch-icon" sizes="180x180" href="/static/david/icons2/apple-touch-icon.png"> |
|
|
|
<link rel="icon" type="image/png" sizes="32x32" href="/static/david/icons2/favicon-32x32.png"> |
|
|
|
<link rel="icon" type="image/png" sizes="16x16" href="/static/david/icons2/favicon-16x16.png"> |
|
|
|
<link rel="manifest" href="/static/david/icons2/site.webmanifest"> |
|
|
|
<link rel="mask-icon" href="/static/david/icons2/safari-pinned-tab.svg" color="#07486c"> |
|
|
|
<link rel="shortcut icon" href="/static/david/icons2/favicon.ico"> |
|
|
|
<meta name="msapplication-TileColor" content="#f7f7f7"> |
|
|
|
<meta name="msapplication-config" content="/static/david/icons2/browserconfig.xml"> |
|
|
|
<meta name="theme-color" content="#f7f7f7" media="(prefers-color-scheme: light)"> |
|
|
|
<meta name="theme-color" content="#272727" media="(prefers-color-scheme: dark)"> |
|
|
|
<!-- Documented, feel free to shoot an email. --> |
|
|
|
<link rel="stylesheet" href="/static/david/css/style_2021-01-20.css"> |
|
|
|
<!-- See https://www.zachleat.com/web/comprehensive-webfonts/ for the trade-off. --> |
|
|
|
<link rel="preload" href="/static/david/css/fonts/triplicate_t4_poly_regular.woff2" as="font" type="font/woff2" media="(prefers-color-scheme: light), (prefers-color-scheme: no-preference)" crossorigin> |
|
|
|
<link rel="preload" href="/static/david/css/fonts/triplicate_t4_poly_bold.woff2" as="font" type="font/woff2" media="(prefers-color-scheme: light), (prefers-color-scheme: no-preference)" crossorigin> |
|
|
|
<link rel="preload" href="/static/david/css/fonts/triplicate_t4_poly_italic.woff2" as="font" type="font/woff2" media="(prefers-color-scheme: light), (prefers-color-scheme: no-preference)" crossorigin> |
|
|
|
<link rel="preload" href="/static/david/css/fonts/triplicate_t3_regular.woff2" as="font" type="font/woff2" media="(prefers-color-scheme: dark)" crossorigin> |
|
|
|
<link rel="preload" href="/static/david/css/fonts/triplicate_t3_bold.woff2" as="font" type="font/woff2" media="(prefers-color-scheme: dark)" crossorigin> |
|
|
|
<link rel="preload" href="/static/david/css/fonts/triplicate_t3_italic.woff2" as="font" type="font/woff2" media="(prefers-color-scheme: dark)" crossorigin> |
|
|
|
<script> |
|
|
|
function toggleTheme(themeName) { |
|
|
|
document.documentElement.classList.toggle( |
|
|
|
'forced-dark', |
|
|
|
themeName === 'dark' |
|
|
|
) |
|
|
|
document.documentElement.classList.toggle( |
|
|
|
'forced-light', |
|
|
|
themeName === 'light' |
|
|
|
) |
|
|
|
} |
|
|
|
const selectedTheme = localStorage.getItem('theme') |
|
|
|
if (selectedTheme !== 'undefined') { |
|
|
|
toggleTheme(selectedTheme) |
|
|
|
} |
|
|
|
</script> |
|
|
|
|
|
|
|
<meta name="robots" content="noindex, nofollow"> |
|
|
|
<meta content="origin-when-cross-origin" name="referrer"> |
|
|
|
<!-- Canonical URL for SEO purposes --> |
|
|
|
<link rel="canonical" href="https://codemanship.wordpress.com/2023/09/25/the-illusion-of-developer-productivity-opens-the-door-to-snake-oil/"> |
|
|
|
|
|
|
|
<body class="remarkdown h1-underline h2-underline h3-underline em-underscore hr-center ul-star pre-tick" data-instant-intensity="viewport-all"> |
|
|
|
|
|
|
|
|
|
|
|
<article> |
|
|
|
<header> |
|
|
|
<h1>The Illusion Of Developer “Productivity” Opens The Door To Snake Oil</h1> |
|
|
|
</header> |
|
|
|
<nav> |
|
|
|
<p class="center"> |
|
|
|
<a href="/david/" title="Aller à l’accueil"><svg class="icon icon-home"> |
|
|
|
<use xlink:href="/static/david/icons2/symbol-defs-2021-12.svg#icon-home"></use> |
|
|
|
</svg> Accueil</a> • |
|
|
|
<a href="https://codemanship.wordpress.com/2023/09/25/the-illusion-of-developer-productivity-opens-the-door-to-snake-oil/" title="Lien vers le contenu original">Source originale</a> |
|
|
|
</p> |
|
|
|
</nav> |
|
|
|
<hr> |
|
|
|
<figure class="wp-block-image size-large"><a href="https://codemanship.files.wordpress.com/2023/09/a_footballer.png"><img data-attachment-id="1781" data-permalink="https://codemanship.wordpress.com/2023/09/25/the-illusion-of-developer-productivity-opens-the-door-to-snake-oil/a_footballer/" data-orig-file="https://codemanship.files.wordpress.com/2023/09/a_footballer.png" data-orig-size="1456,816" data-comments-opened="1" data-image-meta='{"aperture":"0","credit":"","camera":"","caption":"","created_timestamp":"0","copyright":"","focal_length":"0","iso":"0","shutter_speed":"0","title":"","orientation":"0"}' data-image-title="a_footballer" data-image-description="" data-image-caption="" data-medium-file="https://codemanship.files.wordpress.com/2023/09/a_footballer.png?w=300" data-large-file="https://codemanship.files.wordpress.com/2023/09/a_footballer.png?w=840" src="https://codemanship.files.wordpress.com/2023/09/a_footballer.png?w=1024" alt="" class="wp-image-1781" srcset="https://codemanship.files.wordpress.com/2023/09/a_footballer.png?w=1024 1024w, https://codemanship.files.wordpress.com/2023/09/a_footballer.png 1456w, https://codemanship.files.wordpress.com/2023/09/a_footballer.png?w=150 150w, https://codemanship.files.wordpress.com/2023/09/a_footballer.png?w=300 300w, https://codemanship.files.wordpress.com/2023/09/a_footballer.png?w=768 768w" sizes="(max-width: 709px) 85vw, (max-width: 909px) 67vw, (max-width: 1362px) 62vw, 840px"></a><figcaption class="wp-element-caption">A footballer’s productivity is ultimately measured by how many goals the team scores</figcaption></figure> |
|
|
|
|
|
|
|
<p>There’s been much talk about measuring the productivity of software developers, triggered by a <a href="https://www.mckinsey.com/industries/technology-media-and-telecommunications/our-insights/yes-you-can-measure-software-developer-productivity">report from management consultants McKinsey </a>claiming to have succeeded where countless others over many decades have failed.</p> |
|
|
|
|
|
|
|
<p>I’m not going to dwell on the contents, as I prefer not to flatter it with that kind of scrutiny. Suffice to say, their ideas are naive at best. File in the usual place with your used egg shells and empty milk cartons.</p> |
|
|
|
|
|
|
|
<p>What McKinsey’s take suffers from is very, very common: they’ve mistaken activity for outcomes. Activity is easy to measure at an individual level, outcomes not so much. In fact, outcomes are often difficult to quantify at all.</p> |
|
|
|
|
|
|
|
<p>My first observation is that it’s not individual developers who produce outcomes; outcomes are achieved by the <em>team</em>. Just as there are players in a football team who usually don’t score goals, but without them fewer goals would be scored, there are usually people in a dev team who would be viewed as “unproductive” by McKinsey’s yardstick, but without whom the team as a whole would achieve much less. (See <a href="https://www.mckinsey.com/industries/technology-media-and-telecommunications/our-insights/yes-you-can-measure-software-developer-productivity">Dan North’s brilliant skewering of their metrics</a> using the very highly valuable Tim McKinnon – and I know, because I’ve worked with him – as the example.)</p> |
|
|
|
|
|
|
|
<p>My second observation is that outcomes in software development are rarely what they seem. Is our goal really to deliver code? Or is it to solve customers’ problems? Think of a doctor; is their goal to deliver treatments, or is to make us better?</p> |
|
|
|
|
|
|
|
<p>We in software, sadly, tend to be in the treatments business, not in the patients business. We’re Big Pharma. And in the same way that Big Pharma invests massively in persuading us that we have the illness their potion cures, we have a tendency to try to get the customer’s problem to fit our solution. And so it is that “productivity” tends to be about the potion, and not the patient.</p> |
|
|
|
|
|
|
|
<p>And so I wholeheartedly reject this individualist, mechanistic approach to measuring developer productivity. It’s nonsense. But I can understand why the <em>idea</em> appeals to managers in particular. The Illusion of Control<sup>TM</sup> has a strong pull in a situation where, in reality, managers have no real control beyond what to fund and what not to fund, and who to hire and who to fire. Who <em>wouldn’t </em>want those decisions to appear empirical and rational, and not the gambles they actually are?</p> |
|
|
|
|
|
|
|
<p>But more important to me is how this illusion can impact the very real business of solving customers’ problems with software. When all our focus is on potions and not patients, it’s easy for Snake Oil to creep in to the process.</p> |
|
|
|
|
|
|
|
<p>At time of writing, there’s much talk and incredible hype about one particular snake oil that promises much but, as far as I’ve managed to see with concrete examples that can be verified, delivers little to nothing for patients: Large Language Models.</p> |
|
|
|
|
|
|
|
<p>Code generation using LLMs like ChatGPT is, like all generative A.I., <em><a href="https://youtu.be/_nG6d6HSGB4?si=uqr4CehZBXOT6Hpy">impressive but wrong</a></em>. Having spent more than one hundred hours experimenting with GPT-4 and trying to replicate some of the claims people are making, I’ve seen how the illusion of productivity can suck us in. Yes, you <em>are</em> creating code faster. No, that code doesn’t work a lot of the time.</p> |
|
|
|
|
|
|
|
<p>But if we measure our productivity by “how far we kick the ball” instead of “how many goals the team scores”, that can seem like a Win. It falls into the same trap that thinking skimping on developer testing – or skipping it altogether – helps us deliver sooner. Deliver what, exactly? Bugs?</p> |
|
|
|
|
|
|
|
<p>On their website, GitHub claim that 88% of developers using <a href="https://github.com/features/copilot">Copilot</a> <em>feel</em> more productive. But what percentage of developers also <em>feel</em> that skipping some developer testing helps them deliver working software sooner. I could take a wild guess at somewhere in the ballpark of 88%, perhaps.</p> |
|
|
|
|
|
|
|
<p><a href="https://github.blog/2022-09-07-research-quantifying-github-copilots-impact-on-developer-productivity-and-happiness/">They did a study, of course</a>. (There’s always a study!) They tasked developers with writing a web server in JavaScript from scratch, some using Copilot, some doing it all by hand. And lo and behold, the developers who used Copilot completed that task in 55% less time. Isn’t it marvellous how vendor-funded studies always seem to back up their claims?</p> |
|
|
|
|
|
|
|
<p>But let’s look a little closer, shall we? First of all, since when were customer requirements like “Write me a web server”? A typical software system used in business, for example, will have complex rules that are usually not precisely defined up front, but rather discovered through customer feedback. And in that sense, how quickly we converge on a working solution will depend heavily on iterating, and on our ability to evolve the code. This wasn’t part of their exercise.</p> |
|
|
|
|
|
|
|
<p>Also, ahem.. If there was any problem that Copilot was probably already trained on, it’s <a href="https://github.com/search?q=http+server+language%3AJavaScript&type=repositories&l=JavaScript">JavaScript web servers</a>. People have noted how good GPT-4 is at solving online coding problems that were published before the training data cut-off date, but <a href="https://www.aisnakeoil.com/p/gpt-4-and-professional-benchmarks">not so hot at solving problems published after that</a>. I’d like to see how it performs on a novel problem. (In my own experiments with it, poorly.)</p> |
|
|
|
|
|
|
|
<p>And two more observations: </p> |
|
|
|
|
|
|
|
<p>First, this study focuses on developers working alone for a relatively short amount of time. Let’s see how it performs when a <em>team</em> is working on a more complex problem -each working on their own part of the system – for several days. That’s a lot of rapidly-changing context for an LLM. It’s easy to fool ourselves into believing something makes us better at running marathons because it helped us run the 100m dash faster. </p> |
|
|
|
|
|
|
|
<p>Secondly, GitHub’s musings on measuring developer productivity suffer a very similar “potions over patients” bias to the McKinsey report.</p> |
|
|
|
|
|
|
|
<p>And vendors have a very real incentive to want us to believe that the big problems in software development can be solved with their tools.</p> |
|
|
|
|
|
|
|
<p>Given the very high stakes for our industry – probably visible from space by now – I think it would be useful to see bigger, wider and more realistic studies of the impact of tools like Copilot on the capability of <em>teams</em> to <em>solve real customer problems</em>. As with almost every super-duper-we’ll-never-be-poor-or-hungry-again CASE tool revolution that’s come before, I suspect the answer will be “none at all”. But you can’t charge $19 a month for “none at all”. (Well, okay, you <em>can</em>. Just as long as there are enough people out there who focus on potions instead of patients.)</p> |
|
|
|
|
|
|
|
<p>But here’s the thing: I suspect bigger, wider, longer, more realistic studies of the impact on development <em>team</em> productivity might reveal simply that we still don’t know what that means.</p> |
|
|
|
</article> |
|
|
|
|
|
|
|
|
|
|
|
<hr> |
|
|
|
|
|
|
|
<footer> |
|
|
|
<p> |
|
|
|
<a href="/david/" title="Aller à l’accueil"><svg class="icon icon-home"> |
|
|
|
<use xlink:href="/static/david/icons2/symbol-defs-2021-12.svg#icon-home"></use> |
|
|
|
</svg> Accueil</a> • |
|
|
|
<a href="/david/log/" title="Accès au flux RSS"><svg class="icon icon-rss2"> |
|
|
|
<use xlink:href="/static/david/icons2/symbol-defs-2021-12.svg#icon-rss2"></use> |
|
|
|
</svg> Suivre</a> • |
|
|
|
<a href="http://larlet.com" title="Go to my English profile" data-instant><svg class="icon icon-user-tie"> |
|
|
|
<use xlink:href="/static/david/icons2/symbol-defs-2021-12.svg#icon-user-tie"></use> |
|
|
|
</svg> Pro</a> • |
|
|
|
<a href="mailto:david%40larlet.fr" title="Envoyer un courriel"><svg class="icon icon-mail"> |
|
|
|
<use xlink:href="/static/david/icons2/symbol-defs-2021-12.svg#icon-mail"></use> |
|
|
|
</svg> Email</a> • |
|
|
|
<abbr class="nowrap" title="Hébergeur : Alwaysdata, 62 rue Tiquetonne 75002 Paris, +33184162340"><svg class="icon icon-hammer2"> |
|
|
|
<use xlink:href="/static/david/icons2/symbol-defs-2021-12.svg#icon-hammer2"></use> |
|
|
|
</svg> Légal</abbr> |
|
|
|
</p> |
|
|
|
<template id="theme-selector"> |
|
|
|
<form> |
|
|
|
<fieldset> |
|
|
|
<legend><svg class="icon icon-brightness-contrast"> |
|
|
|
<use xlink:href="/static/david/icons2/symbol-defs-2021-12.svg#icon-brightness-contrast"></use> |
|
|
|
</svg> Thème</legend> |
|
|
|
<label> |
|
|
|
<input type="radio" value="auto" name="chosen-color-scheme" checked> Auto |
|
|
|
</label> |
|
|
|
<label> |
|
|
|
<input type="radio" value="dark" name="chosen-color-scheme"> Foncé |
|
|
|
</label> |
|
|
|
<label> |
|
|
|
<input type="radio" value="light" name="chosen-color-scheme"> Clair |
|
|
|
</label> |
|
|
|
</fieldset> |
|
|
|
</form> |
|
|
|
</template> |
|
|
|
</footer> |
|
|
|
<script src="/static/david/js/instantpage-5.1.0.min.js" type="module"></script> |
|
|
|
<script> |
|
|
|
function loadThemeForm(templateName) { |
|
|
|
const themeSelectorTemplate = document.querySelector(templateName) |
|
|
|
const form = themeSelectorTemplate.content.firstElementChild |
|
|
|
themeSelectorTemplate.replaceWith(form) |
|
|
|
|
|
|
|
form.addEventListener('change', (e) => { |
|
|
|
const chosenColorScheme = e.target.value |
|
|
|
localStorage.setItem('theme', chosenColorScheme) |
|
|
|
toggleTheme(chosenColorScheme) |
|
|
|
}) |
|
|
|
|
|
|
|
const selectedTheme = localStorage.getItem('theme') |
|
|
|
if (selectedTheme && selectedTheme !== 'undefined') { |
|
|
|
form.querySelector(`[value="${selectedTheme}"]`).checked = true |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
const prefersColorSchemeDark = '(prefers-color-scheme: dark)' |
|
|
|
window.addEventListener('load', () => { |
|
|
|
let hasDarkRules = false |
|
|
|
for (const styleSheet of Array.from(document.styleSheets)) { |
|
|
|
let mediaRules = [] |
|
|
|
for (const cssRule of styleSheet.cssRules) { |
|
|
|
if (cssRule.type !== CSSRule.MEDIA_RULE) { |
|
|
|
continue |
|
|
|
} |
|
|
|
// WARNING: Safari does not have/supports `conditionText`. |
|
|
|
if (cssRule.conditionText) { |
|
|
|
if (cssRule.conditionText !== prefersColorSchemeDark) { |
|
|
|
continue |
|
|
|
} |
|
|
|
} else { |
|
|
|
if (cssRule.cssText.startsWith(prefersColorSchemeDark)) { |
|
|
|
continue |
|
|
|
} |
|
|
|
} |
|
|
|
mediaRules = mediaRules.concat(Array.from(cssRule.cssRules)) |
|
|
|
} |
|
|
|
|
|
|
|
// WARNING: do not try to insert a Rule to a styleSheet you are |
|
|
|
// currently iterating on, otherwise the browser will be stuck |
|
|
|
// in a infinite loop… |
|
|
|
for (const mediaRule of mediaRules) { |
|
|
|
styleSheet.insertRule(mediaRule.cssText) |
|
|
|
hasDarkRules = true |
|
|
|
} |
|
|
|
} |
|
|
|
if (hasDarkRules) { |
|
|
|
loadThemeForm('#theme-selector') |
|
|
|
} |
|
|
|
}) |
|
|
|
</script> |
|
|
|
</body> |
|
|
|
</html> |
