davidbgk
/
larlet-fr-david-cache

<!doctype html><!-- This is a valid HTML5 document. -->
<!-- Screen readers, SEO, extensions and so on. -->
<html lang=fr>
<!-- Has to be within the first 1024 bytes, hence before the <title>
  See: https://www.w3.org/TR/2012/CR-html5-20121217/document-metadata.html#charset -->
<meta charset=utf-8>
<!-- Why no `X-UA-Compatible` meta: https://stackoverflow.com/a/6771584 -->
<!-- The viewport meta is quite crowded and we are responsible for that.
  See: https://codepen.io/tigt/post/meta-viewport-for-2015 -->
<meta name=viewport content="width=device-width,minimum-scale=1,initial-scale=1,shrink-to-fit=no">
<!-- Required to make a valid HTML5 document. -->
<title>HyperDB architecture (archive) — David Larlet</title>
<!-- Generated from https://realfavicongenerator.net/ such a mess. -->
<link rel="apple-touch-icon" sizes="180x180" href="/static/david/icons/apple-touch-icon.png">
<link rel="icon" type="image/png" sizes="32x32" href="/static/david/icons/favicon-32x32.png">
<link rel="icon" type="image/png" sizes="16x16" href="/static/david/icons/favicon-16x16.png">
<link rel="manifest" href="/manifest.json">
<link rel="mask-icon" href="/static/david/icons/safari-pinned-tab.svg" color="#5bbad5">
<link rel="shortcut icon" href="/static/david/icons/favicon.ico">
<meta name="apple-mobile-web-app-title" content="David Larlet">
<meta name="application-name" content="David Larlet">
<meta name="msapplication-TileColor" content="#da532c">
<meta name="msapplication-config" content="/static/david/icons/browserconfig.xml">
<meta name="theme-color" content="#f0f0ea">
<!-- That good ol' feed, subscribe :p. -->
<link rel=alternate type="application/atom+xml" title=Feed href="/david/log/">

  <meta name="robots" content="noindex, nofollow">
  <meta content="origin-when-cross-origin" name="referrer">
  <!-- Canonical URL for SEO purposes -->
  <link rel="canonical" href="https://github.com/mafintosh/hyperdb/blob/master/ARCHITECTURE.md">

<style>
/* http://meyerweb.com/eric/tools/css/reset/ */
html, body, div, span,
h1, h2, h3, h4, h5, h6, p, blockquote, pre,
a, abbr, address, big, cite, code,
del, dfn, em, img, ins,
small, strike, strong, tt, var,
dl, dt, dd, ol, ul, li,
fieldset, form, label, legend,
table, caption, tbody, tfoot, thead, tr, th, td,
article, aside, canvas, details, embed,
figure, figcaption, footer, header, hgroup,
menu, nav, output, ruby, section, summary,
time, mark, audio, video {
  margin: 0;
  padding: 0;
  border: 0;
  font-size: 100%;
  font: inherit;
  vertical-align: baseline;
}
/* HTML5 display-role reset for older browsers */
article, aside, details, figcaption, figure,
footer, header, hgroup, menu, nav, section { display: block; }
body { line-height: 1; }
blockquote, q { quotes: none; }
blockquote:before, blockquote:after,
q:before, q:after {
  content: '';
  content: none;
}
table {
  border-collapse: collapse;
  border-spacing: 0;
}

/* http://practicaltypography.com/equity.html */
/* https://calendar.perfplanet.com/2016/no-font-face-bulletproof-syntax/ */
/* https://www.filamentgroup.com/lab/js-web-fonts.html */
@font-face {
  font-family: 'EquityTextB';
  src: url('/static/david/css/fonts/Equity-Text-B-Regular-webfont.woff2') format('woff2'),
       url('/static/david/css/fonts/Equity-Text-B-Regular-webfont.woff') format('woff');
  font-weight: 300;
  font-style: normal;
  font-display: swap;
}
@font-face {
  font-family: 'EquityTextB';
  src: url('/static/david/css/fonts/Equity-Text-B-Italic-webfont.woff2') format('woff2'),
       url('/static/david/css/fonts/Equity-Text-B-Italic-webfont.woff') format('woff');
  font-weight: 300;
  font-style: italic;
  font-display: swap;
}
@font-face {
  font-family: 'EquityTextB';
  src: url('/static/david/css/fonts/Equity-Text-B-Bold-webfont.woff2') format('woff2'),
       url('/static/david/css/fonts/Equity-Text-B-Bold-webfont.woff') format('woff');
  font-weight: 700;
  font-style: normal;
  font-display: swap;
}

@font-face {
  font-family: 'ConcourseT3';
  src: url('/static/david/css/fonts/concourse_t3_regular-webfont-20190806.woff2') format('woff2'),
       url('/static/david/css/fonts/concourse_t3_regular-webfont-20190806.woff') format('woff');
  font-weight: 300;
  font-style: normal;
  font-display: swap;
}


/* http://practice.typekit.com/lesson/caring-about-opentype-features/ */
body {
  /* http://www.cssfontstack.com/ Palatino 99% Win 86% Mac */
  font-family: "EquityTextB", Palatino, serif;
  background-color: #f0f0ea;
  color: #07486c;
  font-kerning: normal;
  -moz-osx-font-smoothing: grayscale;
  -webkit-font-smoothing: subpixel-antialiased;
  text-rendering: optimizeLegibility;
  font-variant-ligatures: common-ligatures contextual;
  font-feature-settings: "kern", "liga", "clig", "calt";
}
pre, code, kbd, samp, var, tt {
  font-family: 'TriplicateT4c', monospace;
}
em {
  font-style: italic;
  color: #323a45;
}
strong {
  font-weight: bold;
  color: black;
}
nav {
  background-color: #323a45;
  color: #f0f0ea;
  display: flex;
  justify-content: space-around;
  padding: 1rem .5rem;
}
  nav:last-child {
    border-bottom: 1vh solid #2d7474;
  }
  nav a {
    color: #f0f0ea;
  }
  nav abbr {
    border-bottom: 1px dotted white;
  }

h1 {
  border-top: 1vh solid #2d7474;
  border-bottom: .2vh dotted #2d7474;
  background-color: #e3e1e1;
  color: #323a45;
  text-align: center;
  padding: 5rem 0 4rem 0;
  width: 100%;
  font-family: 'ConcourseT3';
  display: flex;
  flex-direction: column;
}
  h1.single {
    padding-bottom: 10rem;
  }
  h1 span {
    position: absolute;
    top: 1vh;
    left: 20%;
    line-height: 0;
  }
    h1 span a {
      line-height: 1.7;
      padding: 1rem 1.2rem .6rem 1.2rem;
      border-radius: 0 0 6% 6%;
      background: #2d7474;
      font-size: 1.3rem;
      color: white;
      text-decoration: none;
    }
h2 {
  margin: 4rem 0 1rem;
  border-top: .2vh solid #2d7474;
  padding-top: 1vh;
}
h3 {
  text-align: center;
  margin: 3rem 0 .75em;
}
hr {
  height: .4rem;
  width: .4rem;
  border-radius: .4rem;
  background: #07486c;
  margin: 2.5rem auto;
}
time {
  display: bloc;
  margin-left: 0 !important;
}
ul, ol {
  margin: 2rem;
}
ul {
  list-style-type: square;
}
a {
  text-decoration-skip-ink: auto;
  text-decoration-thickness: 0.05em;
  text-underline-offset: 0.09em;
}
article {
  max-width: 50rem;
  display: flex;
  flex-direction: column;
  margin: 2rem auto;
}
  article.single {
    border-top: .2vh dotted #2d7474;
    margin: -6rem auto 1rem auto;
    background: #f0f0ea;
    padding: 2rem;
  }
  article p:last-child {
    margin-bottom: 1rem;
  }
p {
  padding: 0 .5rem;
  margin-left: 3rem;
}
  p + p,
  figure + p {
    margin-top: 2rem;
  }

blockquote {
  background-color: #e3e1e1;
  border-left: .5vw solid #2d7474;
  display: flex;
  flex-direction: column;
  align-items: center;
  padding: 1rem;
  margin: 1.5rem;
}
  blockquote cite {
    font-style: italic;
  }
  blockquote p {
    margin-left: 0;
  }

figure {
  border-top: .2vh solid #2d7474;
  background-color: #e3e1e1;
  text-align: center;
  padding: 1.5rem 0;
  margin: 1rem 0 0;
  font-size: 1.5rem;
  width: 100%;
}
  figure img {
    max-width: 250px;
    max-height: 250px;
    border: .5vw solid #323a45;
    padding: 1px;
  }
figcaption {
  padding: 1rem;
  line-height: 1.4;
}
aside {
  display: flex;
  flex-direction: column;
  background-color: #e3e1e1;
  padding: 1rem 0;
  border-bottom: .2vh solid #07486c;
}
  aside p {
    max-width: 50rem;
    margin: 0 auto;
  }

/* https://fvsch.com/code/css-locks/ */
p, li, pre, code, kbd, samp, var, tt, time, details, figcaption {
  font-size: 1rem;
  line-height: calc( 1.5em + 0.2 * 1rem );
}
h1 {
  font-size: 1.9rem;
  line-height: calc( 1.2em + 0.2 * 1rem );
}
h2 {
  font-size: 1.6rem;
  line-height: calc( 1.3em + 0.2 * 1rem );
}
h3 {
  font-size: 1.35rem;
  line-height: calc( 1.4em + 0.2 * 1rem );
}
@media (min-width: 20em) {
  /* The (100vw - 20rem) / (50 - 20) part
     resolves to 0-1rem, depending on the
     viewport width (between 20em and 50em). */
  p, li, pre, code, kbd, samp, var, tt, time, details, figcaption {
    font-size: calc( 1rem + .6 * (100vw - 20rem) / (50 - 20) );
    line-height: calc( 1.5em + 0.2 * (100vw - 50rem) / (20 - 50) );
    margin-left: 0;
  }
  h1 {
    font-size: calc( 1.9rem + 1.5 * (100vw - 20rem) / (50 - 20) );
    line-height: calc( 1.2em + 0.2 * (100vw - 50rem) / (20 - 50) );
  }
  h2 {
    font-size: calc( 1.5rem + 1.5 * (100vw - 20rem) / (50 - 20) );
    line-height: calc( 1.3em + 0.2 * (100vw - 50rem) / (20 - 50) );
  }
  h3 {
    font-size: calc( 1.35rem + 1.5 * (100vw - 20rem) / (50 - 20) );
    line-height: calc( 1.4em + 0.2 * (100vw - 50rem) / (20 - 50) );
  }
}
@media (min-width: 50em) {
  /* The right part of the addition *must* be a
     rem value. In this example we *could* change
     the whole declaration to font-size:2.5rem,
     but if our baseline value was not expressed
     in rem we would have to use calc. */
  p, li, pre, code, kbd, samp, var, tt, time, details, figcaption {
    font-size: calc( 1rem + .6 * 1rem );
    line-height: 1.5em;
  }
  p, li, pre, details {
    margin-left: 3rem;
  }
  h1 {
    font-size: calc( 1.9rem + 1.5 * 1rem );
    line-height: 1.2em;
  }
  h2 {
    font-size: calc( 1.5rem + 1.5 * 1rem );
    line-height: 1.3em;
  }
  h3 {
    font-size: calc( 1.35rem + 1.5 * 1rem );
    line-height: 1.4em;
  }
  figure img {
    max-width: 500px;
    max-height: 500px;
  }
}

figure.unsquared {
  margin-bottom: 1.5rem;
}
  figure.unsquared img {
    height: inherit;
  }


@media print {
  body { font-size: 100%; }
  a:after { content: " (" attr(href) ")"; }
  a, a:link, a:visited, a:after {
    text-decoration: underline;
    text-shadow: none !important;
    background-image: none !important;
    background: white;
    color: black;
  }
  abbr[title] { border-bottom: 0; }
  abbr[title]:after { content: " (" attr(title) ")"; }
  img { page-break-inside: avoid; }
  @page { margin: 2cm .5cm; }
  h1, h2, h3 { page-break-after: avoid; }
  p3 { orphans: 3; widows: 3; }
  img {
    max-width: 250px !important;
    max-height: 250px !important;
  }
  nav, aside { display: none; }
}

ul.with_columns {
  column-count: 1;
}
@media (min-width: 20em) {
  ul.with_columns {
    column-count: 2;
  }
}
@media (min-width: 50em) {
  ul.with_columns {
    column-count: 3;
  }
}
ul.with_two_columns {
  column-count: 1;
}
@media (min-width: 20em) {
  ul.with_two_columns {
    column-count: 1;
  }
}
@media (min-width: 50em) {
  ul.with_two_columns {
    column-count: 2;
  }
}

.gallery {
  display: flex;
  flex-wrap: wrap;
  justify-content: space-around;
}
  .gallery figure img {
    margin-left: 1rem;
    margin-right: 1rem;
  }
  .gallery figure figcaption {
    font-family: 'ConcourseT3'
  }

footer {
  font-family: 'ConcourseT3';
  display: flex;
  flex-direction: column;
  border-top: 3px solid white;
  padding: 4rem 0;
  background-color: #07486c;
  color: white;
}
  footer > * {
    max-width: 50rem;
    margin: 0 auto;
  }
  footer a {
    color: #f1c40f;
  }
  footer .avatar {
    width: 200px;
    height: 200px;
    border-radius: 50%;
    float: left;
    -webkit-shape-outside: circle();
    shape-outside: circle();
    margin-right: 2rem;
    padding: 2px 5px 5px 2px;
    background: white;
    border-left: 1px solid #f1c40f;
    border-top: 1px solid #f1c40f;
    border-right: 5px solid #f1c40f;
    border-bottom: 5px solid #f1c40f;
  }
</style>

<h1>
  <span><a id="jumper" href="#jumpto" title="Un peu perdu ?">?</a></span>
  HyperDB architecture (archive)
  <time>Pour la pérennité des contenus liés. Non-indexé, retrait sur simple email.</time>
</h1>
<section>
  <article>
    <h3><a href="https://github.com/mafintosh/hyperdb/blob/master/ARCHITECTURE.md">Source originale du contenu</a></h3>
    <article class="markdown-body entry-content" itemprop="text"><h1><a id="user-content-hyperdb-architecture" class="anchor" aria-hidden="true" href="#hyperdb-architecture"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" aria-hidden="true"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"/></svg></a>HyperDB Architecture</h1>
<p>HyperDB is a scalable peer-to-peer key-value database.</p>
<h2><a id="user-content-filesystem-metaphor" class="anchor" aria-hidden="true" href="#filesystem-metaphor"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" aria-hidden="true"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"/></svg></a>Filesystem metaphor</h2>
<p>HyperDB is structured to be used much like a traditional hierarchical
filesystem. A value can be written and read at locations like <code>/foo/bar/baz</code>,
and the API supports querying or tracking values at subpaths, like how watching
for changes on <code>/foo/bar</code> will report both changes to <code>/foo/bar/baz</code> and also
<code>/foo/bar/19</code>.</p>
<h2><a id="user-content-set-of-append-only-logs-feeds" class="anchor" aria-hidden="true" href="#set-of-append-only-logs-feeds"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" aria-hidden="true"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"/></svg></a>Set of append-only logs (feeds)</h2>
<p>A HyperDB is fundamentally a set of
<a href="https://github.com/mafintosh/hypercore">hypercore</a>s. A <em>hypercore</em> is a secure
append-only log that is identified by a public key, and can only be written to
by the holder of the corresponding private key. Because it is append-only, old
values cannot be deleted nor modified. Because it is secure, a feed can be
downloaded from even untrustworthy peers and verified to be accurate. Any
modifications (malicious or otherwise) to the original feed data by someone
other than the author can be readily detected.</p>
<p>Each entry in a hypercore has a <em>sequence number</em>, that increments by 1 with
each write, starting at 0 (<code>seq=0</code>).</p>
<p>HyperDB builds its hierarchical key-value store on top of these hypercore feeds,
and also provides facilities for authorization, and replication of those member
hypercores.</p>
<h3><a id="user-content-directed-acyclic-graph" class="anchor" aria-hidden="true" href="#directed-acyclic-graph"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" aria-hidden="true"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"/></svg></a>Directed acyclic graph</h3>
<p>The combination of all operations performed on a HyperDB by all of its members
forms a DAG (<em>directed acyclic graph</em>). Each write to the database (setting a
key to a value) includes information to point backward at all of the known
"heads" in the graph.</p>
<p>To illustrate what this means, let's say Alice starts a new HyperDB and writes 2
values to it:</p>
<pre><code>// Feed

0 (/foo/bar = 'baz')
1 (/foo/2   = '{ "some": "json" }')


// Graph

Alice:  0  &lt;---  1
</code></pre>
<p>Where sequence number 1 (the second entry) refers to sequence number 0 on the
same feed (Alice's).</p>
<p>Now Alice <em>authorizes</em> Bob to write to the HyperDB. Internally, this means Alice
writes a special message to her feed saying that Bob's feed (identified by his
public key) should be read and replicated in by other participants. Her feed
becomes</p>
<pre><code>// Feed

0 (/foo/bar = 'baz')
1 (/foo/2   = '{ "some": "json" }')
2 (''       = '')


// Graph

Alice: 0  &lt;---  1  &lt;---  2
</code></pre>
<p>Authorization is formatted internally in a special way so that it isn't
interpreted as a key/value pair.</p>
<p>Now Bob writes a value to his feed, and then Alice and Bob sync. The result is:</p>
<pre><code>// Feed

//// Alice
0 (/foo/bar = 'baz')
1 (/foo/2   = '{ "some": "json" }')
2 (''       = '')

//// Bob
0 (/a/b     = '12')


// Graph

Alice: 0  &lt;---  1  &lt;---  2
Bob  : 0
</code></pre>
<p>Notice that none of Alice's entries refer to Bob's, and vice versa. This is
because neither has written any entries to their feeds since the two became
aware of each other (authorized &amp; replicated each other's feeds).</p>
<p>Right now there are two "heads" of the graph: Alice's feed at seq 2, and Bob's
feed at seq 0.</p>
<p>Next, Alice writes a new value, and her latest entry will refer to Bob's:</p>
<pre><code>// Feed

//// Alice
0 (/foo/bar = 'baz')
1 (/foo/2   = '{ "some": "json" }')
2 (''       = '')
3 (/foo/hup = 'beep')

//// Bob
0 (/a/b     = '12')


// Graph

Alice: 0  &lt;---  1  &lt;---  2  &lt;--/  3
Bob  : 0  &lt;-------------------/
</code></pre>
<p>Because Alice's latest feed entry refers to Bob's latest feed entry, there is
now only one "head" in the database. That means there is enough information in
Alice's seq=3 entry to find any other key in the database. In the last example,
there were two heads (Alice's seq=2 and Bob's seq=0); both of which would need
to be read internally in order to locate any key in the database.</p>
<p>Now there is only one "head": Alice's feed at seq 3.</p>
<h2><a id="user-content-authorization" class="anchor" aria-hidden="true" href="#authorization"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" aria-hidden="true"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"/></svg></a>Authorization</h2>
<p>The set of hypercores are <em>authorized</em> in that the original author of the first
hypercore in a hyperdb must explicitly denote in their append-only log that the
public key of a new hypercore is permitted to edit the database. Any authorized
member may authorize more members. There is no revocation or other author
management elements currently.</p>
<h2><a id="user-content-incremental-index" class="anchor" aria-hidden="true" href="#incremental-index"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" aria-hidden="true"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"/></svg></a>Incremental index</h2>
<p>HyperDB builds an <em>incremental index</em> with every new key/value pairs ("nodes")
written. This means a separate data structure doesn't need to be maintained
elsewhere for fast writes and lookups: each node written has enough information
to look up any other key quickly and otherwise navigate the database.</p>
<p>Each node stores the following basic information:</p>
<ul>
<li><code>key</code>: the key that is being created or modified. e.g. <code>/home/sww/dev.md</code></li>
<li><code>value</code>: the value stored at that key.</li>
<li><code>seq</code>: the sequence number of this entry in the owner's hypercore. 0 is the
first, 1 the second, and so forth.</li>
<li><code>feed</code>: the ID of the hypercore writer that wrote this</li>
<li><code>path</code>: a 2-bit hash sequence of the key's components</li>
<li><code>trie</code>: a navigation structure used with <code>path</code> to find a desired key</li>
<li><code>clock</code>: vector clock to determine node insertion causality</li>
<li><code>feeds</code>: an array of { feedKey, seq } for decoding a <code>clock</code></li>
</ul>
<h3><a id="user-content-vector-clock" class="anchor" aria-hidden="true" href="#vector-clock"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" aria-hidden="true"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"/></svg></a>Vector clock</h3>
<p>Each node stores a <a href="https://en.wikipedia.org/wiki/Vector_clock" rel="nofollow">vector clock</a> of
the last known sequence number from each feed it knows about. This is what forms
the DAG structure.</p>
<p>A vector clock on a node of, say, <code>[0, 2, 5]</code> means:</p>
<ul>
<li>when this node was written, the largest seq # in my local fed is 0</li>
<li>when this node was written, the largest seq # in the second feed I have is 2</li>
<li>when this node was written, the largest seq # in the third feed I have is 5</li>
</ul>
<p>For example, Bob's vector clock for Alice's seq=3 entry above would be <code>[0, 3]</code>
since he knows of her latest entry (seq=3) and his own (seq=0).</p>
<p>The vector clock is used for correctly traversing history. This is necessary for
the <code>db#heads</code> API as well as <code>db#createHistoryStream</code>.</p>
<h3><a id="user-content-prefix-trie" class="anchor" aria-hidden="true" href="#prefix-trie"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" aria-hidden="true"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"/></svg></a>Prefix trie</h3>
<p>Given a HyperDB with hundreds of entries, how can a key like <code>/a/b/c</code> be looked
up quickly?</p>
<p>Each node stores a <em>prefix <a href="https://en.wikipedia.org/wiki/Trie" rel="nofollow">trie</a></em> that
assists with finding the shortest path to the desired key.</p>
<p>When a node is written, its <em>prefix hash</em> is computed. This done by first
splitting the key into its components (<code>a</code>, <code>b</code>, and <code>c</code> for <code>/a/b/c</code>), and then
hashing each component into a 32-character hash, where one character is a 2-bit
value (0, 1, 2, or 3). The <code>prefix</code> hash for <code>/a/b/c</code> is</p>
<div class="highlight highlight-source-js"><pre><span class="pl-smi">node</span>.<span class="pl-smi">path</span> <span class="pl-k">=</span> [
<span class="pl-c1">1</span>, <span class="pl-c1">2</span>, <span class="pl-c1">0</span>, <span class="pl-c1">1</span>, <span class="pl-c1">2</span>, <span class="pl-c1">0</span>, <span class="pl-c1">2</span>, <span class="pl-c1">2</span>, <span class="pl-c1">3</span>, <span class="pl-c1">0</span>, <span class="pl-c1">1</span>, <span class="pl-c1">2</span>, <span class="pl-c1">1</span>, <span class="pl-c1">3</span>, <span class="pl-c1">0</span>, <span class="pl-c1">3</span>, <span class="pl-c1">0</span>, <span class="pl-c1">0</span>, <span class="pl-c1">2</span>, <span class="pl-c1">1</span>, <span class="pl-c1">0</span>, <span class="pl-c1">2</span>, <span class="pl-c1">0</span>, <span class="pl-c1">0</span>, <span class="pl-c1">2</span>, <span class="pl-c1">0</span>, <span class="pl-c1">0</span>, <span class="pl-c1">3</span>, <span class="pl-c1">2</span>, <span class="pl-c1">1</span>, <span class="pl-c1">1</span>, <span class="pl-c1">2</span>,
<span class="pl-c1">0</span>, <span class="pl-c1">1</span>, <span class="pl-c1">2</span>, <span class="pl-c1">3</span>, <span class="pl-c1">2</span>, <span class="pl-c1">2</span>, <span class="pl-c1">2</span>, <span class="pl-c1">0</span>, <span class="pl-c1">3</span>, <span class="pl-c1">1</span>, <span class="pl-c1">1</span>, <span class="pl-c1">3</span>, <span class="pl-c1">0</span>, <span class="pl-c1">3</span>, <span class="pl-c1">1</span>, <span class="pl-c1">3</span>, <span class="pl-c1">0</span>, <span class="pl-c1">1</span>, <span class="pl-c1">0</span>, <span class="pl-c1">1</span>, <span class="pl-c1">3</span>, <span class="pl-c1">2</span>, <span class="pl-c1">0</span>, <span class="pl-c1">2</span>, <span class="pl-c1">2</span>, <span class="pl-c1">3</span>, <span class="pl-c1">2</span>, <span class="pl-c1">2</span>, <span class="pl-c1">3</span>, <span class="pl-c1">3</span>, <span class="pl-c1">2</span>, <span class="pl-c1">3</span>,
<span class="pl-c1">0</span>, <span class="pl-c1">1</span>, <span class="pl-c1">1</span>, <span class="pl-c1">0</span>, <span class="pl-c1">1</span>, <span class="pl-c1">2</span>, <span class="pl-c1">3</span>, <span class="pl-c1">2</span>, <span class="pl-c1">2</span>, <span class="pl-c1">2</span>, <span class="pl-c1">0</span>, <span class="pl-c1">0</span>, <span class="pl-c1">3</span>, <span class="pl-c1">1</span>, <span class="pl-c1">2</span>, <span class="pl-c1">1</span>, <span class="pl-c1">3</span>, <span class="pl-c1">3</span>, <span class="pl-c1">3</span>, <span class="pl-c1">3</span>, <span class="pl-c1">3</span>, <span class="pl-c1">3</span>, <span class="pl-c1">0</span>, <span class="pl-c1">3</span>, <span class="pl-c1">3</span>, <span class="pl-c1">2</span>, <span class="pl-c1">3</span>, <span class="pl-c1">2</span>, <span class="pl-c1">3</span>, <span class="pl-c1">0</span>, <span class="pl-c1">1</span>, <span class="pl-c1">0</span>,
<span class="pl-c1">4</span> ]</pre></div>
<p>Each component is divided by a newline. <code>4</code> is a special value indicating the
end of the prefix.</p>
<h4><a id="user-content-example" class="anchor" aria-hidden="true" href="#example"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" aria-hidden="true"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"/></svg></a>Example</h4>
<p>Consider a fresh HyperDB. We write <code>/a/b = 24</code> and get back this node:</p>
<div class="highlight highlight-source-js"><pre>{ key<span class="pl-k">:</span> <span class="pl-s"><span class="pl-pds">'</span>/a/b<span class="pl-pds">'</span></span>,
  value<span class="pl-k">:</span> <span class="pl-s"><span class="pl-pds">'</span>24<span class="pl-pds">'</span></span>,
  clock<span class="pl-k">:</span> [ <span class="pl-c1">0</span> ],
  trie<span class="pl-k">:</span> [],
  feeds<span class="pl-k">:</span> [ [<span class="pl-c1">Object</span>] ],
  feedSeq<span class="pl-k">:</span> <span class="pl-c1">0</span>,
  feed<span class="pl-k">:</span> <span class="pl-c1">0</span>,
  seq<span class="pl-k">:</span> <span class="pl-c1">0</span>,
  path<span class="pl-k">:</span>
   [ <span class="pl-c1">1</span>, <span class="pl-c1">2</span>, <span class="pl-c1">0</span>, <span class="pl-c1">1</span>, <span class="pl-c1">2</span>, <span class="pl-c1">0</span>, <span class="pl-c1">2</span>, <span class="pl-c1">2</span>, <span class="pl-c1">3</span>, <span class="pl-c1">0</span>, <span class="pl-c1">1</span>, <span class="pl-c1">2</span>, <span class="pl-c1">1</span>, <span class="pl-c1">3</span>, <span class="pl-c1">0</span>, <span class="pl-c1">3</span>, <span class="pl-c1">0</span>, <span class="pl-c1">0</span>, <span class="pl-c1">2</span>, <span class="pl-c1">1</span>, <span class="pl-c1">0</span>, <span class="pl-c1">2</span>, <span class="pl-c1">0</span>, <span class="pl-c1">0</span>, <span class="pl-c1">2</span>, <span class="pl-c1">0</span>, <span class="pl-c1">0</span>, <span class="pl-c1">3</span>, <span class="pl-c1">2</span>, <span class="pl-c1">1</span>, <span class="pl-c1">1</span>, <span class="pl-c1">2</span>,
     <span class="pl-c1">0</span>, <span class="pl-c1">1</span>, <span class="pl-c1">2</span>, <span class="pl-c1">3</span>, <span class="pl-c1">2</span>, <span class="pl-c1">2</span>, <span class="pl-c1">2</span>, <span class="pl-c1">0</span>, <span class="pl-c1">3</span>, <span class="pl-c1">1</span>, <span class="pl-c1">1</span>, <span class="pl-c1">3</span>, <span class="pl-c1">0</span>, <span class="pl-c1">3</span>, <span class="pl-c1">1</span>, <span class="pl-c1">3</span>, <span class="pl-c1">0</span>, <span class="pl-c1">1</span>, <span class="pl-c1">0</span>, <span class="pl-c1">1</span>, <span class="pl-c1">3</span>, <span class="pl-c1">2</span>, <span class="pl-c1">0</span>, <span class="pl-c1">2</span>, <span class="pl-c1">2</span>, <span class="pl-c1">3</span>, <span class="pl-c1">2</span>, <span class="pl-c1">2</span>, <span class="pl-c1">3</span>, <span class="pl-c1">3</span>, <span class="pl-c1">2</span>, <span class="pl-c1">3</span>,
     <span class="pl-c1">4</span> ] }</pre></div>
<p>If you compare this path to the one for <code>/a/b/c</code> above, you'll see that the
first 64 2-bit characters match. This is because <code>/a/b</code> is a prefix of <code>/a/b/c</code>.</p>
<p>Since this is the first entry, <code>seq</code> is 0. Since this is the only known feed,
<code>feed</code> is also 0. <code>feeds</code> is an array of entries of the form <code>{ key: Buffer, seq: Number }</code> that let you map the numeric value <code>feed</code> to a hypercore key and
its sequence number head. <code>feeds</code> isn't always set: it only gets included when
it changes compared to <code>node.seq - 1</code>, in the interest of storing less data per
node.</p>
<p>Now we write <code>/a/c = hello</code> and get this node:</p>
<div class="highlight highlight-source-js"><pre>{ key<span class="pl-k">:</span> <span class="pl-s"><span class="pl-pds">'</span>/a/c<span class="pl-pds">'</span></span>,
  value<span class="pl-k">:</span> <span class="pl-s"><span class="pl-pds">'</span>hello<span class="pl-pds">'</span></span>,
  clock<span class="pl-k">:</span> [ <span class="pl-c1">0</span> ],
  trie<span class="pl-k">:</span> [ , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , [ , , [ { feed<span class="pl-k">:</span> <span class="pl-c1">0</span>, seq<span class="pl-k">:</span> <span class="pl-c1">0</span> } ] ] ],
  feeds<span class="pl-k">:</span> [],
  feedSeq<span class="pl-k">:</span> <span class="pl-c1">0</span>,
  feed<span class="pl-k">:</span> <span class="pl-c1">0</span>,
  seq<span class="pl-k">:</span> <span class="pl-c1">1</span>,
  path<span class="pl-k">:</span>
   [ <span class="pl-c1">1</span>, <span class="pl-c1">2</span>, <span class="pl-c1">0</span>, <span class="pl-c1">1</span>, <span class="pl-c1">2</span>, <span class="pl-c1">0</span>, <span class="pl-c1">2</span>, <span class="pl-c1">2</span>, <span class="pl-c1">3</span>, <span class="pl-c1">0</span>, <span class="pl-c1">1</span>, <span class="pl-c1">2</span>, <span class="pl-c1">1</span>, <span class="pl-c1">3</span>, <span class="pl-c1">0</span>, <span class="pl-c1">3</span>, <span class="pl-c1">0</span>, <span class="pl-c1">0</span>, <span class="pl-c1">2</span>, <span class="pl-c1">1</span>, <span class="pl-c1">0</span>, <span class="pl-c1">2</span>, <span class="pl-c1">0</span>, <span class="pl-c1">0</span>, <span class="pl-c1">2</span>, <span class="pl-c1">0</span>, <span class="pl-c1">0</span>, <span class="pl-c1">3</span>, <span class="pl-c1">2</span>, <span class="pl-c1">1</span>, <span class="pl-c1">1</span>, <span class="pl-c1">2</span>,
     <span class="pl-c1">0</span>, <span class="pl-c1">1</span>, <span class="pl-c1">1</span>, <span class="pl-c1">0</span>, <span class="pl-c1">1</span>, <span class="pl-c1">2</span>, <span class="pl-c1">3</span>, <span class="pl-c1">2</span>, <span class="pl-c1">2</span>, <span class="pl-c1">2</span>, <span class="pl-c1">0</span>, <span class="pl-c1">0</span>, <span class="pl-c1">3</span>, <span class="pl-c1">1</span>, <span class="pl-c1">2</span>, <span class="pl-c1">1</span>, <span class="pl-c1">3</span>, <span class="pl-c1">3</span>, <span class="pl-c1">3</span>, <span class="pl-c1">3</span>, <span class="pl-c1">3</span>, <span class="pl-c1">3</span>, <span class="pl-c1">0</span>, <span class="pl-c1">3</span>, <span class="pl-c1">3</span>, <span class="pl-c1">2</span>, <span class="pl-c1">3</span>, <span class="pl-c1">2</span>, <span class="pl-c1">3</span>, <span class="pl-c1">0</span>, <span class="pl-c1">1</span>, <span class="pl-c1">0</span>,
     <span class="pl-c1">4</span> ] }</pre></div>
<p>As expected, this node has the same <code>feed</code> value as before (since we're only
writing to one feed). Its <code>seq</code> is 1, since the last was 0. Notice that <code>feeds</code>
isn't included, because the mapping of the numeric <code>feed</code> value to a key hasn't
changed.</p>
<p>Also, this and the previous node have the first 32 characters of their <code>path</code> in
common (the prefix <code>/a</code>).</p>
<p>Notice though that <code>trie</code> is set. It's a long but sparse array. It has 35
entries, with the last one referencing the first node inserted (<code>a/b/</code>). Why?</p>
<p>(If it wasn't stored as a sparse array, you'd actually see 64 entries (the
length of the <code>path</code>). But since the other 29 entries are also empty, hyperdb
doesn't bother allocating them.)</p>
<p>If you visually compare this node's <code>path</code> with the previous node's <code>path</code>, how
many entries do they have in common? At which entry do the 2-bit numbers
diverge?</p>
<p>At the 35th entry.</p>
<p>What this is saying is "if the hash of the key you're looking for differs from
mine on the 35th entry, you want to travel to <code>{ feed: 0, seq: 0 }</code> to find the
node you're looking for.</p>
<p>This is how finding a node works, starting at any other node:</p>
<ol>
<li>Compute the 2-bit hash sequence of the key you're after (e.g. <code>a/b</code>)</li>
<li>Lookup the newest entry in the feed.</li>
<li>Compare its <code>path</code> against the hash you just computed.</li>
<li>If you discover that the <code>path</code> and your hash match, then this is the node
you're looking for!</li>
<li>Otherwise, once a 2-bit character from <code>path</code> and your hash disagree, note
the index # where they differ and look up that value in the node's <code>trie</code>.
Fetch that node at the given feed and sequence number, and go back to step 3.
Repeat until you reach step 4 (match) or there is no entry in the node's trie
for the key you're after (no match).</li>
</ol>
<p>What if there are multiple feeds in the HyperDB? The lookup algorithm changes
slightly. Replace the above step 2 for:</p>
<blockquote>
<ol start="2">
<li>Fetch the latest entry from <em>every</em> feed. For each head node, proceed to
the next step.</li>
</ol>
</blockquote>
<p>The other steps are the same as before.</p>
</article>
  </article>
</section>


<nav id="jumpto">
  <p>
    <a href="/david/blog/">Accueil du blog</a> |
    <a href="https://github.com/mafintosh/hyperdb/blob/master/ARCHITECTURE.md">Source originale</a> |
    <a href="/david/stream/2019/">Accueil du flux</a>
  </p>
</nav>

<footer>
  <div>
    <img src="/static/david/david-larlet-avatar.jpg" loading="lazy" class="avatar" width="200" height="200">
    <p>
      Bonjour/Hi!
      Je suis <a href="/david/" title="Profil public">David&nbsp;Larlet</a>, je vis actuellement à Montréal et j’alimente cet espace depuis 15 ans. <br>
      Si tu as apprécié cette lecture, n’hésite pas à poursuivre ton exploration. Par exemple via les <a href="/david/blog/" title="Expériences bienveillantes">réflexions bimestrielles</a>, la <a href="/david/stream/2019/" title="Pensées (dés)articulées">veille hebdomadaire</a> ou en t’abonnant au <a href="/david/log/" title="S’abonner aux publications via RSS">flux RSS</a> (<a href="/david/blog/2019/flux-rss/" title="Tiens c’est quoi un flux RSS ?">so 2005</a>).
    </p>
    <p>
      Je m’intéresse à la place que je peux avoir dans ce monde. En tant qu’humain, en tant que membre d’une famille et en tant qu’associé d’une coopérative. De temps en temps, je fais aussi des <a href="https://github.com/davidbgk" title="Principalement sur Github mais aussi ailleurs">trucs techniques</a>. Et encore plus rarement, <a href="/david/talks/" title="En ce moment je laisse plutôt la place aux autres">j’en parle</a>.
    </p>
    
    <p>
      Voici quelques articles choisis :
      <a href="/david/blog/2019/faire-equipe/" title="Accéder à l’article complet">Faire équipe</a>,
      <a href="/david/blog/2018/bivouac-automnal/" title="Accéder à l’article complet">Bivouac automnal</a>,
      <a href="/david/blog/2018/commodite-effondrement/" title="Accéder à l’article complet">Commodité et effondrement</a>,
      <a href="/david/blog/2017/donnees-communs/" title="Accéder à l’article complet">Des données aux communs</a>,
      <a href="/david/blog/2016/accompagner-enfant/" title="Accéder à l’article complet">Accompagner un enfant</a>,
      <a href="/david/blog/2016/senior-developer/" title="Accéder à l’article complet">Senior developer</a>,
      <a href="/david/blog/2016/illusion-sociale/" title="Accéder à l’article complet">L’illusion sociale</a>,
      <a href="/david/blog/2016/instantane-scopyleft/" title="Accéder à l’article complet">Instantané Scopyleft</a>,
      <a href="/david/blog/2016/enseigner-web/" title="Accéder à l’article complet">Enseigner le Web</a>,
      <a href="/david/blog/2016/simplicite-defaut/" title="Accéder à l’article complet">Simplicité par défaut</a>,
      <a href="/david/blog/2016/minimalisme-esthetique/" title="Accéder à l’article complet">Minimalisme et esthétique</a>,
      <a href="/david/blog/2014/un-web-omni-present/" title="Accéder à l’article complet">Un web omni-présent</a>,
      <a href="/david/blog/2014/manifeste-developpeur/" title="Accéder à l’article complet">Manifeste de développeur</a>,
      <a href="/david/blog/2013/confort-convivialite/" title="Accéder à l’article complet">Confort et convivialité</a>,
      <a href="/david/blog/2013/testament-numerique/" title="Accéder à l’article complet">Testament numérique</a>,
      et <a href="/david/blog/" title="Accéder aux archives">bien d’autres…</a>
    </p>
    <p>
      On peut <a href="mailto:david%40larlet.fr" title="Envoyer un courriel">échanger par courriel</a>. Si éventuellement tu souhaites que l’on travaille ensemble, tu devrais commencer par consulter le <a href="http://larlet.com">profil dédié à mon activité professionnelle</a> et/ou contacter directement <a href="http://scopyleft.fr/">scopyleft</a>, la <abbr title="Société coopérative et participative">SCOP</abbr> dont je fais partie depuis six ans. Je recommande au préalable de lire <a href="/david/blog/2018/cout-site/" title="Attention ce qui va suivre peut vous choquer">combien coûte un site</a> et pourquoi je suis plutôt favorable à une <a href="/david/pro/devis/" title="Discutons-en !">non-demande de devis</a>.
    </p>
    <p>
      Je ne traque pas ta navigation mais mon
      <abbr title="Alwaysdata, 62 rue Tiquetonne 75002 Paris, +33.184162340">hébergeur</abbr>
      conserve des logs d’accès.
    </p>
  </div>
</footer>
<script type="text/javascript">
  ;(_ => {
    const jumper = document.getElementById('jumper')
    jumper.addEventListener('click', e => {
      e.preventDefault()
      const anchor = e.target.getAttribute('href')
      const targetEl = document.getElementById(anchor.substring(1))
      targetEl.scrollIntoView({behavior: 'smooth'})
    })
  })()
</script>