Repository with sources and generator of https://larlet.fr/david/ https://larlet.fr/david/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

2 年之前
1 年之前
3 年之前
3 年之前
2 年之前
2 年之前
2 年之前
2 年之前
1 年之前
1 年之前
2 年之前
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624
  1. #!/usr/bin/env python3
  2. import json
  3. import locale
  4. from collections import defaultdict
  5. from dataclasses import dataclass
  6. from datetime import datetime, timedelta
  7. from html import escape
  8. from itertools import groupby
  9. from pathlib import Path
  10. from string import Template
  11. from textwrap import dedent
  12. from time import perf_counter
  13. import mistune
  14. from jinja2 import Environment as Env
  15. from jinja2 import FileSystemLoader
  16. from jinja2.filters import do_striptags
  17. from minicli import cli, run, wrap
  18. from mistune.directives import DirectiveInclude
  19. from PIL import Image
  20. from slugify import slugify
  21. from typography import typographie
  22. from widont import widont
  23. # Useful for dates rendering within Jinja2.
  24. locale.setlocale(locale.LC_ALL, "fr_FR.UTF-8")
  25. HERE = Path(".")
  26. DAVID = HERE / "david"
  27. STATIC = HERE / ".." / "larlet-fr-static"
  28. DOMAIN = "https://larlet.fr"
  29. LOCAL_DOMAIN = "http://larlet.test:3579"
  30. # Hardcoding publication at 12 in Paris timezone.
  31. NORMALIZED_STRFTIME = "%Y-%m-%dT12:00:00+01:00"
  32. TODAY = datetime.today() + timedelta(hours=6)
  33. PUBLICATION_BUFFER = TODAY - timedelta(days=0)
  34. NB_ITEMS_IN_FEED = 30
  35. SOURCES_PATH = DAVID / "2023" / "_sources"
  36. all_tags = set()
  37. pages_by_tags = defaultdict(list)
  38. pages_by_url = {}
  39. class MarkParser(mistune.InlineParser):
  40. """Parses `==foo==` as `<mark>foo</mark>`."""
  41. MARK = (
  42. r"(\={2})(?=[^\s*])("
  43. r"(?:\\[\\*]|[^*])*"
  44. r"(?:" + mistune.InlineParser.ESCAPE + r"|[^\s*]))\1"
  45. )
  46. RULE_NAMES = mistune.InlineParser.RULE_NAMES + ("mark",)
  47. def parse_mark(self, m, state):
  48. _ = m.group(1)
  49. text = m.group(2)
  50. return "mark", self.render(text, state)
  51. class MarkRenderer(mistune.HTMLRenderer):
  52. """To use in conjunction with `MarkParser`."""
  53. def mark(self, text):
  54. return "<mark>" + text + "</mark>"
  55. class TagsRenderer(mistune.HTMLRenderer):
  56. """Make the asumption each line starting with a `#` is a tag."""
  57. def paragraph(self, text):
  58. if text.startswith("#"):
  59. tags = " ".join(
  60. f'<a href="/david/2023/{slugify(tag.strip())}/">#{tag.strip()}</a>'
  61. for tag in text.split("#")
  62. if tag.strip()
  63. )
  64. return f"<nav><p>{tags}</p></nav>\n"
  65. return super().paragraph(text)
  66. class FrenchTypographyRenderer(mistune.HTMLRenderer):
  67. """Apply French typographic rules to text."""
  68. def text(self, text):
  69. return typographie(super().text(text), html=True)
  70. def block_html(self, html):
  71. return typographie(super().block_html(html), html=True)
  72. class InternalLinkTitleRenderer(mistune.HTMLRenderer):
  73. """Automatically generate the title for internal links."""
  74. def link(self, link, text=None, title=None):
  75. if text is None:
  76. text = link
  77. s = '<a href="' + self._safe_url(link) + '"'
  78. if not title and link.startswith("/david/2023/"):
  79. # It will not work for internal links referencing the future.
  80. page = pages_by_url.get(link)
  81. if page:
  82. title = page.title
  83. if title:
  84. s += ' title="' + mistune.escape_html(title) + '"'
  85. return s + ">" + (text or link) + "</a>"
  86. class CustomAndBlockquoteLanguageRenderer(
  87. FrenchTypographyRenderer, InternalLinkTitleRenderer, MarkRenderer, TagsRenderer
  88. ):
  89. """Sets the English language attribute for blockquotes with `[en]` prefix."""
  90. def _get_language(self, text):
  91. if text.startswith("<p>[en] "):
  92. return "en", text.replace("<p>[en] ", "<p>")
  93. else:
  94. return None, text
  95. def block_quote(self, text):
  96. language, text = self._get_language(text)
  97. if language:
  98. return f'\n<blockquote lang="{language}">\n{text}</blockquote>\n'
  99. else:
  100. return f"\n<blockquote>\n{text}</blockquote>\n"
  101. class ImgsWithSizesRenderer(CustomAndBlockquoteLanguageRenderer):
  102. """Renders images as <figure>s and add sizes."""
  103. def paragraph(self, text):
  104. # In case of a figure, we do not want the (non-standard) paragraph.
  105. if text.strip().startswith("<figure>"):
  106. return text
  107. return super().paragraph(text)
  108. def _generate_size(self, src, width, height):
  109. src_size = src.replace(".jpg", f"_{width}x{height}.jpg")
  110. full_path = STATIC / Path(src[1:])
  111. full_path_size = STATIC / Path(src_size[1:])
  112. if full_path_size.exists() or "/2023/" not in src:
  113. return src_size
  114. image = Image.open(full_path)
  115. image.thumbnail((width, height), resample=Image.LANCZOS)
  116. image.save(full_path_size, icc_profile=image.info.get("icc_profile"))
  117. return src_size
  118. def _generate_webp(self, src):
  119. src_webp = src.replace(".jpg", ".webp")
  120. full_path = STATIC / Path(src[1:])
  121. full_path_webp = STATIC / Path(src_webp[1:])
  122. if full_path_webp.exists() or "/2023/" not in src:
  123. return src_webp
  124. image = Image.open(full_path)
  125. image.save(
  126. full_path_webp, format="webp", icc_profile=image.info.get("icc_profile")
  127. )
  128. # command = [
  129. # "cwebp",
  130. # "-q",
  131. # "80",
  132. # full_path,
  133. # "-o",
  134. # full_path_webp,
  135. # "-metadata",
  136. # "icc",
  137. # ]
  138. # subprocess.check_output(command, stderr=subprocess.STDOUT)
  139. return src_webp
  140. def image(self, src, alt="", title=None):
  141. SIZES = [(660, 440), (990, 660), (1320, 880)]
  142. full_path = STATIC / Path(src[1:])
  143. image = Image.open(full_path)
  144. width, height = image.size
  145. jpg_srcs = [(src, width, height)]
  146. # src_webp = self._generate_webp(src)
  147. # webp_srcs = [(src_webp, width, height)]
  148. for size_width, size_height in SIZES:
  149. src_size = self._generate_size(src, size_width, size_height)
  150. jpg_srcs.append((src_size, size_width, size_height))
  151. # src_size_webp = self._generate_webp(src_size)
  152. # webp_srcs.append((src_size_webp, size_width, size_height))
  153. jpg_srcsets = ", ".join(
  154. f"{jpg_src} {jpg_width}w" for jpg_src, jpg_width, jpg_height in jpg_srcs
  155. )
  156. # webp_srcsets = ", ".join(
  157. # f"{webp_src} {webp_width}w"
  158. # for webp_src, webp_width, webp_height in webp_srcs
  159. # )
  160. return dedent(
  161. f"""\
  162. <figure>
  163. <a href="{src}"
  164. title="Cliquer pour une version haute résolution">
  165. <img
  166. src="{src}"
  167. width="{width}" height="{height}"
  168. srcset="{jpg_srcsets}"
  169. sizes="min(100vw, calc(100vh * {width} / {height}))"
  170. loading="lazy"
  171. decoding="async"
  172. alt="{alt}">
  173. </a>
  174. <figcaption>{title}</figcaption>
  175. </figure>
  176. """
  177. )
  178. class H2AnchorsRenderer(ImgsWithSizesRenderer):
  179. """Custom renderer for H2 titles with anchors."""
  180. def heading(self, text, level):
  181. if level == 2:
  182. slug = slugify(text)
  183. return (
  184. f'<h2 id="{slug}">'
  185. f"{text} "
  186. f'<a href="#{slug}" title="Ancre vers cette partie">#</a>'
  187. f"</h2>"
  188. )
  189. else:
  190. return super().heading(text, level)
  191. # We want a custom renderer to create a hash/link for each H2 headings.
  192. markdown_with_h2_anchors = mistune.Markdown(
  193. renderer=H2AnchorsRenderer(escape=False),
  194. inline=MarkParser(H2AnchorsRenderer(escape=False)),
  195. plugins=[DirectiveInclude()],
  196. )
  197. # The second markdown is pertinent to generate articles for the feed,
  198. # we do not need anchors in that case.
  199. markdown_with_img_sizes = mistune.Markdown(
  200. renderer=ImgsWithSizesRenderer(escape=False),
  201. inline=MarkParser(ImgsWithSizesRenderer(escape=False)),
  202. plugins=[DirectiveInclude()],
  203. )
  204. # This is the jinja2 configuration to locate templates.
  205. environment = Env(loader=FileSystemLoader(str(DAVID / "templates")))
  206. def neighborhood(iterable, first=None, last=None):
  207. """
  208. Yield the (previous, current, next) items given an iterable.
  209. You can specify a `first` and/or `last` item for bounds.
  210. """
  211. iterator = iter(iterable)
  212. previous = first
  213. current = next(iterator) # Throws StopIteration if empty.
  214. for next_ in iterator:
  215. yield (previous, current, next_)
  216. previous = current
  217. current = next_
  218. yield (previous, current, last)
  219. def each_file_from(source_dir, pattern="*", exclude=None):
  220. """Walk across the `source_dir` and return the `pattern` file paths."""
  221. for path in _each_path_from(source_dir, pattern=pattern, exclude=exclude):
  222. if path.is_file():
  223. yield path
  224. def each_folder_from(source_dir, exclude=None):
  225. """Walk across the `source_dir` and return the folder paths."""
  226. for path in _each_path_from(source_dir, exclude=exclude):
  227. if path.is_dir():
  228. yield path
  229. def _each_path_from(source_dir, pattern="*", exclude=None):
  230. for path in sorted(Path(source_dir).glob(pattern)):
  231. if exclude is not None and path.name in exclude:
  232. continue
  233. yield path
  234. @dataclass
  235. class Page:
  236. title: str
  237. content: str
  238. tags: list
  239. file_path: str
  240. lang: str = "fr"
  241. def __post_init__(self):
  242. try:
  243. date_str, _ = self.file_path.split(" - ", 1)
  244. except ValueError:
  245. # Fallback for 2020 contents (search index)
  246. suffix = len(".md")
  247. prefix = len("YYYY/MM-DD") + suffix
  248. date_str = "2020-" + self.file_path[-prefix:-suffix]
  249. self.url = f"/david/{date_str.replace('-', '/')}/"
  250. self.date = datetime.strptime(date_str, "%Y-%m-%d").date()
  251. self.full_url = f"{DOMAIN}{self.url}"
  252. self.normalized_date = self.date.strftime(NORMALIZED_STRFTIME)
  253. self.escaped_title = escape(self.title)
  254. tag_template = Template(
  255. f'<a href="{DOMAIN}/david/2023/$tag_slug/">#$tag_name</a>'
  256. )
  257. tag_links = " ".join(
  258. tag_template.substitute(tag_slug=slugify(tag), tag_name=tag)
  259. for tag in self.tags
  260. )
  261. self.escaped_content = escape(
  262. self.content.replace('href="/', f'href="{DOMAIN}/')
  263. .replace('src="/', f'src="{DOMAIN}/')
  264. .replace('href="#', f'href="{self.full_url}#')
  265. + f"<nav><p>{tag_links}</p></nav>"
  266. + '<hr/><p><a href="mailto:david@larlet.fr">Réagir ?</a></p>'
  267. )
  268. # Extract first paragraph.
  269. self.extract = self.content.split("</p>", 1)[0] + "</p>"
  270. # Create the index for the search.
  271. self.search_data = {
  272. "title": self.title,
  273. "url": self.url,
  274. "date": date_str,
  275. "content": do_striptags(self.content)
  276. .replace("\u00a0(cache)", " ")
  277. .replace("'", " "),
  278. }
  279. def __eq__(self, other):
  280. return self.url == other.url
  281. def __lt__(self, other: "Page"):
  282. if not isinstance(other, Page):
  283. return NotImplemented
  284. return self.date < other.date
  285. @staticmethod
  286. def all(source: Path, only_published=True, with_h2_anchors=True):
  287. """Retrieve all pages sorted by desc."""
  288. page_list = []
  289. md = markdown_with_h2_anchors if with_h2_anchors else markdown_with_img_sizes
  290. for file_path in sorted(each_file_from(source, pattern="*.md")):
  291. result = md.read(file_path)
  292. result = widont(result, html=True)
  293. # Extract (and remove) the title from the generated page.
  294. title, content = result.split("</h1>", 1)
  295. h1_opening_size = len("<h1>")
  296. title = title[h1_opening_size:]
  297. tags = {}
  298. if "<nav><p>" in content:
  299. # Extract the tags from the generated page.
  300. content, tags_links = content.split("<nav><p>", 1)
  301. nav_closing_size = len("</p></nav>\n")
  302. tags_links = tags_links[:-nav_closing_size]
  303. try:
  304. tags = sorted(
  305. {
  306. tag.strip().split("#", 1)[1]
  307. for tag in tags_links.split("</a>")
  308. if tag.strip()
  309. },
  310. key=lambda tag: slugify(tag),
  311. )
  312. except IndexError:
  313. # It happens for old contents, parsed for the search index.
  314. pass
  315. page = Page(title, content, tags, file_path.name)
  316. pages_by_url[page.url] = page
  317. if not page.is_draft:
  318. all_tags.update(tags)
  319. for tag in tags:
  320. if page not in pages_by_tags[tag]:
  321. pages_by_tags[tag].append(page)
  322. if only_published and page.is_draft:
  323. continue
  324. page_list.append(page)
  325. return sorted(page_list, reverse=True)
  326. @property
  327. def is_draft(self):
  328. return (
  329. datetime(year=self.date.year, month=self.date.month, day=self.date.day)
  330. > PUBLICATION_BUFFER
  331. )
  332. @cli
  333. def pages():
  334. """Build article pages."""
  335. root_path = DAVID / "2023"
  336. for previous, page, next_ in neighborhood(
  337. reversed(Page.all(source=SOURCES_PATH, only_published=False)),
  338. first={
  339. "url": "/david/2022/",
  340. "title": "Publications 2022",
  341. "is_draft": False,
  342. },
  343. ):
  344. template = environment.get_template("article_2020.html")
  345. content = template.render(page=page, prev=previous, next=next_, slugify=slugify)
  346. target_path = Path(page.url[1:])
  347. target_path.mkdir(parents=True, exist_ok=True)
  348. open(target_path / "index.html", "w").write(content)
  349. if page.is_draft:
  350. print(f"Draft: {LOCAL_DOMAIN}{page.url} ({page.title})")
  351. def group_by_month_year(item):
  352. return item.date.strftime("%B %Y").title()
  353. template = environment.get_template("archives_2020.html")
  354. page_list = reversed(Page.all(source=SOURCES_PATH))
  355. tags = sorted((slugify(tag), tag, len(pages_by_tags[tag])) for tag in all_tags)
  356. content = template.render(
  357. page_list=groupby(page_list, key=group_by_month_year), tags=tags
  358. )
  359. open(root_path / "index.html", "w").write(content)
  360. @cli
  361. def tags():
  362. """Build tags pages."""
  363. # Parse all pages to collect tags.
  364. Page.all(source=SOURCES_PATH, only_published=True)
  365. for tag in all_tags:
  366. template = environment.get_template("tag_2021.html")
  367. content = template.render(
  368. page_list=sorted(pages_by_tags[tag], reverse=True),
  369. tag_name=tag,
  370. )
  371. target_path = DAVID / "2023" / slugify(tag)
  372. target_path.mkdir(parents=True, exist_ok=True)
  373. open(target_path / "index.html", "w").write(content)
  374. @cli
  375. def home():
  376. """Build the home page with last published items."""
  377. template = environment.get_template("profil.html")
  378. page_list = Page.all(source=SOURCES_PATH, only_published=True)
  379. tags = sorted((slugify(tag), tag, len(pages_by_tags[tag])) for tag in all_tags)
  380. content = template.render(page_list=page_list, tags=tags)
  381. open(DAVID / "index.html", "w").write(content)
  382. @cli
  383. def toot():
  384. """Pre-write the Mastodon message."""
  385. page_list = Page.all(source=SOURCES_PATH, only_published=True)
  386. last_published = page_list[0]
  387. print(f"✍️ QUOTE? — {last_published.title}, {last_published.full_url}")
  388. print()
  389. print("#blog #larletfr #rss")
  390. print(" ".join([f"#{tag}" for tag in last_published.tags]))
  391. @cli
  392. def search():
  393. """Build the static search page with custom index."""
  394. template = environment.get_template("recherche.html")
  395. page_list_2023 = Page.all(
  396. source=SOURCES_PATH, only_published=True, with_h2_anchors=False
  397. )
  398. page_list_2022 = Page.all(
  399. source=DAVID / "2022" / "_sources", only_published=True, with_h2_anchors=False
  400. )
  401. page_list_2021 = Page.all(
  402. source=DAVID / "2021" / "sources", only_published=True, with_h2_anchors=False
  403. )
  404. page_list_2020 = Page.all(
  405. source=DAVID / "2020", only_published=True, with_h2_anchors=False
  406. )
  407. blog_page_list_2019 = BlogPage.all(source=DAVID / "blog" / "2019")
  408. blog_page_list_2018 = BlogPage.all(source=DAVID / "blog" / "2018")
  409. blog_page_list_2017 = BlogPage.all(source=DAVID / "blog" / "2017")
  410. stream_page_list_2019 = StreamPage.all(source=DAVID / "stream" / "2019")
  411. stream_page_list_2018 = StreamPage.all(source=DAVID / "stream" / "2018")
  412. page_list = (
  413. page_list_2023
  414. + page_list_2022
  415. + page_list_2021
  416. + page_list_2020
  417. + blog_page_list_2019
  418. + blog_page_list_2018
  419. + blog_page_list_2017
  420. + stream_page_list_2019
  421. + stream_page_list_2018
  422. )
  423. search_index = json.dumps([page.search_data for page in page_list], indent=2)
  424. content = template.render(search_index=search_index)
  425. open(DAVID / "recherche" / "index.html", "w").write(content)
  426. @cli
  427. def feed():
  428. """Generate a feed from last published items."""
  429. template = environment.get_template("feed.xml")
  430. page_list = Page.all(source=SOURCES_PATH, with_h2_anchors=False)
  431. content = template.render(
  432. page_list=page_list[:NB_ITEMS_IN_FEED],
  433. current_dt=TODAY.strftime(NORMALIZED_STRFTIME),
  434. BASE_URL=f"{DOMAIN}/david/",
  435. )
  436. open(DAVID / "log" / "index.xml", "w").write(content)
  437. @wrap
  438. def perf_wrapper():
  439. start = perf_counter()
  440. yield
  441. elapsed = perf_counter() - start
  442. print(f"Done in {elapsed:.5f} seconds.")
  443. # Below are legacy blog contents, still useful for search indexation.
  444. @dataclass
  445. class BlogPage:
  446. title: str
  447. content: str
  448. file_path: str
  449. date_str: str
  450. def __post_init__(self):
  451. self.date = datetime.strptime(self.date_str, "%Y-%m-%d").date()
  452. self.url = f"/{self.file_path}/"
  453. # Create the index for the search.
  454. self.search_data = {
  455. "title": self.title,
  456. "url": self.url,
  457. "date": self.date_str,
  458. "content": do_striptags(self.content)
  459. .replace("\u00a0(cache)", " ")
  460. .replace("'", " ")
  461. .replace("<", "&lt;")
  462. .replace(">", "&gt;"),
  463. }
  464. def __eq__(self, other):
  465. return self.url == other.url
  466. def __lt__(self, other: "BlogPage"):
  467. if not isinstance(other, self.__class__):
  468. return NotImplemented
  469. return self.date < other.date
  470. @staticmethod
  471. def all(source: Path):
  472. """Retrieve all pages sorted by desc."""
  473. page_list = []
  474. for folder in each_folder_from(source):
  475. for path in each_file_from(folder, pattern="*.md"):
  476. metadata, content = path.read_text().split("\n\n", 1)
  477. if "lang:" in metadata:
  478. title, slug, date_, chapo, lang = metadata.split("\n")
  479. else:
  480. title, slug, date_, chapo = metadata.split("\n")
  481. title = title[len("title: ") :].strip()
  482. date_str = date_[len("date: ") :].strip()
  483. content = markdown_with_img_sizes(content)
  484. page = BlogPage(title, content, path.parent, date_str)
  485. page_list.append(page)
  486. return sorted(page_list, reverse=True)
  487. @dataclass
  488. class StreamPage:
  489. title: str
  490. content: str
  491. file_path: str
  492. date_str: str
  493. def __post_init__(self):
  494. self.date = datetime.strptime(self.date_str, "%Y/%m/%d").date()
  495. self.url = f"/{self.file_path}/"
  496. # Create the index for the search.
  497. self.search_data = {
  498. "title": self.title,
  499. "url": self.url,
  500. "date": self.date.isoformat(),
  501. "content": do_striptags(self.content)
  502. .replace("\u00a0(cache)", " ")
  503. .replace("'", " ")
  504. .replace("<", "&lt;")
  505. .replace(">", "&gt;"),
  506. }
  507. def __eq__(self, other):
  508. return self.url == other.url
  509. def __lt__(self, other: "StreamPage"):
  510. if not isinstance(other, self.__class__):
  511. return NotImplemented
  512. return self.date < other.date
  513. @staticmethod
  514. def all(source: Path):
  515. """Retrieve all pages sorted by desc."""
  516. page_list = []
  517. for folder in each_folder_from(source):
  518. for subfolder in each_folder_from(folder):
  519. for path in each_file_from(subfolder, pattern="*.md"):
  520. metadata, content = path.read_text().split("\n\n", 1)
  521. if "lang:" in metadata:
  522. title, lang = metadata.split("\n")
  523. else:
  524. title = metadata.strip()
  525. title = title[len("title: ") :].strip()
  526. date_str = str(path.parent)[-len("YYYY/MM/DD") :]
  527. content = markdown_with_img_sizes(content)
  528. page = StreamPage(title, content, path.parent, date_str)
  529. page_list.append(page)
  530. return sorted(page_list, reverse=True)
  531. if __name__ == "__main__":
  532. run()