#!/usr/bin/env python3 import json import locale import pickle import time from collections import defaultdict from dataclasses import dataclass from datetime import datetime, timedelta from html import escape from itertools import chain, groupby from operator import attrgetter from pathlib import Path from string import Template from textwrap import dedent from time import perf_counter import feedparser import mistune from jinja2 import Environment as Env from jinja2 import FileSystemLoader from jinja2.filters import do_striptags from minicli import cli, run, wrap from mistune.plugins.formatting import mark, strikethrough from mistune.util import safe_entity from PIL import Image from slugify import slugify from typography import typographie from widont import widont # Useful for dates rendering within Jinja2. locale.setlocale(locale.LC_ALL, "fr_FR.UTF-8") VERSION = "2023.09.10" HERE = Path(".") DAVID = HERE / "david" STATIC = HERE / ".." / "larlet-fr-static" DOMAIN = "https://larlet.fr" LOCAL_DOMAIN = "http://larlet.test:3579" # Hardcoding publication at 12 in Paris timezone. NORMALIZED_STRFTIME = "%Y-%m-%dT12:00:00+01:00" TODAY = datetime.today() + timedelta(hours=6) PUBLICATION_BUFFER = TODAY - timedelta(days=0) NB_ITEMS_IN_FEED = 30 SOURCES_PATH = DAVID / "2023" / "_sources" all_tags = set() pages_by_tags = defaultdict(list) pages_by_url = {} class TagsRenderer(mistune.HTMLRenderer): """Make the asumption each line starting with a `#` is a tag.""" def paragraph(self, text): if text.startswith("#"): tags = " ".join( f'#{tag.strip()}' for tag in text.split("#") if tag.strip() ) return f"\n" return super().paragraph(text) class FrenchTypographyRenderer(mistune.HTMLRenderer): """Apply French typographic rules to text.""" def text(self, text): return typographie(super().text(text), html=True) def block_html(self, html): return typographie(super().block_html(html), html=True) class InternalLinkTitleRenderer(mistune.HTMLRenderer): """Automatically generate the title for internal links.""" def link(self, text, url, title=None): s = '" + text + "" class CustomAndBlockquoteLanguageRenderer( FrenchTypographyRenderer, InternalLinkTitleRenderer, TagsRenderer ): """Sets the English language attribute for blockquotes with `[en]` prefix.""" def _get_language(self, text): if text.startswith("

[en] "): return "en", text.replace("

[en] ", "

") else: return None, text def block_quote(self, text): language, text = self._get_language(text) if language: return f'\n

\n{text}
\n' else: return f"\n
\n{text}
\n" class ImgsWithSizesRenderer(CustomAndBlockquoteLanguageRenderer): """Renders images as
s and add sizes.""" def paragraph(self, text): # In case of a figure, we do not want the (non-standard) paragraph. if text.strip().startswith("
"): return text return super().paragraph(text) def _generate_size(self, src, width, height): src_size = src.replace(".jpg", f"_{width}x{height}.jpg") full_path = STATIC / Path(src[1:]) full_path_size = STATIC / Path(src_size[1:]) if full_path_size.exists() or "/2023/" not in src: return src_size image = Image.open(full_path) image.thumbnail((width, height), resample=Image.LANCZOS) image.save(full_path_size, icc_profile=image.info.get("icc_profile")) return src_size def _generate_webp(self, src): src_webp = src.replace(".jpg", ".webp") full_path = STATIC / Path(src[1:]) full_path_webp = STATIC / Path(src_webp[1:]) if full_path_webp.exists() or "/2023/" not in src: return src_webp image = Image.open(full_path) image.save( full_path_webp, format="webp", icc_profile=image.info.get("icc_profile") ) # command = [ # "cwebp", # "-q", # "80", # full_path, # "-o", # full_path_webp, # "-metadata", # "icc", # ] # subprocess.check_output(command, stderr=subprocess.STDOUT) return src_webp def image(self, alt, url, title=None): SIZES = [(660, 440), (990, 660), (1320, 880)] full_path = STATIC / Path(url[1:]) image = Image.open(full_path) width, height = image.size jpg_srcs = [(url, width, height)] # src_webp = self._generate_webp(src) # webp_srcs = [(src_webp, width, height)] for size_width, size_height in SIZES: src_size = self._generate_size(url, size_width, size_height) jpg_srcs.append((src_size, size_width, size_height)) # src_size_webp = self._generate_webp(src_size) # webp_srcs.append((src_size_webp, size_width, size_height)) jpg_srcsets = ", ".join( f"{jpg_src} {jpg_width}w" for jpg_src, jpg_width, jpg_height in jpg_srcs ) # webp_srcsets = ", ".join( # f"{webp_src} {webp_width}w" # for webp_src, webp_width, webp_height in webp_srcs # ) return dedent( f"""\
{alt}
{title}
""" ) class H2AnchorsRenderer(ImgsWithSizesRenderer): """Custom renderer for H2 titles with anchors.""" def heading(self, text, level): if level == 2: slug = slugify(text) return ( f'

' f"{text} " f'#' f"

" ) else: return super().heading(text, level) # We want a custom renderer to create a hash/link for each H2 headings. markdown_with_h2_anchors = mistune.Markdown( renderer=H2AnchorsRenderer(escape=False), plugins=[mark, strikethrough], ) # The second markdown is pertinent to generate articles for the feed, # we do not need anchors in that case. markdown_with_img_sizes = mistune.Markdown( renderer=ImgsWithSizesRenderer(escape=False), plugins=[mark, strikethrough], ) # This is the jinja2 configuration to locate templates. environment = Env(loader=FileSystemLoader(str(DAVID / "templates"))) def format_struct_time(value, format="%d-%m-%Y"): return time.strftime(format, value) environment.filters["format_struct_time"] = format_struct_time def neighborhood(iterable, first=None, last=None): """ Yield the (previous, current, next) items given an iterable. You can specify a `first` and/or `last` item for bounds. """ iterator = iter(iterable) previous = first current = next(iterator) # Throws StopIteration if empty. for next_ in iterator: yield (previous, current, next_) previous = current current = next_ yield (previous, current, last) def each_file_from(source_dir, pattern="*", exclude=None): """Walk across the `source_dir` and return the `pattern` file paths.""" for path in _each_path_from(source_dir, pattern=pattern, exclude=exclude): if path.is_file(): yield path def each_folder_from(source_dir, exclude=None): """Walk across the `source_dir` and return the folder paths.""" for path in _each_path_from(source_dir, exclude=exclude): if path.is_dir(): yield path def _each_path_from(source_dir, pattern="*", exclude=None): for path in sorted(Path(source_dir).glob(pattern)): if exclude is not None and path.name in exclude: continue yield path @dataclass class Page: title: str content: str tags: list file_path: str lang: str = "fr" def __post_init__(self): try: date_str, _ = self.file_path.split(" - ", 1) except ValueError: # Fallback for 2020 contents (search index) suffix = len(".md") prefix = len("YYYY/MM-DD") + suffix date_str = "2020-" + self.file_path[-prefix:-suffix] self.url = f"/david/{date_str.replace('-', '/')}/" self.date = datetime.strptime(date_str, "%Y-%m-%d").date() self.full_url = f"{DOMAIN}{self.url}" self.normalized_date = self.date.strftime(NORMALIZED_STRFTIME) self.escaped_title = escape(self.title) tag_template = Template( f'#$tag_name' ) tag_links = " ".join( tag_template.substitute(tag_slug=slugify(tag), tag_name=tag) for tag in self.tags ) self.escaped_content = escape( self.content.replace('href="/', f'href="{DOMAIN}/') .replace('src="/', f'src="{DOMAIN}/') .replace('href="#', f'href="{self.full_url}#') + f"" + '

Réagir ?

' ) # Extract first paragraph. self.extract = self.content.split("

", 1)[0] + "

" # Create the index for the search. self.search_data = { "title": self.title, "url": self.url, "date": date_str, "content": do_striptags(self.content) .replace("\u00a0(cache)", " ") .replace("'", " "), } def __eq__(self, other): return self.url == other.url def __lt__(self, other: "Page"): if not isinstance(other, Page): return NotImplemented return self.date < other.date @staticmethod def all(source: Path, only_published=True, with_h2_anchors=True): """Retrieve all pages sorted by desc.""" page_list = [] md = markdown_with_h2_anchors if with_h2_anchors else markdown_with_img_sizes for file_path in sorted(each_file_from(source, pattern="*.md")): result, state = md.read(file_path) result = widont(result, html=True) # Extract (and remove) the title from the generated page. title, content = result.split("", 1) h1_opening_size = len("

") title = title[h1_opening_size:] tags = {} if "