diff --git a/.github/scripts/render-site-diff.sh b/.github/scripts/render-site-diff.sh new file mode 100644 index 00000000..11aa3887 --- /dev/null +++ b/.github/scripts/render-site-diff.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 3 ]]; then + echo "usage: $0 OLD_OUTPUT NEW_OUTPUT REPORT_DIR" >&2 + exit 2 +fi + +old_output=$(realpath "$1") +new_output=$(realpath "$2") +report=$(realpath -m "$3") +staging=$(mktemp -d) +trap 'rm -rf "$staging"' EXIT + +mkdir -p "$report" "$staging/old" "$staging/new" + +stage_output() { + local source=$1 destination=$2 + + # HTML is the review target. Feeds, gzip output and build_info.txt are + # intentionally absent, as their generated representation is not stable. + while IFS= read -r -d '' path; do + relative=${path#"$source"/} + mkdir -p "$destination/$(dirname "$relative")" + cp "$path" "$destination/$relative" + done < <(find "$source" -type f -name '*.html' -print0) + + # Binary files do not belong in a textual diff, but their checksums catch + # missing or changed uploads (the failure mode that prompted this check). + if [[ -d "$source/uploads" ]]; then + ( + cd "$source/uploads" + find . -type f -print0 | sort -z | xargs -0 sha256sum + ) > "$destination/uploads.sha256" + fi +} + +stage_output "$old_output" "$staging/old" +stage_output "$new_output" "$staging/new" + +set +e +diff -ruN -U 5 "$staging/old" "$staging/new" > "$report/site.diff" +diff_status=$? +set -e +if [[ $diff_status -gt 1 ]]; then + echo "diff failed with status $diff_status" >&2 + exit "$diff_status" +fi + +old_html=$(find "$staging/old" -type f -name '*.html' | wc -l) +new_html=$(find "$staging/new" -type f -name '*.html' | wc -l) +changed=$(diff -qr "$staging/old" "$staging/new" 2>/dev/null | wc -l || true) +cat > "$report/summary.md" < "$report/site-diff.html" <<'EOF' +No renderer differences +

No renderer differences

+

The staged HTML and upload checksum manifests are identical.

+EOF +fi diff --git a/.github/workflows/content-diff.yaml b/.github/workflows/content-diff.yaml new file mode 100644 index 00000000..638d40e6 --- /dev/null +++ b/.github/workflows/content-diff.yaml @@ -0,0 +1,62 @@ +name: Compare rendering after content changes + +on: + pull_request: + paths: + - 'content/**' + workflow_dispatch: + inputs: + base_ref: + description: Git ref to compare against + required: true + default: master + +permissions: + contents: read + +jobs: + content-diff: + runs-on: ubuntu-latest + steps: + - name: Check out base revision + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.base.sha || inputs.base_ref }} + path: base + + - name: Check out branch revision + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + path: branch + + - name: Build unminified base output + run: docker build --target builder --build-arg BUILD_MODE=debug -t website-base base + + - name: Build unminified branch output + run: docker build --target builder --build-arg BUILD_MODE=debug -t website-branch branch + + - name: Export generated sites + run: | + mkdir -p output/base output/branch + docker create --name website-base-export website-base + docker create --name website-branch-export website-branch + docker cp website-base-export:/site/output/. output/base/ + docker cp website-branch-export:/site/output/. output/branch/ + + - name: Install HTML diff renderer + run: npm install --global diff2html-cli@5.1.2 + + - name: Generate browsable comparison + run: bash branch/.github/scripts/render-site-diff.sh output/base output/branch report + + - name: Add comparison summary + run: cat report/summary.md >> "$GITHUB_STEP_SUMMARY" + + - name: Upload HTML and raw diff + uses: actions/upload-artifact@v7 + with: + name: base-branch-content-diff + path: report/ + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/renderer-diff.yaml b/.github/workflows/renderer-diff.yaml new file mode 100644 index 00000000..8db5d159 --- /dev/null +++ b/.github/workflows/renderer-diff.yaml @@ -0,0 +1,63 @@ +name: Compare rendering after code changes + +on: + pull_request: + paths-ignore: + - 'content/**' + workflow_dispatch: + inputs: + base_ref: + description: Git ref to compare against + required: true + default: master + +permissions: + contents: read + +jobs: + renderer-diff: + runs-on: ubuntu-latest + steps: + - name: Check out base revision + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.base.sha || inputs.base_ref }} + path: base + + - name: Check out branch revision + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + path: branch + + - name: Build unminified base output + run: docker build --target builder --build-arg BUILD_MODE=debug -t website-base base + + - name: Build unminified branch output + run: docker build --target builder --build-arg BUILD_MODE=debug -t website-branch branch + + - name: Export generated sites + run: | + mkdir -p output/base output/branch + docker create --name website-base-export website-base + docker create --name website-branch-export website-branch + docker cp website-base-export:/site/output/. output/base/ + docker cp website-branch-export:/site/output/. output/branch/ + + - name: Install HTML diff renderer + run: npm install --global diff2html-cli@5.1.2 + + - name: Generate browsable comparison + run: bash branch/.github/scripts/render-site-diff.sh output/base output/branch report + + - name: Add comparison summary + run: cat report/summary.md >> "$GITHUB_STEP_SUMMARY" + + - name: Upload HTML and raw diff + uses: actions/upload-artifact@v7 + with: + name: base-branch-renderer-diff + path: report/ + if-no-files-found: error + retention-days: 30 + diff --git a/Dockerfile b/Dockerfile index 374f0ff6..c7a39587 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,26 +1,13 @@ -FROM python:3.5-slim AS builder +FROM python:3.13-slim AS builder WORKDIR /site ARG BUILD_MODE=prod ENV BUILD_MODE=${BUILD_MODE} -RUN printf '%s\n' \ - 'deb https://archive.debian.org/debian buster main' \ - 'deb https://archive.debian.org/debian-security buster/updates main' \ - > /etc/apt/sources.list \ - && printf 'Acquire::Check-Valid-Until "false";\n' > /etc/apt/apt.conf.d/99archive \ - && apt-get update \ - && apt-get install -y --no-install-recommends \ - build-essential \ - libxml2-dev \ - libxslt1-dev \ - zlib1g-dev \ - openssh-client \ - && rm -rf /var/lib/apt/lists/* COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt COPY . . -RUN fab build:${BUILD_MODE} +RUN pelican -s pelicanconf-${BUILD_MODE}.py FROM alpine:latest AS final diff --git a/deploy.sh b/deploy.sh deleted file mode 100644 index cf2a37cf..00000000 --- a/deploy.sh +++ /dev/null @@ -1,11 +0,0 @@ -set -x - -[ "$TRAVIS_BRANCH" = "master" ] || exit 1 -[ "$TRAVIS_PULL_REQUEST" = "false" ] || exit 1 -[ "$TRAVIS_PYTHON_VERSION" = "3.5" ] || exit 1 - -eval "$(ssh-agent -s)" - -chmod 600 $TRAVIS_BUILD_DIR/deploy_key -ssh-add $TRAVIS_BUILD_DIR/deploy_key -fab publish:ci diff --git a/fabfile.py b/fabfile.py deleted file mode 100644 index 647d9c35..00000000 --- a/fabfile.py +++ /dev/null @@ -1,143 +0,0 @@ -# -*- coding: utf-8 -*- - -from fabric.api import env, local, puts -import fabric.contrib.project as project -import os - -import sys -PY3 = sys.version_info > (3,) - -# Local path configuration (can be absolute or relative to fabfile) -env.deploy_path = 'output' -DEPLOY_PATH = env.deploy_path - -# Remote server configuration -env.user = 'www-data' -env.hosts = ['chtoes.li'] -path = { - 'dev': '/var/www/dev.chtoes.li/public/', - 'prod': '/var/www/chtoes.li/public/', - 'ci': '/var/www/ci.chtoes.li/public/' -} - -TEMPLATE = 'Title:\n\ -Date: {{date}}\n\ -Slug: {{slug}}\n\ -Category: {{ category }}\n\ -Source: http://what-if.xkcd.com/{{num}}/\n\ -SourceNum: {{num}}\n\ -SourceTitle: {{title}}\n\ -Formulas: False\n\ -Description: \n\ -Image: https://chtoes.li/uploads/{{num}}-{{slug}}/front.png\n\ -\n\ -\n' - - -def new(num, title=None, category='What If?', overwrite='no'): - import datetime - - if title is None: - if PY3: - from urllib.request import urlopen - else: - from urllib2 import urlopen - from bs4 import BeautifulSoup - html = urlopen('http://what-if.xkcd.com/{}/'.format(num)) - title = BeautifulSoup(html, 'lxml').title.string - - slug = slugify(title) - - CATEGORIES = { - 'What If?': 'what-if', - 'Новости проекта': 'news', - 'Прочее': 'other', - } - - category_slug = CATEGORIES[category] - now = datetime.datetime.now() - post_date = now.strftime('%Y-%m-%d') - - params = dict( - date=post_date, - title=title, - slug=slug, - num=num, - category=category - ) - - out_file = 'content/{}/{:03d}-{}.md'.format(category_slug, int(num), slug) - local("mkdir -p '{}' || true".format(os.path.dirname(out_file))) - if not os.path.exists(out_file) or overwrite.lower() == 'yes': - render(TEMPLATE, out_file, **params) - else: - print("{} already exists. Pass 'overwrite=yes' to destroy it." - .format(out_file)) - - -def slugify(text): - import re - normalized = ''.join([c.lower() if c.isalnum() else '-' - for c in text]) - no_repetitions = re.sub(r'--+', '-', normalized) - clean_start = re.sub(r'^-+', '', no_repetitions) - clean_end = re.sub(r'-+$', '', clean_start) - return clean_end - - -def render(template, destination, **kwargs): - from jinja2 import Template - template = Template(TEMPLATE) - text = template.render(**kwargs) - with open(destination, 'w') as output: - puts('Rendering to {}'.format(destination)) - if PY3: - output.write(text) - else: - output.write(text.encode('utf-8')) - - -def clean(): - if os.path.isdir(DEPLOY_PATH): - local('rm -rf {deploy_path}'.format(**env)) - local('mkdir {deploy_path}'.format(**env)) - - -def build(environment): - local('pelican --debug -s pelicanconf-{}.py'.format(environment)) - - -def rebuild(environment): - clean() - build(environment) - - -def regenerate(environment): - local('pelican -r -s pelicanconf-{}.py'.format(environment)) - - -def serve(): - # Determine major version of python as the one executes this script now. - if PY3: - local('cd {deploy_path} && python3 -m http.server'.format(**env)) - else: - local('cd {deploy_path} && python2 -m SimpleHTTPServer'.format(**env)) - - -def reserve(environment): - build(environment) - serve() - - -def preview(environment): - local('pelican -s pelicanconf-{}.py'.format(environment)) - - -def publish(environment): - local('pelican -s pelicanconf-{}.py'.format(environment)) - project.rsync_project( - remote_dir=path[environment], - exclude='.DS_Store', - local_dir=DEPLOY_PATH.rstrip('/') + '/', - delete=True - ) diff --git a/localconf.py b/localconf.py index 6799444c..55e7b0b4 100644 --- a/localconf.py +++ b/localconf.py @@ -13,19 +13,13 @@ PLUGIN_PATHS = ['plugins'] PLUGINS = ['neighbors', 'sitemap', 'assets', 'minify', - 'block', 'gzip_cache', 'preserve_old_feed_items', + 'block', 'gzip_cache', 'feed_alter_settings'] THEME = 'themes/whatif' PATH = 'content' OUTPUT_PATH = 'output' DELETE_OUTPUT_DIRECTORY = True -MINIFY = { - 'remove_comments': True, - 'remove_empty_space': True, - 'remove_optional_attribute_quotes': True, -} - # RSS feeds FEED_ALL_RSS = 'feed/index.xml' CATEGORY_FEED_RSS = None @@ -64,11 +58,10 @@ 'rewrite.html': 'rewrite.map' } -SLUG_SUBSTITUTIONS = [ - ('what if?', 'what-if'), - ('novosti proekta', 'news'), - ('prochee', 'other'), - # ('',''), +SLUG_REGEX_SUBSTITUTIONS = [ + (r'^what if\?$', 'what-if'), + (r'^novosti proekta$', 'news'), + (r'^prochee$', 'other'), ] DEFAULT_PAGINATION = False @@ -111,7 +104,10 @@ MARKDOWN = { 'extensions': [ 'markdown.extensions.meta', - 'markdown.extensions.extra', + 'markdown.extensions.fenced_code', + 'markdown.extensions.attr_list', + 'markdown.extensions.def_list', + 'markdown.extensions.tables', 'markdown.extensions.abbr', 'markdown.extensions.footnotes', md_extensions.QuestionExtension(), @@ -142,4 +138,5 @@ def FEED_ALTER_SETTINGS(settings): elif isinstance(ext, md_extensions.FootnoteExtExtension): markdown_opts['extensions'].remove(ext) # it's necessary to create instance to get unique_prefix being persistent - markdown_opts['extensions'].append(FootnoteExtension(UNIQUE_IDS=True)) + markdown_opts['extensions'].append( + FootnoteExtension(UNIQUE_IDS=True, SEPARATOR='-')) diff --git a/md_extensions/article_links.py b/md_extensions/article_links.py index c67b8d04..1883dec9 100644 --- a/md_extensions/article_links.py +++ b/md_extensions/article_links.py @@ -5,7 +5,7 @@ # Lean on the 'meta' extension. class ArticleLinksTreeprocessor(Treeprocessor): def run(self, root): - meta = self.markdown.Meta + meta = self.md.Meta if 'category' not in meta or meta['category'] != ['What If?']: return None links = root.findall('.//a[@href]') @@ -15,5 +15,6 @@ def run(self, root): class ArticleLinksExtension(Extension): - def extendMarkdown(self, md, md_globals): - md.treeprocessors['article_links'] = ArticleLinksTreeprocessor(md) + def extendMarkdown(self, md): + md.treeprocessors.register( + ArticleLinksTreeprocessor(md), 'article_links', 0) diff --git a/md_extensions/escape_ext.py b/md_extensions/escape_ext.py index d5da727a..cfe44d33 100644 --- a/md_extensions/escape_ext.py +++ b/md_extensions/escape_ext.py @@ -1,5 +1,10 @@ +import re + from markdown.extensions import Extension -from .md_utils import TrivialPreprocessor, TrivialTextPattern +from markdown.postprocessors import Postprocessor +from markdown.preprocessors import Preprocessor +from markdown.treeprocessors import Treeprocessor +from .md_utils import TrivialPreprocessor, TrivialTextPattern, html_entity # Markdown extension @@ -8,7 +13,7 @@ class AbbrTitlePreprocessor(TrivialPreprocessor): def run(self, lines): - for k, v in self.markdown.inlinePatterns.items(): + for k, v in self.md.inlinePatterns._data.items(): if not k.startswith('abbr-'): continue v.title = self.compiled_re.sub(self.handleMatch, v.title) @@ -18,23 +23,72 @@ def run(self, lines): class ReferenceTitlePreprocessor(TrivialPreprocessor): def run(self, lines): new_ref = {} - for k, v in self.markdown.references.items(): + for k, v in self.md.references.items(): url, title = v if title: new_title = self.compiled_re.sub(self.handleMatch, title) new_ref[k] = (url, new_title) else: new_ref[k] = v - self.markdown.references = new_ref + self.md.references = new_ref return lines +class QuotedTitlePreprocessor(Preprocessor): + pattern = re.compile(r'''(\]\([^\n]*?")'([^\n]*?)'("\))''') + + def run(self, lines): + result = [] + for line in lines: + line = self.pattern.sub(r'\1'\2'\3', line) + line = line.replace('] (/', '](/') + line = line.replace(' " :', ' " :') + if line.startswith('[^') and line.endswith(' '): + # Protect the existing trailing space; do not add a second + # one. Markdown 3 otherwise trims it before parsing the note. + line = line[:-1] + ' ' + if result and result[-1].endswith(']') and line.startswith('(/'): + result[-1] += line + continue + result.append(line) + return result + + +class ReferenceTitleTreeprocessor(Treeprocessor): + def run(self, root): + for elem in root.iter(): + for name, value in elem.attrib.items(): + # Keep the legacy entity spelling, not merely the equivalent + # Unicode character. Besides making old/new output stable, + #   makes the typographic intent visible in HTML diffs. + value = re.sub( + r'(?<=\d)\\ (?=\d)', html_entity(' '), value) + value = value.replace(r'\ ', html_entity(' ')) + value = value.replace(r"\'", html_entity('ʼ')) + elem.set(name, value) + + +class LegacyEntityPostprocessor(Postprocessor): + """Undo compatibility entities needed only while parsing Markdown. + + Modern Markdown interprets apostrophe-delimited text inside an image title + differently from the old release. QuotedTitlePreprocessor protects those + apostrophes until parsing is complete; the old renderer emitted them as + literal characters, so restore that representation in the final fragment. + """ + + def run(self, text): + return text.replace(''', "'").replace(' ', ' ') + + class EscapeExtExtension(Extension): - def extendMarkdown(self, md, md_globals): + def extendMarkdown(self, md): + md.preprocessors.register( + QuotedTitlePreprocessor(md), 'quoted_titles', 28) processors = [ - (md.preprocessors, AbbrTitlePreprocessor, '>abbr'), - (md.preprocessors, ReferenceTitlePreprocessor, '>reference'), - (md.inlinePatterns, TrivialTextPattern, 'smth' used patterns_local = patterns - if order.startswith('>'): + if cls is not TrivialTextPattern: patterns_local = list(reversed(patterns)) for prefix, re, repl in patterns_local: name = prefix + cls_name inst = cls(md, re, repl) - parsers.add(name, inst, order) + parsers.register(inst, name, order) + + md.treeprocessors.register( + ReferenceTitleTreeprocessor(md), 'reference_titles', 1) + md.postprocessors.register( + LegacyEntityPostprocessor(md), 'legacy_entities', 0) diff --git a/md_extensions/etree_utils.py b/md_extensions/etree_utils.py index eedc6a8b..27aa10e0 100644 --- a/md_extensions/etree_utils.py +++ b/md_extensions/etree_utils.py @@ -1,4 +1,4 @@ -from markdown.util import etree +from xml.etree import ElementTree as etree from .six_mini import string_types @@ -58,7 +58,7 @@ def remove_suffix(elem, suffix): else: text = elem[-1].tail - if text.endswith(suffix): + if text and text.endswith(suffix): text = text[:-len(suffix)] if len(elem) == 0: elem.text = text diff --git a/md_extensions/figures.py b/md_extensions/figures.py index 5a5bb40e..05849053 100644 --- a/md_extensions/figures.py +++ b/md_extensions/figures.py @@ -1,6 +1,6 @@ from markdown.extensions import Extension from markdown.treeprocessors import Treeprocessor -from markdown.util import etree +from xml.etree import ElementTree as etree from .etree_utils import replace_element @@ -54,5 +54,5 @@ def run(self, root): class FiguresExtension(Extension): - def extendMarkdown(self, md, md_globals): - md.treeprocessors['figures'] = FiguresTreeprocessor() + def extendMarkdown(self, md): + md.treeprocessors.register(FiguresTreeprocessor(md), 'figures', 0) diff --git a/md_extensions/footnote_ext.py b/md_extensions/footnote_ext.py index a2ddff71..88a4ff8b 100644 --- a/md_extensions/footnote_ext.py +++ b/md_extensions/footnote_ext.py @@ -4,8 +4,9 @@ from __future__ import unicode_literals import re from markdown.extensions import Extension +from markdown.postprocessors import Postprocessor from markdown.treeprocessors import Treeprocessor -from markdown.util import etree +from xml.etree import ElementTree as etree from markdown.extensions.footnotes import NBSP_PLACEHOLDER from .etree_utils import replace_element_inplace, create_etree, \ remove_suffix, add_suffix @@ -129,6 +130,78 @@ def get_sup_parent(root, id_): root.remove(footnotes_div) +class ImageMapFootnotePostprocessor(Postprocessor): + """Restore inline footnotes attached to raw-HTML image maps. + + ``FootnoteExtTreeprocessor`` normally decides whether to emit the closing + ``ellipsis`` marker by looking for elements after the original ``sup`` in + its parent. That works for Markdown elements, but not for a raw-HTML + ``map``. Python-Markdown removes raw HTML into ``htmlStash`` before tree + processors run and restores it later in its ``raw_html`` postprocessor. + Consequently, while ``FootnoteExtTreeprocessor`` is running, it sees the + footnote as the final child of a paragraph: the following map is not an + ElementTree sibling, ``has_text_after`` is false, and both the closing + arrow and the empty punctuation anchor are omitted. When raw HTML is + restored, Markdown 3 also places the map after ``

``. Markdown 2 kept + that map inside the paragraph, which is the DOM expected by the site's + inline-footnote CSS and JavaScript. + + This cannot be corrected reliably in ``FootnoteExtTreeprocessor`` without + depending on Python-Markdown's private stash-placeholder format and moving + opaque placeholder text between ElementTree ``text``/``tail`` fields. + Such a workaround would be more fragile than waiting until raw HTML has + been restored. This postprocessor therefore runs after ``raw_html`` (its + priority is lower), matches an image and map through their standard + ``usemap``/``name`` relationship, and restores only the legacy relationship + needed by the footnote widget: + + * the map is moved back inside the image paragraph; + * the empty ``punctum`` span is restored; and + * the closing ``ellipsis`` arrow is restored. + + Ordinary footnotes and raw-HTML blocks without a matching image map are + returned unchanged. + """ + + def run(self, source): + search_from = 0 + while True: + map_start = source.find('', usemap, map_start) + map_end = source.find('', map_start) + if usemap < 0 or paragraph_end < 0 or map_end < 0: + search_from = name_end + continue + + segment = source[usemap:paragraph_end] + segment = segment.replace( + '', + '' + '', + 1, + ) + final_span = segment.rfind('') + segment = (segment[:final_span] + + '↳' + + segment[final_span:]) + source = ( + source[:usemap] + segment + + source[paragraph_end + len('

'): + map_end + len('')] + + '

' + source[map_end + len(''):] + ) + search_from = map_end + len('') + + class FootnoteExtExtension(Extension): - def extendMarkdown(self, md, md_globals): - md.treeprocessors['footnote_ext'] = FootnoteExtTreeprocessor() + def extendMarkdown(self, md): + md.treeprocessors.register( + FootnoteExtTreeprocessor(md), 'footnote_ext', 0) + md.postprocessors.register( + ImageMapFootnotePostprocessor(md), 'image_map_footnotes', 0) diff --git a/md_extensions/mathjax.py b/md_extensions/mathjax.py index 35b3d541..aa7c8b6a 100644 --- a/md_extensions/mathjax.py +++ b/md_extensions/mathjax.py @@ -12,23 +12,23 @@ def available(self, meta): return # ESCAPED_CHARS is class variable, so we copy the list and assing it to # the instance to prevent sharing between Markdown instances - chars = list(self.markdown.ESCAPED_CHARS) + chars = list(self.md.ESCAPED_CHARS) chars.append('$') - self.markdown.ESCAPED_CHARS = chars + self.md.ESCAPED_CHARS = chars class MathJaxExtension(Extension): - def extendMarkdown(self, md, md_globals): + def extendMarkdown(self, md): def repl(match): return ''.join(match.group('edge', 'body', 'edge')) # '\$' -> '$' when formulas disabled; # that needed for consistency: when formulas enabled, MathJax with # 'processEscapes: true' will do that - md.preprocessors.add( - 'maybe_escape_dollar', MaybeEscapeDollar(md), '>meta') + md.preprocessors.register( + MaybeEscapeDollar(md), 'maybe_escape_dollar', 26) # exclude formulas from processing as a markdown MATHJAX_RE = r'(?\$\$?)(?P.+?)(?P=edge)' mathJaxPattern = TrivialTextPattern(md, MATHJAX_RE, repl) - md.inlinePatterns.add('mathjax', mathJaxPattern, '