From 57ef5530d9d8473cb940d4dee5bde31d1c96b597 Mon Sep 17 00:00:00 2001 From: Timid Robot Zehta Date: Fri, 23 Oct 2026 12:44:51 +0200 Subject: [PATCH 1/3] remove legacy migration configuration, documentation, and script --- migrate/Dockerfile.bullseye | 69 --- migrate/README.md | 199 -------- migrate/cache-docker/.gitignore | 4 - migrate/cache-legacy/.gitignore | 4 - migrate/docker-compose.yml | 83 ---- migrate/migrate_cc_mediawiki.sh | 791 -------------------------------- 6 files changed, 1150 deletions(-) delete mode 100644 migrate/Dockerfile.bullseye delete mode 100644 migrate/README.md delete mode 100644 migrate/cache-docker/.gitignore delete mode 100644 migrate/cache-legacy/.gitignore delete mode 100644 migrate/docker-compose.yml delete mode 100755 migrate/migrate_cc_mediawiki.sh diff --git a/migrate/Dockerfile.bullseye b/migrate/Dockerfile.bullseye deleted file mode 100644 index 8c955bc..0000000 --- a/migrate/Dockerfile.bullseye +++ /dev/null @@ -1,69 +0,0 @@ -# https://docs.docker.com/engine/reference/builder/ - -# https://hub.docker.com/_/debian -FROM debian:bullseye-slim -# NOTE: Occurrences of "NOTE: PHP version", below, where a specific PHP version -# is required (based on version supported by installed Debian version) - -# https://docs.docker.com/build/building/best-practices/#apt-get -# - Resynchronize the package index, update packages, install packages, -# clean-up, and update CA certificates -# - git is included because MediaWiki says: "Git version control software not -# found. [...] Note Special:Version will not display commit hashes." -RUN apt-get update --no-allow-insecure-repositories \ - && apt-get dist-upgrade --no-install-recommends --no-install-suggests \ - --yes \ - && apt-get install --no-install-recommends --no-install-suggests --yes \ - apache2 \ - apache2-utils \ - ca-certificates \ - curl \ - git \ - imagemagick \ - less \ - libapache2-mod-php \ - mariadb-client \ - mediawiki \ - php \ - php-apcu \ - php-cli \ - php-common \ - php-curl \ - php-gd \ - php-imagick \ - php-intl \ - php-mbstring \ - php-mysql \ - php-xml \ - php-zip \ - rsync \ - unzip \ - vim \ - && apt-get clean \ - && rm --recursive --force /var/lib/apt/lists/* \ - && update-ca-certificates - -# Add Apache2 virtualhost configuration -COPY config/web-sites-available /etc/apache2/sites-available/ - -# Add housekeeping + Apache2 service startup script -COPY config/startupservice.sh /usr/local/sbin/ -RUN chmod +x /usr/local/sbin/startupservice.sh -CMD ["/usr/local/sbin/startupservice.sh"] - -# Enable Apache modules - NOTE: PHP version -RUN a2enmod headers \ - && a2enmod php7.4 \ - && a2enmod rewrite - -# Add MediaWiki configuration script -COPY config/configure_mediawiki.sh /usr/local/sbin/ -RUN chmod +x /usr/local/sbin/configure_mediawiki.sh - -# Configure PHP - NOTE: PHP version -COPY config/90-local.ini /etc/php/7.4/apache2/conf.d/ - -# Expose ports for Apache -EXPOSE 80 - -WORKDIR /var/lib/mediawiki diff --git a/migrate/README.md b/migrate/README.md deleted file mode 100644 index 09ffb2d..0000000 --- a/migrate/README.md +++ /dev/null @@ -1,199 +0,0 @@ -# Migration - - -## Overview - -The migration process moves MediaWiki hosting from the Bytemark legacy virtual -machine, to local Docker containers to perform the upgrades, and then up to the -new production Railway project. - -The following local Docker containers are used: -1. `db`: MySQL (latest) - - MySQL was selected instead of MariaDB only because production - infrastructure provider runs MySQL - - For the MariaDB version, browse the repository at [`8809929`][mariadb-sha] -2. `web-bullseye`: Debian 11 (Bullseye) running MediaWiki 1.35.13 - - [`Dockerfile.bullseye`](`Dockerfile.bullseye`) -3. `web`: Debian 13 (Trixie) container running MediaWiki 1.43.8 - - [`Dockerfile`](../Dockerfile) - - The same `Dockerfile` is also used in production - -The following script is used: -- [`migrate_cc_mediawiki.sh`](migrate_cc_mediawiki.sh) - -[mariadb-sha]: https://github.com/creativecommons/public-wiki/tree/880992985fabac015d8e8c62564420e7bdd25cec - - -### Process - -1. Ensure you are in this directory (`migrate/`) -2. Create and start Docker containers - ```shell - docker compose up - ``` -3. Export MediaWiki data (database SQL and images files) from Bytemark legacy - virtual machine running MediaWiki 1.30.0 - ```shell - ./migrate_cc_mediawiki.sh pull - ``` - 1. Store data in `cache-legacy/` - 1. Copy `images/` (uploaded files) from legacy server - 2. Dump database to a SQL file - 2. Approximate duration: 3 minutes -4. Import MediaWiki data to local Docker containers and upgrade MediaWiki to - 1.43.8 - ```shell - ./migrate_cc_mediawiki.sh import - ``` - 1. Prepare SQL file for import - 1. Update `ENGINE` to `InnoDB` for all tables - 2. Update `CHARSET` to `binary` for all tables - 3. Update `CHARSET` to `utf8mb4` for `searchindex` table - 4. Update datetime default to a valid (non-zero) value - 2. Import and upgrade MediaWiki on Docker `web-bullseye` - 1. Copy `images/` (uploaded files) to Docker volume - 2. Import database from the SQL file - 3. Run MediaWiki update to version 1.35.13 - 4. Clean-up MediaWiki users with no ID - 3. Upgrade MediaWiki on Docker `web` container - 1. Run MediaWiki update to version 1.43.8 - 2. Account maintenance - - Remove unused accounts - - Delete local passwords - - Remove all users from legacy groups - - Remove all users from privileged groups - - Add appropriate and verified users to sysop group - 3. Image maintenance - - Refresh image metadata - - Refresh file headers - - Clean-up upload stash - 4. Clean-up page titles - 5. Rebuild maintenance - - Rebuild text index - - Rebuild recent changes - - Refresh links - 6. Database maintenance - - Check tables - - Optimize tables - - Analyze tables - 4. Approximate duration: 8 minutes -5. Export MediaWiki data (database SQL and images file) from local Docker - containers - ```shell - ./migrate_cc_mediawiki.sh export - ``` - 1. Store data in `cache-docker/` - 1. Copy `images/` (uploaded files) from legacy cache - 2. Dump database to a SQL file - 2. Approximate duration: 25 seconds -6. Import to production - 1. Verify connectivity - ```shell - railway ssh - ``` - ```shell - railway connect MySQL - ``` - 2. Import images - ```shell - railway volume files -v public-wiki-volume upload \ - cache-docker/images /mnt/wiki/ - ``` - 3. Import database - ```shell - railway connect MySQL < cache-docker/docker_mediawiki_export.sql - ``` - 4. Apprximate duration: 24 minutes - - -## Related documentation - - -### Docker - -- [Dockerfile reference | Docker Docs][dockerfile] -- [Compose file reference | Docker Docs][composefile] -- [Best practices | Docker Docs][practices] - -[dockerfile]: https://docs.docker.com/reference/dockerfile -[composefile]: https://docs.docker.com/reference/compose-file/ -[practices]: https://docs.docker.com/build/building/best-practices/ - - -#### Docker images - -- [mediawiki - Official Image | Docker Hu](https://hub.docker.com/_/mediawiki/) -- [mysql - Official Image | Docker Hub](https://hub.docker.com/_/mysql/) - - -### MediaWiki - - -#### MediaWiki on Debian - -- [Manual:Running MediaWiki on Debian or Ubuntu - MediaWiki][mw_on_debian] -- [MediaWiki - Debian Wiki][deb_wiki_mw] -- [Manual:Short URL/Apache - MediaWiki][mw_short_url] -- [Manual:Configuring file uploads - MediaWiki][mw_file_up] -- [MediaWiki 1.43 - MediaWiki][mw_1_43] LTS (long term support) - -[mw_on_debian]: https://www.mediawiki.org/wiki/Manual:Running_MediaWiki_on_Debian_or_Ubuntu -[deb_wiki_mw]: https://wiki.debian.org/MediaWiki -[mw_short_url]: https://www.mediawiki.org/wiki/Manual:Short_URL/Apache -[mw_file_up]: https://www.mediawiki.org/wiki/Manual:Configuring_file_uploads -[mw_1_43]: https://www.mediawiki.org/wiki/MediaWiki_1.43 - - -#### Mediawiki database schema - -- [Manual:Database layout - MediaWiki][mw_db_layout] -- [sql/mysql/tables-generated.sql - mediawiki/core - Gitiles][tables_sql] -- [standard UTF8 encoding for MediaWiki databases · Issue #373 · - openstreetmap/operations][osm_issue_373] - -[mw_db_layout]: https://www.mediawiki.org/wiki/Manual:Database_layout -[tables_sql]: https://gerrit.wikimedia.org/g/mediawiki/core/%2B/HEAD/sql/mysql/tables-generated.sql -[osm_issue_373]: https://github.com/openstreetmap/operations/issues/373 - - -#### MediaWiki maintenance - -- [Manual:**Maintenance scripts/List of scripts** - MediaWiki][mw_list_scripts] - -[mw_list_scripts]: https://www.mediawiki.org/wiki/Manual:Maintenance_scripts/List_of_scripts - - -#### MediaWiki release notes - -- `2018-06-13`: [MediaWiki 1.31 - - MediaWiki](https://www.mediawiki.org/wiki/MediaWiki_1.31) -- `2019-01-10`: [MediaWiki 1.32 - - MediaWiki](https://www.mediawiki.org/wiki/MediaWiki_1.32) -- `2019-07-02`: [MediaWiki 1.33 - - MediaWiki](https://www.mediawiki.org/wiki/MediaWiki_1.33) -- `2019-12-19`: [MediaWiki 1.34 - - MediaWiki](https://www.mediawiki.org/wiki/MediaWiki_1.34) -- `2020-09-25`: [MediaWiki 1.35 - - MediaWiki](https://www.mediawiki.org/wiki/MediaWiki_1.35) -- `2021-05-27`: [MediaWiki 1.36 - - MediaWiki](https://www.mediawiki.org/wiki/MediaWiki_1.36) -- `2021-11-18`: [MediaWiki 1.37 - - MediaWiki](https://www.mediawiki.org/wiki/MediaWiki_1.37) -- `2022-06-02`: [MediaWiki 1.38 - - MediaWiki](https://www.mediawiki.org/wiki/MediaWiki_1.38) -- `2022-11-30`: [MediaWiki 1.39 - - MediaWiki](https://www.mediawiki.org/wiki/MediaWiki_1.39) -- `2023-06-23`: [MediaWiki 1.40 - - MediaWiki](https://www.mediawiki.org/wiki/MediaWiki_1.40) -- `2023-12-21`: [MediaWiki 1.41 - - MediaWiki](https://www.mediawiki.org/wiki/MediaWiki_1.41) -- `2024-06-27`: [MediaWiki 1.42 - - MediaWiki](https://www.mediawiki.org/wiki/MediaWiki_1.42) -- `2024-12-21`: [MediaWiki 1.43 - - MediaWiki](https://www.mediawiki.org/wiki/MediaWiki_1.43) - - -### Railway - -- [Railway Docs](https://docs.railway.com/) - - [CLI | Railway Docs](https://docs.railway.com/cli) diff --git a/migrate/cache-docker/.gitignore b/migrate/cache-docker/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/migrate/cache-docker/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/migrate/cache-legacy/.gitignore b/migrate/cache-legacy/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/migrate/cache-legacy/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/migrate/docker-compose.yml b/migrate/docker-compose.yml deleted file mode 100644 index e1bb94c..0000000 --- a/migrate/docker-compose.yml +++ /dev/null @@ -1,83 +0,0 @@ -# https://docs.docker.com/compose/compose-file/ -name: public-wiki - -services: - - web: - container_name: web - depends_on: - - db - build: - context: ../ - links: - - db - env_file: - - ../.env - init: true - ports: - - '8080:80' - restart: on-failure - security_opt: - # For compatibility with Northflank - # (Even though this configuration is dev-only this reduces deltas) - - no-new-privileges:true - volumes: - # Apache2 web server configuration (read-only) - # (Supersedes copied file for easier development) - - ../config/web-sites-available:/etc/apache2/sites-available:ro - # MediaWiki volume - - volume-wiki:/mnt/wiki - # Legacy migration cache (read-only) - - ./cache-legacy:/var/cache-legacy:ro - - web-bullseye: - container_name: web-bullseye - depends_on: - - db - build: - args: - MW_SERVER_URL: 'http://localhost:8081' - context: ../ - dockerfile: migrate/Dockerfile.bullseye - environment: - MW_SERVER_URL: 'http://localhost:8081' - links: - - db - env_file: - - ../.env - init: true - ports: - - '8081:80' - restart: on-failure - security_opt: - # For compatibility with Northflank - # (Even though this configuration is dev-only this reduces deltas) - - no-new-privileges:true - volumes: - # Apache2 web server configuration (read-only) - - ../config/web-sites-available:/etc/apache2/sites-available:ro - # MediaWiki volume - - volume-wiki-bullseye:/mnt/wiki - # Legacy migration cache (read-only) - - ./cache-legacy:/var/cache-legacy:ro - - db: - container_name: db - # https://hub.docker.com/_/mysql/ - image: mysql - env_file: - - ../.env - restart: on-failure - volumes: - # Configuration (read-only) - - ../config/db.cnf:/etc/mysql/conf.d/db.cnf:ro - # Data - - volume-mysql:/var/lib/mysql - # Migration caches - - ./cache-legacy:/var/cache-legacy:ro - - ./cache-docker:/var/cache-docker - -volumes: - volume-mysql: - volume-wiki: - volume-wiki-bullseye: diff --git a/migrate/migrate_cc_mediawiki.sh b/migrate/migrate_cc_mediawiki.sh deleted file mode 100755 index 74892ed..0000000 --- a/migrate/migrate_cc_mediawiki.sh +++ /dev/null @@ -1,791 +0,0 @@ -#!/bin/bash -# -# Notes: -# - See NOTICE_ variables below -# - If you modify this file, please re-check it with shellcheck -# - https://www.mediawiki.org/wiki/Manual:ImportImages.php -# mw_run_web importImages is unnecessary because we are using a database -# dump. This was verified by running the import (every file found was -# skipped). -# -set -o errexit -set -o errtrace -set -o nounset - -# shellcheck disable=SC2154 -trap '_es=${?}; - printf "${0}: line ${LINENO}: \"${BASH_COMMAND}\""; - printf " exited with a status of ${_es}\n"; - exit ${_es}' ERR - -# File-path based variables -DIR_MIGRATE="$(cd -P -- "${0%/*}" && pwd -P)" -# shellcheck disable=SC2034 -DIR_REPO="$(cd -P -- "${0%/*}/.." && pwd -P)" -SCRIPT_NAME="${0##*/}" - -# Docker cache variables -DCACHE_DOCKER_SQL=/var/cache-docker/docker_mediawiki_export.sql -DCACHE_LEGACY_IMAGES_DIR=/var/cache-legacy/images -DCACHE_LEGACY_SQL=/var/cache-legacy/legacy_mediawiki_export.sql - -# Local cache variables -LCACHE_DOCKER_DIR="${DIR_MIGRATE}/cache-docker" -LCACHE_DOCKER_SQL="${LCACHE_DOCKER_DIR}/docker_mediawiki_export.sql" -LCACHE_LEGACY_DIR="${DIR_MIGRATE}/cache-legacy" -LCACHE_LEGACY_IMAGES_DIR="${LCACHE_LEGACY_DIR}/images" -LCACHE_LEGACY_SQL="${LCACHE_LEGACY_DIR}/legacy_mediawiki_export.sql" - -DOCKER_MW_IMAGES_DIR=/mnt/wiki/var-lib-mediawiki-images -# The command_parse() function sets the COMMAND variables: -COMMAND='' -# https://en.wikipedia.org/wiki/ANSI_escape_code -E0="$(printf "\e[0m")" # reset -E30="$(printf "\e[30m")" # foreground: black -E31="$(printf "\e[31m")" # foreground: red -E33="$(printf "\e[33m")" # foreground: yellow -E43="$(printf "\e[43m")" # foreground: yellow -E90="$(printf "\e[90m")" # foreground: bright black (gray) -E92="$(printf "\e[92m")" # foreground: bright green -E93="$(printf "\e[93m")" # foreground: bright yellow -E97="$(printf "\e[97m")" # foreground: bright white -E100="$(printf "\e[100m")" # background: bright black (gray) -E107="$(printf "\e[107m")" # background: bright white -MYSQL_CLIENT_CONFIG="\ -[client] -password = 'REPlACEPASSWORD' - -[mysql] -batch -database = 'REPLACEDATABASE' -skip-column-names - -[mysqldump] -no-tablespaces -set-gtid-purged = OFF -single-transaction" -NOTICE_CONTAINERS="\ -⚠️ This script's import command requires the services in - DIR_REPO/migrate/docker-compose.yml, which includes both web-bullseye - (MediaWiki 1.35.13) and web (MediaWiki 1.43.8). Care should be taken as they - share the same database." -NOTICE_HELP=" -${SCRIPT_NAME} COMMAND - -${E97}Commands${E0} - -help print this help message and exit - -info run tests and print setup information - -pull export MediaWiki data (database SQL and images files) from Bytemark - legacy virtual machine - -import import MediaWiki data to local Docker containers and upgrade - MediaWiki - -export export MediaWiki data (database SQL and images file) from local - Docker containers -" -NOTICE_STAFF="\ -⚠️ This script's pull command can only be run by Creative Commons (CC) team - members--it requires shell access to the Bytemark legacy virtual machine." -LEGACY_IMAGES_DIR=/var/www/images/ -LEGACY_MW_DB=ccwiki -LEGACY_SERVER=wiki.default.creativecommons.uk0.bigv.io -# The rsync_version() function sets the RSYNC_PROT_VER global variable: -declare -i RSYNC_PROT_VER=0 -declare -i RSYNC_PROT_VER_MIN=31 - -#### FUNCTIONS ################################################################ - -command_help() { - print_header 'Usage' - echo "${NOTICE_HELP}" -} - - -command_parse() { - if [[ -z "${1}" ]] - then - error_exit 'a COMMAND is required' - elif [[ -n "${2:-}" ]] - then - error_exit 'only a single COMMAND is allowed' - fi - case "${1:-}" in - -h*|--h*|h*|'-?'|'--?'|'?') COMMAND='help';; - pull) COMMAND='pull';; - import) COMMAND='import';; - info) COMMAND='info';; - export) COMMAND='export';; - *) error_exit "invalid COMMAND: ${1}";; - esac -} - - -danger_confirm() { - local _confirm _i _prompt _rand - - if [[ "${DANGER_BYPASS:-}" == 'i will be careful' ]] - then - return - fi - - printf "${E43}${E30} %-71s$(date '+%T') ${E0}\n" \ - 'Confirmation required' - echo -e "${E33}WARNING:${E0} the '${COMMAND}' command is destructive" - # Loop until user enters random number - _rand=${RANDOM}${RANDOM}${RANDOM} - _rand=${_rand:0:4} - _prompt="Type the number, ${_rand}, to continue: " - _i=0 - while read -p "${_prompt}" -r _confirm - do - if [[ "${_confirm}" == "${_rand}" ]] - then - echo - return - fi - (( _i > 1 )) && error_exit 'invalid confirmation number' - _i=$(( ++_i )) - done -} - - -database_maintenance(){ - local _note _note_one _note_two _table _tables - print_header 'Optimize MediaWiki database tables' - _tables="$(docker compose exec db sh -c \ - 'echo "SELECT TABLE_NAME FROM information_schema.tables \ - WHERE TABLE_SCHEMA=\"my_wiki\";" \ - | /usr/bin/mysql')" - # Check - echo 'Check tables' - for _table in ${_tables} - do - docker compose exec db sh -c \ - "echo 'CHECK TABLE ${_table}' | /usr/bin/mysql \ - | sed --null-data \ - -e's/my[^\n]*OK\n/${E92}.${E0}/g' \ - -e's/my[^\n]*support check\n/${E93}.${E0}/g'" - #| awk -F'\\t' '{printf \"%-30s %s\\n\", \$1, \$4}'" - done - echo - # Optimize - echo 'Optimize tables' - for _table in ${_tables} - do - docker compose exec db sh -c \ - "echo 'OPTIMIZE TABLE ${_table}' | /usr/bin/mysql \ - | sed --null-data \ - -e's/my[^\n]*analyze instead\n//g' \ - -e's/my[^\n]*OK\n/${E92}.${E0}/g' \ - -e's/my[^\n]*support optimize\n/${E93}.${E0}/g'" - done - echo - # Analyze - echo 'Analyze tables' - for _table in ${_tables} - do - docker compose exec db sh -c \ - "echo 'ANALYZE TABLE ${_table}' | /usr/bin/mysql \ - | sed --null-data \ - -e's/my[^\n]*OK\n/${E92}.${E0}/g' \ - -e's/my[^\n]*support analyze\n/${E93}.${E0}/g'" - done - echo - echo -} - - -database_update_phase1() { - print_header 'Update database - phase 1' - print_key_val 'Container context' 'web-bullseye' - - # https://www.mediawiki.org/wiki/Manual:Update.php - echo -n "Update to MediaWiki 1.35.13 (web-bullseye) ${E90}from MediaWiki" - echo " 1.30.0 (Bytemark)${E0}" - mw_run_web_bullseye update.php --quiet --quick 2>&1 \ - | sed \ - -e'/cleanupUsersWithNoId.php to fix this situation./d' \ - -e'/^$/d' - - # https://www.mediawiki.org/wiki/Manual:CleanupUsersWithNoId.php - echo 'Clean-up MediaWiki users with no ID on web-bullseye' - mw_run_web_bullseye cleanupUsersWithNoId.php --quiet --prefix '*' - - # https://www.mediawiki.org/wiki/Manual:MigrateActors.php - # Proven to be unneeded; probably handled by update - - # The above command should be the last one executed on web-bullseye - echo -n "${E93}Remove mw_run_web_bullseye() function for remaining" - echo " script run${E0}" - unset -f mw_run_web_bullseye - - echo -} - - -database_update_phase2() { - print_header 'Update database - phase 2' - print_key_val 'Container context' 'web' - - # https://www.mediawiki.org/wiki/Manual:Update.php - echo -n "Update to MediaWiki to 1.43.8 (web) ${E90}from MediaWiki 1.35.13" - echo " (web-bullseye)${E0}" - mw_run_web update --quiet --quick - - echo -} - - -error_exit() { - # Echo error message and exit with error - echo -e "${E31}ERROR:${E0} ${*}" 1>&2 - exit 1 -} - - -export_images() { - print_header 'Export images (copy from legacy cache)' - print_var LCACHE_LEGACY_IMAGES_DIR - print_var LCACHE_DOCKER_DIR - echo - echo 'Rsync cache images MediaWiki images (removes files not in cache)' - # The rsync options below are ordered to match `man rsync` - rsync \ - --recursive \ - --links \ - --delete \ - --delete-excluded \ - --partial \ - --prune-empty-dirs \ - --times \ - --stats \ - --human-readable \ - "${LCACHE_LEGACY_IMAGES_DIR}" \ - "${LCACHE_DOCKER_DIR}/" - echo -} - - -export_sql() { - # https://www.mediawiki.org/wiki/Manual:Backing_up_a_wiki - print_header 'Export MediaWiki database from Docker container' - print_key_val 'Container context' 'db' - print_var DCACHE_DOCKER_SQL - print_var LCACHE_DOCKER_SQL - echo - mkdir -p "${LCACHE_DOCKER_DIR}" - # https://dev.mysql.com/doc/refman/9.7/en/mysqldump.html - docker compose exec --env DCACHE_DOCKER_SQL="${DCACHE_DOCKER_SQL}" db \ - sh -c '/usr/bin/mysqldump "${MYSQL_DATABASE}" \ - > "${DCACHE_DOCKER_SQL}.tmp"' - mv "${LCACHE_DOCKER_SQL}.tmp" "${LCACHE_DOCKER_SQL}" - du -sh "${LCACHE_DOCKER_SQL}" | repo_rel_path - echo -} - - -import_database() { - print_header 'Import prepared MediaWiki SQL dump' - print_key_val 'Container context' 'db' - print_var DCACHE_LEGACY_SQL - echo 'Import database dump SQL' - docker compose exec --env DCACHE_LEGACY_SQL="${DCACHE_LEGACY_SQL}" db \ - sh -c '/usr/bin/mysql < "${DCACHE_LEGACY_SQL}"' - echo -} - - -import_images() { - print_header 'Import images (pulled from Bytemark)' - print_key_val 'Container context' 'web' - print_var DCACHE_LEGACY_IMAGES_DIR - print_var DOCKER_MW_IMAGES_DIR - echo 'Rsync cache images MediaWiki images (removes files not in cache)' - # The rsync options below are ordered to match `man rsync` - docker compose exec --user root web \ - rsync \ - --recursive \ - --links \ - --delete \ - --delete-excluded \ - --partial \ - --prune-empty-dirs \ - --times \ - --stats \ - --human-readable \ - "${DCACHE_LEGACY_IMAGES_DIR}/" \ - "${DOCKER_MW_IMAGES_DIR}/" - echo 'Set ownership of entire images dir to www-data:wwww-data' - docker compose exec --user root web chown -R www-data:www-data \ - "${DOCKER_MW_IMAGES_DIR}" - echo -} - - -mw_maintenance_accounts() { - local _group _user - print_header 'MediaWiki account maintenance' - - # https://www.mediawiki.org/wiki/Manual:RemoveUnusedAccounts.php - echo 'Remove unused accounts' - mw_run_web removeUnusedAccounts --delete --ignore-touched 0 2>&1 \ - | grep '^\.\.\.found.*[0-9]\.$' | gsed \ - -e"s/^\\.\\.\\.found/${E90}- deleted/" \ - -e"s/\\.$/ unused accounts${E0}/" - - # https://www.mediawiki.org/wiki/Manual:DeleteLocalPasswords.php - echo 'Delete local passwords' - mw_run_web deleteLocalPasswords --quiet --delete - - # https://www.mediawiki.org/wiki/Manual:EmptyUserGroup.php - echo 'Remove all users from legacy groups' - for _group in affiliate approved community regional staff - do - mw_run_web emptyUserGroup "${_group}" 2>&1 \ - | gsed -e'/^Removing users from /d' \ - -e"s/^ ...done! R/- ${_group} ${E90}(r/" \ - -e"s/^ ...nothing to do, /- ${_group} ${E90}(/" \ - -e"s/\\.\$/)${E0}/" - done - echo 'Remove all users from privileged groups' - for _group in bot bureaucrat sysop - do - mw_run_web emptyUserGroup "${_group}" 2>&1 \ - | gsed -e'/^Removing users from /d' \ - -e"s/^ ...done! R/- ${_group} ${E90}(r/" \ - -e"s/^ ...nothing to do, /- ${_group} ${E90}(/" \ - -e"s/\\.\$/)${E0}/" - done - - # https://www.mediawiki.org/wiki/Manual:CreateAndPromote.php - echo 'Add appropriate and verified users to sysop group' - for _user in CCID-marimoreshead CCID-shinchpearson CCID-timidrobot - do - echo "- ${_user}" - mw_run_web createAndPromote "${_user}" \ - "x${RANDOM}${RANDOM}${RANDOM}${RANDOM}${RANDOM}${RANDOM}" \ - --sysop --force --quiet - done - # Remove temporary random passwords, added above - mw_run_web deleteLocalPasswords --quiet --delete - echo "- WikiSysop ${E90}(and restore password from environment)${E0}" - docker compose exec --user www-data web sh -c '/usr/bin/php \ - /usr/share/mediawiki/maintenance/run.php createAndPromote \ - WikiSysop "${MW_ADMIN_PASS}" --sysop --force --quiet' - - echo -} - - -mw_maintenance_images() { - local _dir - print_header 'MediaWiki image maintenance' - print_var DOCKER_MW_IMAGES_DIR - - ## https://www.mediawiki.org/wiki/Manual:ImportImages.php - ## https://www.mediawiki.org/wiki/Manual:RebuildImages.php - # Unneeded as we are using a database dump (already includes image info) - - ## https://www.mediawiki.org/wiki/Manual:CheckImages.php - - ## https://www.mediawiki.org/wiki/Manual:FindMissingFiles.php - # Useless: provides data that is unactionable. - - # https://www.mediawiki.org/wiki/Manual:RefreshImageMetadata.php - echo 'Refresh image metadata' - mw_run_web refreshImageMetadata --quiet - - # https://www.mediawiki.org/wiki/Manual:RefreshFileHeaders.php - echo 'Refresh file headers' - mw_run_web refreshFileHeaders --quiet - - # https://www.mediawiki.org/wiki/Manual:CleanupUploadStash.php - echo 'Clean-up upload stash' - mw_run_web cleanupUploadStash --quiet - - echo -} - - -mw_maintenance_rebuild() { - print_header 'MediaWiki rebuild maintenance' - # Note: rebuildall.php is not used because individual scripts allow more - # user feedback (more echo statements) - - # https://www.mediawiki.org/wiki/Manual:Rebuildtextindex.php - echo 'Rebuild text index (rebuild the searchindex table)' - mw_run_web rebuildtextindex --quiet - - # https://www.mediawiki.org/wiki/Manual:Rebuildrecentchanges.php - echo 'Rebuild recent changes' - mw_run_web rebuildrecentchanges --quiet - - # https://www.mediawiki.org/wiki/Manual:RefreshLinks.php - echo -n 'Refresh links (refill pagelinks, categorylinks, and imagelinks' - echo ' tables)' - mw_run_web refreshLinks --quiet - - echo -} - - -mw_maintenance_titles() { - print_header 'MediaWiki titles maintenance' - # https://www.mediawiki.org/wiki/Manual:cleanupTitles.php - echo 'Clean-up page titles' - mw_run_web cleanupTitles --quiet - echo -} - - -mw_run_web() { - # https://www.mediawiki.org/wiki/Manual:Maintenance_scripts - # Since MediaWiki 1.40, maintenance scripts should be invoked - # indirectly through php ./maintenance/run.php. Invoking maintenance - # scripts directly will trigger a warning. - docker compose exec --user www-data web /usr/bin/php \ - /usr/share/mediawiki/maintenance/run "${@}" -} - - -mw_run_web_bullseye() { - # https://www.mediawiki.org/wiki/Manual:Sql.php, for example: - # In MediaWiki version MediaWiki 1.39 and earlier, you must invoke - # maintenance scripts using php maintenance/scriptName.php instead of - # php maintenance/run.php scriptName. - # shellcheck disable=SC2145 - docker compose exec --user www-data web-bullseye /usr/bin/php \ - /usr/share/mediawiki/maintenance/"${@}" -} - - -msyql_client_config() { - print_header 'Write MySQL client configuration' - docker compose exec db sh -c \ - "echo \"${MYSQL_CLIENT_CONFIG}\" > /root/.my.cnf" - docker compose exec db chmod 0400 root/.my.cnf - docker compose exec db sh -c 'sed \ - -e"s/REPlACEPASSWORD/${MYSQL_ROOT_PASSWORD}/" \ - -e"s/REPLACEDATABASE/${MYSQL_DATABASE}/" \ - -i /root/.my.cnf' - docker compose exec db ls -l /root/.my.cnf - docker compose exec db cat /root/.my.cnf - echo -} - - -mysql_disable_redo_log() { - print_header 'Disable MySQL redo log' - docker compose exec db sh -c \ - 'echo "ALTER INSTANCE DISABLE INNODB REDO_LOG;" \ - | /usr/bin/mysql' - echo -n 'InnoDB redo logging is disabled. All data could be lost in case' - echo ' of a server' - echo 'crash.' - echo -} - - -mysql_enable_redo_log() { - print_header 'Disable MySQL redo log' - docker compose exec db sh -c \ - 'echo "ALTER INSTANCE ENABLE INNODB REDO_LOG;" \ - | /usr/bin/mysql' - echo -n 'InnoDB redo logging is enabled. Data is now safe and can be' - echo ' recovered in case of' - echo 'a server crash.' - echo -} - - -notice_staff_only() { - echo "${E93}${NOTICE_STAFF}${E0}" - echo -} - - -notice_containers() { - echo "${E93}${NOTICE_CONTAINERS}${E0}" - echo -} - - -prep_sql() { - print_header 'Prepare MediaWiki SQL dump (pulled from Bytemark)' - print_var LCACHE_LEGACY_SQL - echo "Update ENGINE to InnoDB ${E90}(from MyISAM)${E0}" - echo "Update CHARSET to binary ${E90}(from latin1)${E0}" - # shellcheck disable=SC2016 - echo "Update \`searchindex\` CHARSET to utf8mb4 ${E90}(from latin1)${E0}" - echo 'Update datetime default to a valid (non-zero) value' - # shellcheck disable=SC2016 - gsed --regexp-extended --null-data \ - -e's/ENGINE=MyISAM/ENGINE=InnoDB/g' \ - -e's/CHARSET=latin1/CHARSET=binary/g' \ - -e's/(FULLTEXT KEY `si_text` \(`si_text`\)\n\) ENGINE=)InnoDB( DEFAULT CHARSET=)binary/\1InnoDB\2utf8mb4/' \ - -e"s/datetime NOT NULL DEFAULT '0000-00-00 00:00:00'/datetime NOT NULL DEFAULT '1000-01-01 00:00:00'/" \ - -i "${LCACHE_LEGACY_SQL}" - echo -} - - -print_header() { - # Print 80 character wide black on white heading - printf "${E30}${E107}# %-69s$(date '+%T') ${E0}\n" "${@}" -} - - -print_key_val() { - printf "${E97}${E100}%25s${E0} %s\n" "${1}:" "${2}" | repo_rel_path -} - - -print_var() { - print_key_val "${1}" "${!1}" -} - - -pull_database() { - # https://www.mediawiki.org/wiki/Manual:Backing_up_a_wiki - print_header 'Pull MediaWiki database from legacy production server' - print_var LEGACY_SERVER - print_var LEGACY_MW_DB - print_var LCACHE_LEGACY_SQL - mkdir -p "${LCACHE_LEGACY_DIR}" - echo 'Dump database' - # https://dev.mysql.com/doc/refman/9.7/en/mysqldump.html - ssh "${LEGACY_SERVER}" \ - sudo mysqldump --defaults-extra-file=/etc/mysql/debian.cnf \ - --no-tablespaces --single-transaction --skip-lock-tables \ - "${LEGACY_MW_DB}" \ - > "${LCACHE_LEGACY_SQL}.tmp" - mv "${LCACHE_LEGACY_SQL}.tmp" "${LCACHE_LEGACY_SQL}" - echo - echo 'Back up database export and compress it (keep uncompressed)' - gzip --force --keep "${LCACHE_LEGACY_SQL}" - du -sh "${LCACHE_LEGACY_SQL}" | repo_rel_path - du -sh "${LCACHE_LEGACY_SQL}.gz" | repo_rel_path - echo -} - - -pull_images() { - # https://www.mediawiki.org/wiki/Manual:Backing_up_a_wiki - print_header 'Pull MediaWiki images files from legacy production server' - print_var LEGACY_SERVER - print_var LEGACY_IMAGES_DIR - print_var LCACHE_LEGACY_DIR - mkdir -p "${LCACHE_LEGACY_DIR}" - # The rsync options below are ordered to match `man rsync` - rsync \ - --recursive \ - --links \ - --delete \ - --delete-excluded \ - --partial \ - --prune-empty-dirs \ - --times \ - --exclude 'CVS' \ - --exclude 'lock_*' \ - --exclude '.htaccess' \ - --exclude 'temp' \ - --stats \ - --human-readable \ - "${LEGACY_SERVER}:${LEGACY_IMAGES_DIR}" \ - "${LCACHE_LEGACY_IMAGES_DIR}/" - echo - du -sh "${LCACHE_LEGACY_IMAGES_DIR}" | repo_rel_path - echo -} - - -repo_rel_path() { - gsed -e"s#${DIR_MIGRATE}#DIR_REPO/migrate#" -} - - -rsync_version() { - local _rsync_version - _rsync_version="$(rsync --version 2>&1 \ - | awk '/version/ {print $3", "$4" "$5" "$6}')" - RSYNC_PROT_VER=${_rsync_version##* } - print_key_val 'rsync version' "${_rsync_version}" - # Check rsync version - if (( RSYNC_PROT_VER < 31 )) - then - _err="rsync protocol version ${RSYNC_PROT_VER} is less than" - _err="${_err} ${RSYNC_PROT_VER_MIN}--please install via" - _err="${_err} \`brew install rsync\` (you may need to open a" - _err="${_err} new terminal to see the new rsync version)" - error_exit "${_err}" - fi -} - - -script_setup() { - local _cache_dir_filesystem _err _rsync_ver _service _var - if [[ "$(uname)" != 'Darwin' ]] - then - error_exit 'only a macOS (Darwin) environment is supported' - fi - - print_header "Setup environment: local docker development" - # Check execution environment - if [[ "${PWD##*/}" != 'migrate' ]] - then - _err='this script must be executed from a clone of the public-wiki' - _err="${_err} repository (this check requires the current directory" - _err="${_err} to me named 'migrate')" - error_exit "${_err}" - fi - if ! command -v gsed >/dev/null - then - # shellcheck disable=SC2016 - error_exit \ - 'GNU sed is required. If on macOS install `gnu-sed` via brew.' - fi - - print_var DIR_REPO - print_var DIR_MIGRATE - print_key_val "$(sw_vers --productName) version" \ - "$(sw_vers --productVersion)" - rsync_version - print_key_val 'Docker version' \ - "$(docker --version | sed -e's/^Docker *version *//')" - print_var COMMAND - - echo -} - - -success() { - printf "${E92}Success:${E0} %s\n" "${@}" -} - - -test_ssh_to_prod() { - print_header 'Test SSH connection to production server' - print_var LEGACY_SERVER - if ! ssh "${LEGACY_SERVER}" true - then - error_exit 'unable to connect--verify config and public key' - else - success 'connection verified' - echo - fi -} - - -verify_docker_services() { - local _msg _target _service _webs - _target="${1}" - case "${_target}" in - all) _webs='web web-bullseye';; - minimal) _webs='web';; - esac - # Ensure docker daemon is running - if [[ ! -S /var/run/docker.sock ]] - then - error_exit 'docker daemon is not running' - fi - # Ensure db service is running - if ! docker compose exec db true 2>/dev/null - then - error_exit 'docker service is not running: db' - fi - printf "${E30}${E107}%-25s${E0}\n" ' db container' - print_key_val 'Debian version' \ - "$(docker compose exec db cat /etc/oracle-release)" - print_key_val 'MySQL version' \ - "$(docker compose exec db mysql --version \ - | sed -e's/mysql Ver //')" - - # Ensure web services are running - for _service in ${_webs} - do - if ! docker compose exec --user www-data "${_service}" true 2>/dev/null - then - error_exit "docker service is not running: ${_service}" - fi - if ! docker compose exec --user www-data "${_service}" \ - test -f /etc/mediawiki/LocalSettings.php - then - _msg="${_service}: initial MediaWiki install has not been" - _msg="${_msg} completed" - error_exit "${_msg}" - fi - printf "${E30}${E107}%-25s${E0}\n" " ${_service} container" - print_key_val 'Debian version' \ - "$(docker compose exec --user www-data "${_service}" \ - cat /etc/debian_version)" - print_key_val 'PHP version' \ - "$(docker compose exec --user www-data "${_service}" \ - /usr/bin/php --version \ - | awk '/^PHP/ {print $2}')" - print_key_val 'MediaWiki version' \ - "$(docker compose exec --user www-data "${_service}" \ - apt-cache show mediawiki \ - | awk '/^Version:/ {print $2}')" - print_key_val 'MySQL client library' \ - "mariadb-client $(docker compose exec --user www-data \ - "${_service}" apt-cache show mariadb-client \ - | awk '/^Version:/ {print $2}')" - done - echo -} - -#### MAIN ##################################################################### - -cd "${DIR_MIGRATE}" -command_parse "${@:-}" -case "${COMMAND}" in - # the following are sorted by order of operations - 'help') - command_help - ;; - 'info') - script_setup - verify_docker_services 'all' - notice_staff_only - notice_containers - test_ssh_to_prod - ;; - 'pull') - script_setup - notice_staff_only - test_ssh_to_prod - pull_images - pull_database - ;; - 'import') - script_setup - verify_docker_services 'all' - notice_containers - danger_confirm - msyql_client_config - import_images - prep_sql - mysql_disable_redo_log - import_database - database_update_phase1 - database_update_phase2 - mw_maintenance_accounts - mw_maintenance_images - mw_maintenance_titles - mw_maintenance_rebuild - mysql_enable_redo_log - database_maintenance - ;; - 'export') - script_setup - verify_docker_services 'minimal' - notice_containers - msyql_client_config - export_images - export_sql - ;; -esac From 7e667465dc21dcc6169f79d530eac9b08e62c5b9 Mon Sep 17 00:00:00 2001 From: Timid Robot Zehta Date: Fri, 23 Oct 2026 12:45:45 +0200 Subject: [PATCH 2/3] update README to reflect production --- README.md | 98 +++++++++++++++++++++++++------------------------------ 1 file changed, 45 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 0791c3d..195ea68 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,19 @@ # public-wiki -_Development_ Creative Commons (CC) Public Wiki +Creative Commons (CC) Public Wiki ## Overview -The aim of the project is to establish a robust and localized development -environment for MediaWiki using Docker. This repository should represent the -most advanced and closest implementation of the -[CC Public Wiki](https://wiki.creativecommons.org). +The goals of this project are to: +1. Maintain the [Package public-wiki][public-wiki-pkg] in support of the CC + Public Wiki remote production Railway environment +2. Maintain the CC Public Wiki local Docker development environment +3. Maintain parity between production and development environments + +Production website: [CC Public Wiki](https://wiki.creativecommons.org/) + +[public-wiki-pkg]: https://github.com/creativecommons/public-wiki/pkgs/container/public-wiki ## Code of Conduct @@ -33,7 +38,10 @@ section, below. [org-contrib]: https://github.com/creativecommons/.github/blob/main/CONTRIBUTING.md -## Docker containers +## Development + + +### Docker containers The [`docker-compose.yml`](docker-compose.yml) file defines the following containers: @@ -42,48 +50,6 @@ containers: - **db** - Database server (MySQL) -## Dev configuration - - -### Apache2 - -See [`config/web-sites-available/000-default.conf`][dev-webconfig]. - -[dev-webconfig]: config/web-sites-available/000-default.conf - - -### MediaWiki configuration - -| Name | Version | Notes -| --------- | -------- | ------------------------------- | -| MediaWiki | `1.43.6` | Packaged for Debian 13 (trixie) | - -Also see: -- [`.env.example`](.env.example). -- [`config/configure_mediawiki.sh`](config/configure_mediawiki.sh) - - -### Migration from legacy server - -- [`migrate/`](migrate/) - - [`README.md`](migrate/README.md) - - Includes related Debian, Docker, and MediaWiki documentation links - - -### Related development links - -- [creativecommons/index-dev-env][index-dev-env]: _Local development - environment for CreativeCommons.org (product name: index)_ -- [creativecommons/sre-salt-prime][sre-salt-prime]: _Site Reliability - Engineering / DevOps SaltStack configuration files_ - -[index-dev-env]: https://github.com/creativecommons/index-dev-env -[sre-salt-prime]: https://github.com/creativecommons/sre-salt-prime - - -## Development - - ### Dependencies 1. Install dependencies @@ -98,10 +64,6 @@ Also see: 5. Install Rsync ([rsync download][rsync-install]) 6. Install shellcheck ([Installing - koalaman/shellcheck][shellcheck-install]) -2. Prepare and install prek Git hooks: - ```shell - prek install --prepare-hooks - ``` [docker-install]: https://docs.docker.com/get-started/get-docker/ [hadolint-install]: https://github.com/hadolint/hadolint#install @@ -113,7 +75,11 @@ Also see: ## Setup -1. Create the enviornment `.env` file by copying +1. Prepare and install prek Git hooks: + ```shell + prek install --prepare-hooks + ``` +2. Create the enviornment `.env` file by copying [`.env.example`](.env.example): ```shell cp .env.example .env @@ -126,6 +92,32 @@ Also see: 4. Wait for the build and initialization to complete +## Web container configuration + +See the `config/configure_*` configuration scripts called from [`config/startupservice.sh`][config/startupservice.sh]. + + +#### MediaWiki configuration + +| Name | Version | Notes +| --------- | -------- | ------------------------------- | +| MediaWiki | `1.43.6` | Packaged for Debian 13 (trixie) | + +Also see: +- [`.env.example`](.env.example). +- [`config/configure_mediawiki.sh`](config/configure_mediawiki.sh) + + +## Migration from legacy server + +The legacy migration directory has been deleted. Links below are to commit `e1ea34af3cc8151df8e73d80b2d0c2e7dff2d5e8`. +- [`migrate/`][migrate-dir] + - [`README.md`][migrate-readme] + - Includes related Debian, Docker, and MediaWiki documentation links + +[migrate-dir]: https://github.com/creativecommons/public-wiki/tree/e1ea34af3cc8151df8e73d80b2d0c2e7dff2d5e8/migrate +[migrate-readme]: https://github.com/creativecommons/public-wiki/blob/e1ea34af3cc8151df8e73d80b2d0c2e7dff2d5e8/migrate/README.md + ## License From f730326fed1b0d90d92912d1751097a7a8a12be2 Mon Sep 17 00:00:00 2001 From: Timid Robot Zehta Date: Fri, 23 Oct 2026 12:55:10 +0200 Subject: [PATCH 3/3] update for production --- .cc-metadata.yml | 4 ++-- .gitignore | 3 --- cache/export/.gitignore | 4 ++++ cache/legacy/.gitignore | 4 ++++ cache/pull/.gitignore | 4 ++++ prek.toml | 2 +- 6 files changed, 15 insertions(+), 6 deletions(-) create mode 100644 cache/export/.gitignore create mode 100644 cache/legacy/.gitignore create mode 100644 cache/pull/.gitignore diff --git a/.cc-metadata.yml b/.cc-metadata.yml index b88b3c5..0e961a7 100644 --- a/.cc-metadata.yml +++ b/.cc-metadata.yml @@ -1,7 +1,7 @@ # Whether this GitHub repo is engineering related -engineering_project: false +engineering_project: true # Name of the repository/project in English -english_name: Creative Commons Public Wiki +english_name: Creative Commons (CC) Public Wiki # All technologies used technologies: MediaWiki, Debian, Docker # Whether this repository should be featured on the CC Open Source site diff --git a/.gitignore b/.gitignore index e2e4b8c..845c5b6 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,3 @@ Thumbs.db # Local configuration .env .railway_token - -# Caches -cache/ diff --git a/cache/export/.gitignore b/cache/export/.gitignore new file mode 100644 index 0000000..5e7d273 --- /dev/null +++ b/cache/export/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/cache/legacy/.gitignore b/cache/legacy/.gitignore new file mode 100644 index 0000000..5e7d273 --- /dev/null +++ b/cache/legacy/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/cache/pull/.gitignore b/cache/pull/.gitignore new file mode 100644 index 0000000..5e7d273 --- /dev/null +++ b/cache/pull/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/prek.toml b/prek.toml index d2e508a..f24a176 100644 --- a/prek.toml +++ b/prek.toml @@ -1,4 +1,4 @@ -exclude = "^migrate/cache-" +exclude = "^cache/" # meta