% \iffalse meta-comment
%
% Copyright (C) 2026 Alan J. Cain
%
% This file may be distributed and/or modified under the conditions of the LaTeX Project Public License, either version
% 1.3c of this license or (at your option) any later version. The latest version of this license is in:
%
% http://www.latex-project.org/lppl.txt
%
% and version 1.3c or later is part of all distributions of LaTeX version 2008-05-04 or later.
%
% \fi
%
% \iffalse
%<*driver>
\PassOptionsToPackage{inline}{enumitem}
\documentclass{l3doc}

\usepackage{polyglossia}
\setmainlanguage[variant=british]{english}

\pghyphenation[variant=british]{english}{
  div-ision
  hy-phen-ation
  hy-phen-ations
  ver-bose-ly
}


\makeatletter
\ExplSyntaxOn

\cs_gset:Npn \l@subsection { \@dottedtocline{2}{2.5em}{2.8em} }  % #2 = 1.5em
\cs_gset:Npn \l@subsubsection { \@dottedtocline{3}{5.3em}{3.5em} }  % #2 = 1.5em
\cs_gset:Npn \l@paragraph { \@dottedtocline{4}{8.8em}{3.2em} }  % #2 = 1.5em

\ExplSyntaxOff
\makeatother


\usepackage{xcolor}

\definecolor{linkcolor}{rgb}{0.0,0.4,0.7}
\colorlet{citecolor}{linkcolor}
\colorlet{urlcolor}{linkcolor}

\hypersetup{
  linkcolor=linkcolor,%
  citecolor=citecolor,%
  urlcolor=urlcolor,%
}
\usepackage{xurl}
\renewcommand*\UrlBigBreaks{}

\newcommand*\fullref[2]{%
  \hyperref[#2]{#1\penalty 200\ \ref*{#2}}%
}
\newcommand*\fullpageref[1]{%
  \hyperref[#1]{page\penalty 200\ \pageref*{#1}}%
}


\setcounter{tocdepth}{7}
\numberwithin{figure}{section}



\usepackage{lua-list-hyphen}

\usepackage{lipsum}
\usepackage{graphicx}
\usepackage{tikz}
\usetikzlibrary{calc}

% Listings
\usepackage{listings}

\lstset{
  language=,
  basicstyle=\small\ttfamily,
  basewidth=0.5em,
}


% Options

\newcommand*\key[1]{\texttt{#1}}
\newcommand*\val[1]{\texttt{#1}}
\newcommand*\keyvalue[2]{\texttt{#1=#2}}

\newlist{vallist}{description}{1}
\setlist[vallist]{
  leftmargin=3em,
  style=unboxed,
  labelsep=1em,
  font=\descriptionitemcolon,
  nosep,
}
\newcommand*{\descriptionitemcolon}[1]{\kern 1em #1:}

\NewDocumentCommand{\default}{ m }{(\textit{Default:}\nobreakspace #1)}


% Other general documentation
\newcommand*\defterm[1]{\textit{#1}}

\newcommand*\prefixedurl[1]{\textsc{url}:~\url{#1}}

\newcommand*\examplehyph[1]{\hbox{\textsl{#1}}}


% Code documentation
\newcommand*\luafunc[1]{\texttt{#1}}
\newcommand*\luavar[1]{\texttt{#1}}

\newcommand*\param[1]{\texttt{\##1}}



\begin{document}

\DocInput{lua-list-hyphen.dtx}

\PrintIndex

\end{document}
%</driver>
% \fi
%
%
%
% \GetFileInfo{lua-list-hyphen.sty}
%
%
%
% \title{^^A
% \pkg{lua-list-hyphen} ^^A
%   --- Per-language checking and listing of hyphenated words for Lua\LaTeX^^A
%   \footnote{This document describes \fileversion, last revised \filedate.}^^A
% }
%
% \author{^^A
%  Alan J. Cain\footnote{\texttt{a.j.cain (AT) gmail.com}}^^A
% }
%
% \date{Released \filedate}
%
% \maketitle
%
%
%
% \begin{abstract}
%   This Lua\LaTeX\ package can check each word that has been hyphenated across lines against language-specific
%   dictionaries of valid divisions, optionally adding visual flags to the generated document, and write files listing
%   hyphenations (and their contexts), for external checking.
% \end{abstract}
%
%
%
% \section*{Demonstration}
%
% \begingroup
%
% \newsavebox{\divisiondictbox}
% \savebox{\divisiondictbox}{\lstinputlisting{division-dict-ukenglish.txt}}
% \newsavebox{\resultbox}
% \savebox{\resultbox}{\lstinputlisting{lua-list-hyphen-doc-demo-ukenglish.hyph}}
%
% \noindent
% \begin{tikzpicture}[
%   every node/.style={
%     inner sep=0,
%   },
%   description/.style={
%     node font=\small\sffamily\raggedright,
%     align=left,
%   },
%     ]
%   \path (0,2pt); ^^A Tweak bounding box
%
%   \node[
%   description,
%   text width=35mm,
%   anchor=north west,
%   inner ysep=4pt
%   ] (dict-desc) at (0,0) {^^A
%     The (small) division dictionary for British English used for classifying hyphenations in this demonstration:^^A
%   };
%
%   \node[anchor=north west] at ($ (dict-desc.south west) + (0,12pt) $) {\usebox{\divisiondictbox}};
%
%   \node[inner sep=-5mm,anchor=north west] (result) at ($ (dict-desc.north east) + (8mm,0) $) {\includegraphics{lua-list-hyphen-doc-demo.pdf}};
%
%   \begin{pgfinterruptboundingbox}
%   \node[
%   description,
%   text width=90mm,
%   anchor=south west,
%   node font=\sffamily\raggedright,
%   inner ysep=4pt
%   ] at (result.north west) {^^A
%   From Gibbon’s \emph{Decline and Fall of the Roman Empire}, ch.III\\
%   {\footnotesize (set in narrow columns to increase frequency of hyphenation)\par}^^A
%   };
%   \end{pgfinterruptboundingbox}
%
%   \node[
%   description,
%   text width=\textwidth,
%   anchor=north west,
%   inner ysep=4pt
%   ] (result-desc) at ($ (result.south -| dict-desc.west) + (0,-7pt) $) {^^A
%   Using the supplied division dictionary, \pkg{lua-list-hyphen} has classified the hyphenations, flagged the
%   results visually, and written to a file those that are either (1)~invalid (not matching the dictionary, like
%   \examplehyph{irreg-ular}), (2)~ambiguous (matching one but not all entries for the word in the dictionary, like
%   \examplehyph{pres-ent}), or (3)~unknown (not in the dictionary, like the name \examplehyph{Ves-pasian}):^^A
%   };
%
%   \node[anchor=north west] at (result-desc.south west) {\usebox{\resultbox}};
%
% \end{tikzpicture}
%
% \endgroup
%
%
%
% \tableofcontents
%
%
%
% \begin{documentation}
%
%
%
% \section{Introduction}
%
% \TeX's algorithm for finding points where a word can be hyphenated is good, but not perfect.\footnote{For a
% description of the algorithm and its limitations, see Knuth's account in Appendix~H of \textit{The \TeX book}
% (Addison-Wesley, 2021. ISBN:~\texttt{978-0-201-13447-6})} The present author writes in British English, where the
% valid division points can depend on both the pronunciation of a word and its internal structure (and hence its
% etymology). Currently, \TeX's pattern-based approach produces \examplehyph{bio-lo-gic}, \examplehyph{bio-logy},
% \examplehyph{bio-lo-gist}, rather than the standard \examplehyph{bio-logic}, \examplehyph{biol-ogy},
% \examplehyph{biolo-gist}.\footnote{See the \textit{New Oxford Spelling Dictionary}, which is the authority for word
% divisions in British English (Oxford University Press, 2005. ISBN:~\texttt{978-0-19-860881-3}).} To deal with such
% cases generally, at least a substantially larger number of patterns would be required than are available at present.
% There are also various words where the valid division points in British English cannot be deduced from their spelling
% alone: for instance, the verbs \examplehyph{at-trib-ute}, \examplehyph{pre-sent}, \examplehyph{pro-duce},
% \examplehyph{re-cord} have different division points from the orthographically identical nouns
% \examplehyph{at-tri-bute}, \examplehyph{pres-ent}, \examplehyph{prod-uce}, \examplehyph{rec-ord}. For another example,
% compare \examplehyph{cur-ric-ulum vitae} and \examplehyph{school cur-ricu-lum}.
%
% Easy checking of the chosen hyphenations is desirable. With Lua\TeX, it is possible to extract the hyphenated words.
% Patrick Gundlach’s Lua\LaTeX\ package
% \pkg{lua-check-hyphen}\footnote{\prefixedurl{https://ctan.org/pkg/lua-check-hyphen}} offers this facility. It checks
% hyphenated words against a list of accepted hyphenations, writes unknown hyphenations to a file, and optionally flags
% them in the generated PDF. But it was first written in 2012, when Lua\TeX\ was at an earlier stage of development, and
% so it has certain problems, such as with words containing ligatures. It also lacks multi-language support.
%
% This Lua\LaTeX\ package, \pkg{lua-list-hyphen}, uses some ideas from \pkg{lua-check-hyphen} but was written from
% scratch to work with a modern Lua\TeX. It writes hyphenated words from each language to a separate file, so that they
% can be checked (manually or by an external program). The user can optionally supply per-language dictionaries of valid
% divisions, which are used to classify each hyphenation as either valid, invalid, ambiguous (like
% \examplehyph{re-cord}/\examplehyph{rec-ord}), or unknown (not in the relevant division dictionary). The user can
% choose to filter and not write out those hyphenations that are known to be valid with respect to the relevant division
% dictionary. \pkg{lua-list-hyphen} can also add visual flags either to every hyphenation in the generated PDF, or only
% those that are not classified as valid.
%
%
%
% \paragraph*{Licence.} \noindent\pkg{lua-list-hyphen} is released under the \LaTeX\ Project Public Licence v1.3c or
% later.\footnote{\textsc{url}: \url{https://www.latex-project.org/lppl.txt}}
%
%
%
% \paragraph*{Acknowledgements.} The author thanks Keno Wehr for corrections and comments on the documentation.
%
%
%
% \paragraph*{Feature requests and bug reports}
%
% The development code and issue tracker are hosted at Codeberg.\footnote{\textsc{url}:
% \url{https://codeberg.org/ajcain/lua-list-hyphen}}
%
%
%
% \paragraph*{Other resources}
%
% \phantomsection\label{page:hyphenassist}The author has written a simple console Python application
% \texttt{hyphenassist}\footnote{\textsc{url}: \url{https://codeberg.org/ajcain/hyphenassist}.} that checks the output
% lists of hyphenations against a dictionary of valid divisions and allows the user to quickly choose to add entries to
% the division dictionary, add hyphenation exceptions, or ignore particular hyphenations. He has used this program in
% conjunction with this package to check hyphenations in his own books.\footnote{In particular, \textit{Form \& Number:
% A History of Mathematical Beauty}. \textsc{url}: \url{https://archive.org/details/cain_formandnumber_ebook_large}.}
%
% The author's very limited and partial division dictionary for British English is available at
% Codeberg.\footnote{\textsc{url}: \url{https://codeberg.org/ajcain/division-dict}} He has assembled it through gradual
% checking of hyphenations in his own books, and so it is heavily skewed towards mathematical, philosophical, and
% historical terms and names. It includes divisions for some rarer technical terms and more obscure names, not found in
% references, which have been obtained by analysis, deduction, and analogy.
%
%
%
% \section{Requirements}
%
% \pkg{lua-list-hyphen} requires
% \begin{enumerate}[label={(\arabic*)}]
%   \item Lua\LaTeX,
%   \item a recent \LaTeX\ kernel with \pkg{expl3} support (any kernel version since 2020-02-02 should suffice).
% \end{enumerate}
% It does not depend on any other packages, but will interface with \pkg{babel} or \pkg{polyglossia} (if one of them is
% loaded) to determine the association of Lua\TeX\ numerical language~IDs with language names.
%
%
%
% \section{Installation}
%
% To install \pkg{lua-list-hyphen} manually, run \texttt{luatex lua-list-hyphen.ins} and copy
% \texttt{lua-list-hyphen.sty} and \texttt{lua-list-hyphen.lua} to somewhere Lua\LaTeX\ can find them.
%
%
%
% \section{Getting started}
%
% Simply load the package with \cs{usepackage}\texttt{\{lua-list-hyphen\}}. By default, \pkg{lua-list-package} writes
% hyphenations to files \cs{jobname}\file{-}\meta{lang}\file{.hyph}, without classification, sorting, or removal of
% duplicates. If either \pkg{babel} or \pkg{polyglossia} is loaded, \meta{lang} will usually be the name of the language
% as defined in \pkg{hyph-utf8} (which does not always coincide with the \pkg{babel} or \pkg{polyglossia} name or
% variant). If neither of these packages is in use, or if the language has been defined otherwise, \meta{lang} is the
% Lua\TeX\ numerical language~ID.
%
% To modify this basic functionality, add package options, which are a comma-separated list of key--value pairs. Here
% are a few to get started. Note that these options can be applied either globally or per-language; see the introduction
% to \fullref{Section}{sec:options} for an explanation.
% \begin{itemize}
%     \item To sort the output hyphenations case-sensitively or case-insensitively, add the package option
%           \keyvalue{sort}{case} or \keyvalue{sort}{nocase}. See
%           \fullref{Subsection}{subsec:options-sorting-deduplication} for further details.
%
%     \item To remove duplicates from the output hyphenations case-sensitively or case-insensitively, add the package
%           option \keyvalue{unique}{case} or \keyvalue{unique}{nocase}. See
%           \fullref{Subsection}{subsec:options-sorting-deduplication} for further details.
%
%     \item To classify hyphenations as valid/invalid/ambiguous/unknown, specify a file containing a dictionary of valid
%           divisions using the package option \keyvalue{division-dict-path}{\meta{path}} (or the synonymous
%           \keyvalue{allowlist-path}{\meta{path}}).
%
%           In particular, to use a division dictionary as a basic allowlist of hyphenations that should not be output,
%           add \keyvalue{output-mode}{non-valid}.
%
%           See \fullref{Section}{sec:classification} for a full account of classification of hyphenations,
%           \fullref{Subsection}{subsec:options-division-dict-files} for further details of \key{division-dict-path}
%           description, and \fullref{Subsection}{subsec:output-filtering} for \key{output-mode}.
%
%     \item To add visual flags for hyphenations to the output PDF, add the package option \keyvalue{flags}{all} or
%           \keyvalue{flags}{non-valid}. See \fullref{Subsection}{subsec:flags} for further details.
% \end{itemize}
% See \fullref{Section}{sec:options} for all the options.
%
%
%
% \section{Classifications of hyphenations}
% \label{sec:classification}
%
% \pkg{lua-list-hyphen} can read use a dictionary of valid divisions to check each hyphenation that occurs during
% typesetting. The user can specify the file containing the dictionary (either globally or per-language) using the
% option \key{division-dict-path} (see \fullref{Subsection}{subsec:options-division-dict-files}).
%
% ^^A Use \symbol{124} here since | is an active character.
% These files should list accepted divisions of words, one per line. Multiple division points can be marked in each word
% (as in the \TeX\ \cs{hyphenation} command and its \pkg{babel} and \pkg{polyglossia} equivalents). A division point can
% be marked with a hyphen \texttt{-}, a vertical bar \texttt{\symbol{124}}, or a broken vertical bar \texttt{¦}. (The
% latter two symbols are allowed to match the notation used in \emph{New Oxford Spelling Dictionary} for preferred and
% acceptable division points.) Whitespace at the start of the line is ignored. Any text following whitespace after the
% accepted division is ignored and can be used for a note, perhaps to clarify different divisions of orthographically
% identical words. Thus lines in a division dictionary for British English might be:
%\iffalse
%<*example>
% \fi
\begin{lstlisting}
  at-tri-bute     (noun)
  at-trib-ute     (verb)
  at-trib-uted
  at-tri-butes    (noun)
  at-trib-utes    (verb)
  at-tri-bu-tion
\end{lstlisting}
% \iffalse
%</example>
% \fi
%
% Each hyphenation is compared to the division dictionary (if specified) for the appropriate language and is
% classified as one of: \defterm{valid}, \defterm{invalid}, \defterm{ambiguous}, or \defterm{unknown}:
% \begin{description}
%
%     \item[valid] A hyphenation is \defterm{valid} if there is an entry in the division dictionary for that word and
%           the chosen hyphenation matches \emph{every} entry for that word. Using the example lines from the division
%           dictionary above, \examplehyph{at-tributed} is valid because it matches the entry \texttt{at-trib-uted}. The
%           hyphenation \examplehyph{at-tribute} is also valid because it matches \emph{both} the entries
%           \texttt{at-trib-ute} and \texttt{at-tri-bute}.
%
%     \item[invalid] A hyphenation is \defterm{invalid} if there is an entry in the division dictionary for that word
%           and the chosen hyphenation matches \emph{no} entry for that word. Using the example lines from the division
%           dictionary above, \examplehyph{attrib-ution} is invalid because it does no match the entry
%           \texttt{at-tri-bu-tion}.
%
%     \item[ambiguous] A hyphenation is \defterm{ambiguous} if there is are at least two entries in the division
%           dictionary for that word and the chosen hyphenation matches at least one entry for that word \emph{and} does
%           not match at least one entry for that word. Using the example lines from the division dictionary above,
%           \examplehyph{attrib-ute} is ambiguous because it matches the entry \texttt{at-trib-ute} and does not match
%           the entry \texttt{at-tri-bute}.
%
%     \item[unknown] A hyphenation is \defterm{unknown} if there is no entry for the word in the division dictionary.
%           (If no division dictionary is supplied for a language, all hyphenations in that language are classified as
%           unknown.)
%
% \end{description}
%
% If \keyvalue{output-mode}{all}, all hyphenations, regardless of classification, are written out. If
% \keyvalue{output-mode}{non-valid}, only hyphenations that have been classified as invalid, ambiguous, or unknown are
% written out. Thus a division dictionary could be used as a simple allowlist: it could simply contain hyphenations that
% have been checked, with \keyvalue{output-mode}{non-valid} meaning that any hyphenations not appearing in the allowlist
% would be written out.
%
% When \keyvalue{output-verbose}{true}, the classification is written to the output file along with the other data about
% the hyphenation.
%
% If \keyvalue{flags}{all}, visual flags are added next to every hyphenation according to its classification: valid
% \directlua{ lualisthyphen.write_flag_valid() }, invalid \directlua{ lualisthyphen.write_flag_invalid() }, ambiguous
% \directlua{ lualisthyphen.write_flag_ambiguous() }, unknown \directlua{ lualisthyphen.write_flag_unknown() }. If
% \keyvalue{flags}{non-valid}, the flags on valid hyphenations are suppressed and only \directlua{
% lualisthyphen.write_flag_invalid() }, \directlua{ lualisthyphen.write_flag_ambiguous() }, and \directlua{
% lualisthyphen.write_flag_unknown() } appear.
%
%
%
% \section{Output format}
%
% Each output files begins with a header (which can be disabled) and then lists the hyphenations for the corresponding
% language, one per line.
%
% If \keyvalue{output-header}{true}, each output file begins with a header (each line of which begins with a ‘comment’
% symbol \texttt{\%}) that includes information about the language and the package options that were used, and the path
% to the division dictionary that was used to classify the words (if specified). \keyvalue{output-header}{false}, this
% header does not appear.
%
% Each line of the remainder of the file describes one hyphenation.
% \begin{itemize}
%     \item When \keyvalue{output-verbose}{false}, the line contains only the hyphenated word.
%
% ^^A Use \symbol{34} here since " is an active character.
%     \item When \keyvalue{output-verbose}{true}, the line contains the original undivided word, the hyphenated word, the
%           classification of the hyphenation (see \fullref{Section}{sec:classification}), the page number where the
%           hyphenated word appears (or, to be precise, begins), and the context in which the hyphenated word appears.
%           Each part of the output is padded so that the various lines align. The original and undivided words are
%           separated by the ASCII ‘arrow’ \texttt{->}; the page number is prefixed by \texttt{p.}; and the context is
%           surrounded by (straight) quotation marks \texttt{\symbol{34}}\kern .5em\texttt{\symbol{34}}. Thus a line of
%           output might be:
%
%           \texttt{thinking~~-> think-ing~~(Valid)~~p.4 \symbol{34}and visual thinking, is its\symbol{34}}
%
%           If no division dictionary was specified for the language, the classification of the hyphenation
%           (\texttt{(Valid)} in this example) does not appear (since all hyphenations are unknown).
% \end{itemize}
%
%
%
% \section{Package options}
% \label{sec:options}
%
% Package options are given as a comma-separated list of \keyvalue{\meta{key}}{\meta{value}} pairs. Most package options
% can be set either \emph{globally}, applying to all languages, or \emph{per-language}, applying only to a specific
% language. Global options apply to all languages unless overrriden for that language.
%
% Two options (\key{output-prefix}, \key{output-extension}) can only be set globally and one (\key{output-path}) can
% only be set per-language. All other options can be set both globally and per-language.
%
% Top-level \keyvalue{\meta{key}}{\meta{value}} options apply globally. Options for a specific language are given in the
% form \keyvalue{\meta{lang-name}}{\{\meta{options}\}}, where \meta{lang-name} can be any of the synonyms for the
% language specified in the \pkg{hyph-utf8} documentation, and \meta{options} is a comma-separated list of
% \keyvalue{\meta{key}}{\meta{value}} pairs.
%
% For example, suppose package options are given as follows:
%\iffalse
%<*example>
% \fi
\begin{lstlisting}
\usepackage[
  output-verbose=true,
  flag-mode=all,
  ukenglish={
    output-verbose=false,
    division-dict-path=division-dict-en-GB.txt,
  },
]{lua-list-hyphen}
\end{lstlisting}
% \iffalse
%</example>
% \fi
%
% Then the option \keyvalue{flag-mode}{all} applies to all languages, including \key{ukenglish}. The option
% \keyvalue{output-verbose}{true} applies to all language except \key{ukenglish}, for which
% \keyvalue{output-verbose}{false} applies. The option \keyvalue{division-dict-path}{division-dict-en-GB.txt} applies
% only  to \key{ukenglish}.
%
%
%
% \subsection{Output files}
% \label{subsec:options-output-files}
%
% The following keys specify the paths to the files where hyphenations are written. If an output directory has been
% specified (using the command-line option \texttt{--output-directory}), paths are relative to this directory.
%
%
%
% \medskip
%
% \DescribeOption{output-path}
% (\emph{Per-language only option.}) \key{output-path} can be used to specify, for a particular language, the file to
% which hyphenations for that language are written. If no \key{output-path} is specified for a language, the fallback
% output file is determined by \key{output-prefix} and \key{output-extension}. \default{\val{None}}
%
% This option cannot be set globally, since the intention is to write each language is written to a different output
% file. If the same \key{output-path} is given for two different languages, and there are hyphenations in both
% languages, one list of hyphenations will overwrite the other.
%
%
%
% \medskip
%
% \DescribeOption{output-prefix}
% \DescribeOption{output-extension}
% (\emph{Global-only options.}) These two keys are used for the fallback output file for a language when no path has
% been specified using \key{output-path}. The fallback is \meta{prefix}\meta{lang}\meta{extension}, where \meta{prefix}
% is the value of \key{output-prefix}, \meta{extension} is the value of \key{output-extension}, and \meta{lang} will
% usually be the name of the language as defined in \pkg{hyph-utf8} (which does not always coincide with the \pkg{babel}
% or \pkg{polyglossia} name). If neither of these packages is in use, or if the language has been defined otherwise,
% \meta{lang} will be the Lua\TeX\ numerical language~ID. \default{\keyvalue{output-prefix}{\cs{jobname}\file{-}} (note
% the hyphen); \keyvalue{output-extension}{\file{.hyph}} (note the dot).}
%
% These options cannot be set per-language, only globally. To alter the output file per-language, use the
% \key{output-path} option.
%
%
%
%
% \subsection{Division dictionary files}
% \label{subsec:options-division-dict-files}
%
% \DescribeOption{division-dict-path}
% The key \key{division-dict-path} can be used to specify the file where \pkg{lua-list-hyphen} looks for a dictionary
% of valid divisions against which to check each hyphenation found during typesetting. \default{\val{None}}
%
% Any path specified using this key is passed to the \pkg{kpathsea} library, so a specified division dictionary can be
% anywhere in the directories it searches (such as the user or system \file{texmf/} hierarchies or paths specified using
% \texttt{TEXINPUTS}).
%
% \DescribeOption{allowlist-path}
% The key \key{allowlist-path} is a synonym for \key{division-dict-path}.
%
%
%
% \subsection{Hyphenation checking}
%
% \DescribeOption{validation-preprocessor}
% The key \key{validation-preprocessor} can be used to specify a Lua function that is applied to each hyphenated word
% before it is checked against the division dictionary. Its value should be the name of a Lua function that takes a word
% --- perhaps including a hyphen --- and returns a possibly modified version. The result is used in place of the
% original when checking against the division dictionary, and may thus affect the classification of hyphenations, but
% does not alter the words written out. \default{\val{None}}
%
% For example, the following function deletes \texttt{’s} from the end of a word:
%\iffalse
%<*example>
% \fi
\begin{lstlisting}
function english_strip_possessives(s)
  return string.gsub(s,'’s$','')
end
\end{lstlisting}
% \iffalse
%</example>
% \fi
% Users may wish to use a function like this for English, so that the division dictionary does not have to contain
% separate entries for possessives of nouns and names. If \keyvalue{validation-preprocessor}{english_strip_possessives}
% is specified then the hyphenation \examplehyph{math-ematician’s} would be changed to \examplehyph{math-ematician} for
% checking against the division dictionary, which could contain \examplehyph{math-em-at-ician} without having to contain
% \examplehyph{math-em-at-ician’s}.
%
% \pkg{lua-list-hyphen} has two built-in Lua functions for this purpose:
% \begin{description}
%     \item[\luafunc{lualisthyphen.preprocessors.identity}] This function simplye returns the word unaltered.
%     \item[\luafunc{lualisthyphen.preprocessors.english_strip_possessives}] The same function as shown above, which
%           removes \texttt{’s} from the end of a word.
% \end{description}
%
% \pkg{lua-list-hyphen} first looks in \luavar{lualisthyphen.preprocessors} for \meta{function}, and, if a function with
% the given name is not found there, in the global context. Thus
% \keyvalue{validation-preprocessor}{english_strip_possessives} specifies the built-in function listed above.
% \luavar{lualisthyphen.preprocessors} can be used a location for further such functions written by the user, but this
% is not required.
%
%
%
% \medskip
%
% \DescribeOption{validation-case-sensitve}
% This boolean key controls whether checking hyphenations against dictionaries of valid divisions is case-sensitive. If
% it is set to \val{true}, then (for example) the hyphenation \examplehyph{Pol-ish} (of Poland) would not match the
% division \examplehyph{pol-ish} (shine). \default{\val{false}}
%
%
%
% \subsection{Flags}
% \label{subsec:flags}
%
% \DescribeOption{flag-mode}
% The option \key{flag-mode} controls whether hyphenations are ‘flagged’ with an adjacent mark, indicating whether they
% are valid \directlua{ lualisthyphen.write_flag_valid() }, invalid \directlua{ lualisthyphen.write_flag_invalid() },
% ambiguous \directlua{ lualisthyphen.write_flag_ambiguous() } (like \examplehyph{record}, which has different valid
% divisions as a noun and as a verb), or unknown \directlua{ lualisthyphen.write_flag_unknown() }, with respect to the
% dictionary of valid divisions for the corresponding language. (If no dictionary is found for the language, all
% hyphenations in that language will be treated as unknown.) Its possible values are:
% \begin{vallist}
%   \item[\val{none}] No flags are added.
%   \item[\val{all}] Flags are shown for all hyphenations.
%   \item[\val{non-valid}] Flags are shown only for non-valid hyphenations (that is, those that classified as invalid,
%   ambiguous, or unknown).
% \end{vallist}
% \default{\val{none}}
%
%
%
% \subsection{Output filtering}
% \label{subsec:output-filtering}
%
% \DescribeOption{output-mode}
% The option \key{output-mode} controls which hyphenations are listed in the output files. Its possible values are:
% \begin{vallist}
%   \item[\val{all}] All hyphenations are listed
%   \item[\val{non-valid}] Only non-valid hyphenations (with respect to the dictionary of valid divisions) are listed
%   (that is, those that are invalid, ambiguous, or unknown).
% \end{vallist}
% \default{\val{none}}%
%
%
%
% \medskip
%
% \DescribeOption{output-non-typeset}
% Boolean option determining whether hyphenated words that are never typeset on the page are listed in the output. (For
% instance, a hyphenated word might occur in text that a package temporarily typesets into a box, measures, and then
% discards.) If \keyvalue{output-verbose}{true}, then \texttt{p.}\meta{page} is replaced by \texttt{<none>} for such.
% hyphenations. Whether these hyphenations are listed is afftected by \key{output-mode}. \default{\val{false}}
%
%
%
% \subsection{Output format}
%
% \DescribeOption{output-header}
% The boolean key \key{output-header} controls whether a header is written to the output file, specifying the language,
% the relevant package options used, and (if relevant) the dictionary of valid divisions. \default{\val{true}}
%
%
%
% \medskip
%
% \DescribeOption{output-verbose}
% The boolean option \key{output-verbose} controls how much information is written to the file about each hyphenation.
% When \val{true}, for each hyphenation, both the undivided original and the divided word are written out, the
% classification of the hyphenation as valid/invalid/ambiguous/unknown, as well as the page number on which the
% hyphenated word appears (or, more precisely, begins) and the undivided word in context (as specified by the
% \key{context} keys; see below). When \val{false}, only the hyphenated word is written. \default{\val{false}}
%
%
%
% \medskip
%
% \DescribeOption{context}
% \DescribeOption{context-before}
% \DescribeOption{context-after}
% Integer options controlling how many words before (\key{context-before}) and after (\key{context-after}) the
% hyphenation are written as context when \keyvalue{output-verbose}{true}. The key \key{context} is simply a shortcut
% for setting \key{context-before} and \key{context-after} to the same value. Only words from the same paragraph are
% written as context. \default{\val{2}}
%
%
%
% \subsection{Output sorting and deduplication}
% \label{subsec:options-sorting-deduplication}
%
% \DescribeOption{unique}
% The option \key{unique} controls removal of duplicates from the list of hyphenated words written out. It can be be
% set to one of the following three values:
% \begin{vallist}
%   \item[\val{none}] Duplicate hyphenations are not removed.
%   \item[\val{case}] Hyphenations that are duplicate (case-sensitively) are removed. In this case, the hyphenations
%     \texttt{geo-metry} and \texttt{Geo-metry} are considered to be distinct.
%   \item[\val{nocase}] Hyphenations that are duplicate (case-insensitively) are removed. In this case, the hyphenations
%     \texttt{geo-metry} and \texttt{Geo-metry} are considered to be duplicates. The case of each listed hyphenation
%     will be that of the first appearance of that hyphenation.
% \end{vallist}
% Note that removal of duplicates is unaffected by the page number or context that is written out when
% \keyvalue{output-verbose}{true}. \default{\val{none}}
%
%
%
% \medskip
%
% \DescribeOption{sort}
% The option \key{sort} controls sorting of the list of hyphenated words. It can be be
% set to one of the following three values:
% \begin{vallist}
%   \item[\val{none}] Hyphenations appear in the same order as the occur in the document, or, if duplicates are removed,
%     in the order of first appearance in the document.
%     \item[\val{case}] Hyphenations are sorted case-sensitively. In this case, \texttt{Geo-metry} precedes
%     \texttt{geo-meter}.
%   \item[\val{nocase}] Hyphenations are sorted case-insensitively. In this case, \texttt{geo-meter} precedes
%     \texttt{Geo-metry}.
% \end{vallist}
% \default{\val{none}}
%
%
%
% \section{Usage notes}
%
% \subsection{Languages}
%
% To determine the language of a word, \pkg{lua-list-hyphen} looks at what language is applied at the first possible
% hyphenation point, first considering the part of the word before it, then the part after it. In the (presumably rare)
% case of a ‘mixed-language’ word like ‘near-Zugzwang’ being specified (using, for example, \pkg{babel}) with
% \texttt{near-\cs{foreignlanguage}\{german\}\{Zugzwang\}}, it would be assigned to the language in which \hbox{‘near-’}
% is set.
%
% Duplicates are removed within each language. If the same hyphenation occurs in two different languages, it will appear
% in both files, regardless of the value of the \key{unique} package option.
%
%
%
% \subsection{Limitations}
%
% \subsubsection{Unicode}
%
% \pkg{lua-list-hyphen} uses Lua\TeX's built-in Unicode functions for pattern matching and converting between upper and
% lower case. These function are based on the \texttt{slnunicode} library. This library has not been updated for some
% time and is based on an out-of-date version of the Unicode standard. Thus there may be problems with languages added
% to Unicode more recently. Hyphenated words from such languages should still be listed, but may contain extraneous
% characters (such as adjacent punctuation) and may not be classified, sorted, or deduplicated correctly. Users may
% prefer to leave these tasks to an external program that adheres to the current Unicode standard. (The author's
% \texttt{hyphenassist} program, mentioned on page~\ref{page:hyphenassist} is as up-to-date as the Python installation
% on which it runs.)
%
%
%
% \subsubsection{Association of language names and numerical Lua\TeX\ IDs}
%
% The \pkg{lua-list-hyphen} only interfaces with \pkg{babel} or \pkg{polyglossia} to determine the \pkg{hyph-utf8} name
% for the language (in the sense of a set of hyphenation patterns) that has been assigned to each Lua\TeX\ numerical
% language ID. The \pkg{hyph-utf8} name does not necessarily match the \pkg{babel} or \pkg{polyglossia} name or variant:
% for instance, for British English, \pkg{polyglossia} uses the language \texttt{english} and the variant
% \texttt{british}, while \pkg{hyph-utf8} uses the name \texttt{ukenglish} with synonyms \texttt{UKenglish} and
% \texttt{british}. (There seems to be no straightforward way of mapping between Lua\TeX\ IDs and specified
% \pkg{babel}/\pkg{polyglossia} names and variants.)
%
%
%
% \clearpage
%
% \section{Appendix: demonstration source}
% \label{sec:full-example}
%
% This following subsections contain the source code and example division dictionary used to produce the demonstration
% on the first page of this document.
%
%
%
% \subsection{Lua\LaTeX\ source}
%
% \lstinputlisting{lua-list-hyphen-doc-demo.tex}
%
%
%
% \subsection{Division dictionary}
%
% (Saved as \file{division-dict-ukenglish.txt}, to match \key{division-dict-paths} in the package options shown in the
% Lua\LaTeX\ source code above.)
%
% \lstinputlisting{division-dict-ukenglish.txt}
%
%
%
% \end{documentation}
%
%
%
% \clearpage
% \begin{implementation}
%
%
%
% \section{Implementation (\LaTeX\ package)}
%
%    \begin{macrocode}
%<*package>
%<@@=lualisthyphen>
%    \end{macrocode}
%
%
%
% \subsection{Initial set-up}
%
% Package identification/version information.
%    \begin{macrocode}
\NeedsTeXFormat{LaTeX2e}[2020-02-02]
\ProvidesExplPackage{lua-list-hyphen}{2026-07-25}{0.42.7}
  {Listing hyphenated words for LuaLaTeX}
%    \end{macrocode}
% Check that Lua\TeX\ is in use.
%    \begin{macrocode}
\sys_if_engine_luatex:F
  {
    \msg_new:nnn{ lua-list-hyphen }{ lualatex_required }
      { LuaLaTeX~required.~Package~loading~will~abort. }
    \msg_critical:nn{ lua-list-hyphen }{ lualatex_required }
  }
%    \end{macrocode}
%
%
%
% \subsection{Options}
%
% \subsubsection{Global-only options}
%
% \begin{macro}{\l_@@_output_prefix_str}
%   String option for the prefix of files to which hyphenations are written.
%    \begin{macrocode}
\keys_define:nn { lua-list-hyphen }{
  output-prefix .str_set:N = \l_@@_output_prefix_str,
  output-prefix .initial:e = { \c_sys_jobname_str- },
}
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}{\l_@@_output_extension_str}
%   String option for the extension of files to which hyphenations are written.
%    \begin{macrocode}
\keys_define:nn { lua-list-hyphen }{
  output-extension .str_set:N = \l_@@_output_extension_str,
  output-extension .initial:n = { .hyph },
}
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsubsection{Per-language and global options}
%
% \begin{macro}{
%   \l_@@_division_dict_path_str
% }
%   String option to specify the path from which to read the division dictionary.
%    \begin{macrocode}
\keys_define:nn { lua-list-hyphen/global-per-lang }{
  division-dict-path .str_set:N = \l_@@_division_dict_path_str,
}
%    \end{macrocode}
%    \begin{macrocode}
\keys_define:nn { lua-list-hyphen/global-per-lang }{
  allowlist-path .meta:n = { division-dict-path=#1 },
}
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}{
%   \l_@@_validation_preprocessor_str
% }
%   String option to specify validation preprocessor.
%    \begin{macrocode}
\keys_define:nn { lua-list-hyphen/global-per-lang }{
  validation-preprocessor .str_set:N = \l_@@_validation_preprocessor_str,
}
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}{
%   \l_@@_validation_case_sensitive_bool
% }
%   Boolean option to specify whether validation of divisions is case-sensitive.
%    \begin{macrocode}
\keys_define:nn { lua-list-hyphen/global-per-lang }{
  validation-case-sensitve .bool_set:N
    = \l_@@_validation_case_sensitive_bool
}
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}{
%   \l_@@_flag_mode_int
% }
%   Choice option to specify whether flags are added to the output PDF file to indicate the classification of
%   hyphenations.
%    \begin{macrocode}
\int_new:N\l_@@_flag_mode_int
\keys_define:nn { lua-list-hyphen/global-per-lang }{
  flag-mode .choices:nn = { none, all, non-valid }{
    \int_set:Nn\l_@@_flag_mode_int{ \l_keys_choice_int - 1 }
  },
}
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}{\l_@@_output_mode_int}
%   Choice option to indicate which hyphenations should be listed in the output files.
%    \begin{macrocode}
\int_new:N\l_@@_output_mode_int
\keys_define:nn { lua-list-hyphen/global-per-lang }{
  output-mode .choices:nn = { all, non-valid }{
    \int_set:Nn\l_@@_output_mode_int{ \l_keys_choice_int - 1 }
  },
}
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}{\l_@@_output_non_typeset_bool}
%   Boolean option to indicate whether output lists of hyphenations should include those that are never typeset to the
%   page.
%    \begin{macrocode}
\keys_define:nn { lua-list-hyphen/global-per-lang }{
  output-non-typeset .bool_set:N = \l_@@_output_non_typeset_bool,
}
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}{\l_@@_output_header_bool}
%   Boolean option to indicate whether a header should be written in the output.
%    \begin{macrocode}
\keys_define:nn { lua-list-hyphen/global-per-lang }{
  output-header .bool_set:N = \l_@@_output_header_bool,
  output-header .initial:n = {true},
}
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}{\l_@@_output_verbose_bool}
%   Boolean option to indicate whether output lists of hyphenations should be written verbosely.
%    \begin{macrocode}
\keys_define:nn { lua-list-hyphen/global-per-lang }{
  output-verbose .bool_set:N = \l_@@_output_verbose_bool,
}
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}{\l_@@_context_before_int,\l_@@_context_after_int}
%   Integer options to determine the number of words before and after a hyphenation shown as context in verbose output.
%    \begin{macrocode}
\keys_define:nn { lua-list-hyphen/global-per-lang }{
  context-before .int_set:N = \l_@@_context_before_int,
  context-before .initial:n = { 2 },
  context-after .int_set:N = \l_@@_context_after_int,
  context-after .initial:n = { 2 },
  context .meta:n = {
    context-before=#1,
    context-after=#1,
  },
}
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}{\l_@@_unique_int}
%   Choice option to indicate whether lists of hyphenations should have duplicates removed, case-sensitively or
%   case-insensitively.
%    \begin{macrocode}
\int_new:N\l_@@_unique_int
\keys_define:nn { lua-list-hyphen/global-per-lang }{
  unique .choices:nn = { none, case, nocase }{
    \int_set:Nn\l_@@_unique_int{ \l_keys_choice_int - 1 }
  },
}
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}{\l_@@_sort_int}
%   Choice option to indicate whether lists of hyphenations should be sorted, case-sensitively or case-insensitively.
%    \begin{macrocode}
\int_new:N\l_@@_sort_int
\keys_define:nn { lua-list-hyphen/global-per-lang }{
  sort .choices:nn = { none, case, nocase }{
    \int_set:Nn\l_@@_sort_int{ \l_keys_choice_int - 1 }
  },
}
%    \end{macrocode}
% \end{macro}
%
%
%
% Make options in the \key{global-per-lang} submodule options available at the top level in the \key{lua-list-hyphen}
% module.
%    \begin{macrocode}
\keys_define:nn {  }{
  lua-list-hyphen .inherit:n = lua-list-hyphen/global-per-lang,
}
%    \end{macrocode}
%
%
%
% \subsubsection{Per-language options}
%
% Make options in the \key{global-per-lang} submodule options available in the \key{per-lang} submodule.
%    \begin{macrocode}
\keys_define:nn { lua-list-hyphen }{
   per-lang .inherit:n = lua-list-hyphen/global-per-lang,
}
%    \end{macrocode}
%
%
%
% There is one option that is is strictly per-language:
%
% \begin{macro}{
%   \l_@@_output_path_str
% }
%   Option to specify the path to write hyphenations.
%    \begin{macrocode}
\keys_define:nn { lua-list-hyphen/per-lang }{
  output-path .str_set:N = \l_@@_output_path_str,
}
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsubsection{Warnings for per-language-only and global-only options}
%
% Emit a warning when the per-language-only option \key{output-path} is set globally.
%    \begin{macrocode}
\msg_new:nnnn{ lua-list-hyphen }{ per-language-only }
  { The~option~"#1"~only~has~effect~per-language. }
  { The~option~"#1"~has~been~specified~globally~and~will~be~ignored. }
\keys_define:nn { lua-list-hyphen }{
  output-path .code:n = {
    \msg_warning:nne{ lua-list-hyphen }{ per-language-only }
      { \str_use:N\l_keys_key_str }
  },
}
%    \end{macrocode}
%
%
%
% Emit a warning when either of the global-only options \key{output-prefix} or \key{output-extension} is set
% per-language.
%    \begin{macrocode}
\msg_new:nnnn{ lua-list-hyphen }{ global-only }
  { The~option~"#1"~only~has~effect~globally. }
  { The~option~"#1"~has~been~specified~for~a~particular~language~and~will~be~ignored. }
\keys_define:nn { lua-list-hyphen/per-lang }{
  output-prefix .code:n = {
    \msg_warning:nne{ lua-list-hyphen }{ global-only }
      { \str_use:N\l_keys_key_str }
  },
  output-extension .code:n = {
    \msg_warning:nne{ lua-list-hyphen }{ global-only }
      { \str_use:N\l_keys_key_str }
  },
}
%    \end{macrocode}
%
%
%
% \subsection{Deprecated options}
%
% For deprecated options, emit a warning and forward the setting to the replacement option.
%    \begin{macrocode}
\msg_new:nnnn{ lua-list-hyphen }{ deprecated-option }
  { The~option~"#1"~works~but~is~deprecated.~Please~use~"#2"~instead. }
  { The~option~"#1"~may~be~removed~in~a~future~version. }
\keys_define:nn { lua-list-hyphen }{
  verbose .code:n = {
    \msg_warning:nnee{ lua-list-hyphen }{ deprecated-option }
      { \str_use:N\l_keys_key_str }{ output-verbose }
    \keys_set:nn{ lua-list-hyphen }{ output-verbose=#1 }
  },
  include-non-output .code:n = {
    \msg_warning:nnee{ lua-list-hyphen }{ deprecated-option }
      { \str_use:N\l_keys_key_str }{ include-non-typeset }
    \keys_set:nn{ lua-list-hyphen }{ include-non-typeset=#1 }
  },
  prefix .code:n = {
    \msg_warning:nnee{ lua-list-hyphen }{ deprecated-option }
      { \str_use:N\l_keys_key_str }{ output-prefix }
    \keys_set:nn{ lua-list-hyphen }{ output-prefix=#1 }
  },
  extension .code:n = {
    \msg_warning:nnee{ lua-list-hyphen }{ deprecated-option }
      { \str_use:N\l_keys_key_str }{ output-extension }
    \keys_set:nn{ lua-list-hyphen }{ output-extension=#1 }
  }
}
%    \end{macrocode}
%
%
%
% \subsubsection{Developer option}
%
% \begin{macro}{
%   \l_@@_debug_bool
% }
%   Option to specify whether debug information is written to the terminal. Undocumented and not intended for end-users.
%    \begin{macrocode}
\keys_define:nn { lua-list-hyphen }{
  debug .bool_set:N = \l_@@_debug_bool,
}
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsubsection{Processing per-language options}
%
% Language-specific options should be processed \emph{after} all other options, so that they inherit the ‘general’
% options. \cs{ProcessKeyOptions} does not seem to allow for selective processing similar to \cs{keys_set_known:nn}, so
% store all unknown options in a clist for later processing.
%    \begin{macrocode}
\keys_define:nn { lua-list-hyphen }{
  unknown .code:n = {
    \@@_store_lang_options:en{\str_use:N\l_keys_key_str}{#1}
  }
}
%    \end{macrocode}
% \begin{macro}{
%   \@@_store_lang_options:nn
% }
%   Store the key--value option whose key is \param{1} and whose value is \param{2} in \cs{l_@@_lang_options_clist}.
%    \begin{macrocode}
\clist_new:N\l_@@_lang_options_clist

\cs_new:Npn \@@_store_lang_options:nn #1#2
  {
    \clist_put_right:Nn\l_@@_lang_options_clist{#1={#2}}
  }
\cs_generate_variant:Nn\@@_store_lang_options:nn{ en }
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsection{Lua backend}
%
% Load the Lua backend.
%    \begin{macrocode}
\lua_load_module:n{lua-list-hyphen}
%    \end{macrocode}
%
%
%
% \subsection{Processing package options}
%
% Process the package options. All the unknown options --- in principle, language-specific options --- are stored in
% \cs{l_@@_lang_options_clist}.
%    \begin{macrocode}
\ProcessKeyOptions [ lua-list-hyphen ]
%    \end{macrocode}
%
%
%
% \subsubsection{Interface for passing options to the Lua backend}
%
% \begin{macro}{\@@_call_lua_boolean:nN}
%   Call the Lua function given in \param{1} with the value of the boolean given in \param{2}.
%    \begin{macrocode}
\cs_new:Npn \@@_call_lua_boolean:nN #1#2
  {
    \lua_now:e{ lualisthyphen.#1(\bool_to_str:N #2) }
  }
%    \end{macrocode}
% \end{macro}
%
%
%
%
% \begin{macro}{\@@_luastr_or_nil:N}
%   Leave in the input stream the single-quoted content of a string variable, if it is non-empty, otherwise
%   \luavar{nil}.
%    \begin{macrocode}
\cs_new:Npn \@@_luastr_or_nil:N #1
  {
    \str_if_empty:NTF #1
      { nil }
      { '\str_use:N #1' }
  }
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}{
%   \@@_lua_set_global_option:nn,\@@_lua_set_global_option:ne
% }
%   pass a global option to the Lua backend. \param{1} is the name for the option in the Lua table in the backend.
%   \param{2} is the value as a literal string, boolean, integer, or \luavar{nil}. In particular, strings literals
%   passed as \param{2} must be quoted.
%    \begin{macrocode}
\cs_new:Npn \@@_lua_set_global_option:nn #1#2
  {
    \lua_now:n{ lualisthyphen.set_global_option('#1',#2) }
  }
\cs_generate_variant:Nn\@@_lua_set_global_option:nn{ ne }
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}{
%   \@@_lua_set_lang_option:nnn,\@@_lua_set_lang_option:nne
% }
%   Pass a language-specific option to the Lua backend. \param{1} should be a literal Lua string with the language name.
%   \param{2} is the name for the option in the Lua table in the backend. \param{3} is the value as a literal string,
%   boolean, or integer. In particular, strings literals passed as \param{1} or \param{3} must be quoted.
%    \begin{macrocode}
\cs_new:Npn \@@_lua_set_lang_option:nnn #1#2#3
  {
    \lua_now:n{ lualisthyphen.set_lang_option(#1,'#2',#3) }
  }
\cs_generate_variant:Nn\@@_lua_set_lang_option:nnn{ nne }
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsubsection{Passing global options to the Lua backend}
%
% First of all, pass the \key{debug} setting to the Lua backend.
%    \begin{macrocode}
\@@_call_lua_boolean:nN
  {set_debug}\l_@@_debug_bool
%    \end{macrocode}
%
%
%
% At this point the variables contain ‘global’ options.
%    \begin{macrocode}
\@@_lua_set_global_option:ne{division-dict-path}
  {\@@_luastr_or_nil:N\l_@@_division_dict_path_str}
\@@_lua_set_global_option:ne{output-prefix}
  {\@@_luastr_or_nil:N\l_@@_output_prefix_str}
\@@_lua_set_global_option:ne{output-extension}
  {\@@_luastr_or_nil:N\l_@@_output_extension_str}
\@@_lua_set_global_option:ne{validation-preprocessor}
  {\@@_luastr_or_nil:N\l_@@_validation_preprocessor_str}
\@@_lua_set_global_option:ne{validation-case-sensitive}
  {\bool_to_str:N\l_@@_validation_case_sensitive_bool}
\@@_lua_set_global_option:ne{flag-mode}
  {\int_use:N\l_@@_flag_mode_int}
\@@_lua_set_global_option:ne{output-mode}
  {\int_use:N\l_@@_output_mode_int}
\@@_lua_set_global_option:ne{output-non-typeset}
  {\bool_to_str:N\l_@@_output_non_typeset_bool}
\@@_lua_set_global_option:ne{output-header}
  {\bool_to_str:N\l_@@_output_header_bool}
\@@_lua_set_global_option:ne{output-verbose}
  {\bool_to_str:N\l_@@_output_verbose_bool}
\@@_lua_set_global_option:ne{context-before}
  {\int_use:N\l_@@_context_before_int}
\@@_lua_set_global_option:ne{context-after}
  {\int_use:N\l_@@_context_after_int}
\@@_lua_set_global_option:ne{unique}
  {\int_use:N\l_@@_unique_int}
\@@_lua_set_global_option:ne{sort}
  {\int_use:N\l_@@_sort_int}
%    \end{macrocode}
%
%
%
% \subsubsection{Passing per-language options to the Lua backend}
%
% \begin{macro}{
%   \@@_lua_set_per_lang_options:n
% }
%   Pass all options for a given language to the Lua backend. \param{1} should be any name of the language as a (quoted)
%   Lua string literal.
%    \begin{macrocode}
\cs_new:Npn \@@_lua_set_per_lang_options:n #1
  {
    \@@_lua_set_lang_option:nne{#1}{division-dict-path}
      {\@@_luastr_or_nil:N\l_@@_division_dict_path_str}
    \@@_lua_set_lang_option:nne{#1}{output-path}
      {\@@_luastr_or_nil:N\l_@@_output_path_str}
    \@@_lua_set_lang_option:nne{#1}{validation-preprocessor}
      {\@@_luastr_or_nil:N\l_@@_validation_preprocessor_str}
    \@@_lua_set_lang_option:nne{#1}{validation-case-sensitive}
      {\bool_to_str:N\l_@@_validation_case_sensitive_bool}
    \@@_lua_set_lang_option:nne{#1}{flag-mode}
      {\int_use:N\l_@@_flag_mode_int}
    \@@_lua_set_lang_option:nne{#1}{output-mode}
      {\int_use:N\l_@@_output_mode_int}
    \@@_lua_set_lang_option:nne{#1}{output-non-typeset}
      {\bool_to_str:N\l_@@_output_non_typeset_bool}
    \@@_lua_set_lang_option:nne{#1}{output-header}
      {\bool_to_str:N\l_@@_output_header_bool}
    \@@_lua_set_lang_option:nne{#1}{output-verbose}
      {\bool_to_str:N\l_@@_output_verbose_bool}
    \@@_lua_set_lang_option:nne{#1}{context-before}
      {\int_use:N\l_@@_context_before_int}
    \@@_lua_set_lang_option:nne{#1}{context-after}
      {\int_use:N\l_@@_context_after_int}
    \@@_lua_set_lang_option:nne{#1}{unique}
      {\int_use:N\l_@@_unique_int}
    \@@_lua_set_lang_option:nne{#1}{sort}
      {\int_use:N\l_@@_sort_int}
  }
\cs_generate_variant:Nn\@@_lua_set_per_lang_options:n{ e }
%    \end{macrocode}
% \end{macro}
%
%
%
% Define a new module that only handles unknown keys, in order to process the stored per-language options. Each unknown
% key is saved as \cs{l_@@_lang_name_str} and its value is used to set \key{lua-list-hyphen/per-lang}. The handling of
% this value takes place inside a group, so the variables defined for the options are set locally and passed to the Lua
% backend.
%    \begin{macrocode}
\str_new:N\l_@@_lang_name_str
\keys_define:nn { lua-list-hyphen/per-lang-processing }{
  unknown .code:n = {
    \group_begin:

    \str_set_eq:NN\l_@@_lang_name_str\l_keys_key_str
    \keys_set:nn{ lua-list-hyphen/per-lang }{ #1 }

    \@@_lua_set_per_lang_options:e
      {'\str_use:N\l_@@_lang_name_str'}

    \group_end:
  }
}
%    \end{macrocode}
% Use this new module to process the stored language-specific options.
%    \begin{macrocode}
\keys_set:ne{ lua-list-hyphen/per-lang-processing }
  { \clist_use:N\l_@@_lang_options_clist }
%    \end{macrocode}
%
%
%
% \subsection{Initialization}
%
% At \texttt{begindocument}, run initialization code.
%    \begin{macrocode}
\hook_gput_code:nnn{ begindocument }{ lua-list-hyphen } {
  \@@_initialize:
}
%    \end{macrocode}
%
% \begin{macro}{\@@_initialize:}
%   Store \pkg{babel}'s language names if it is in use. (When \pkg{polyglossia} is in use, the association of names and
%   language IDs will be set on the fly.)
%    \begin{macrocode}
\cs_new:Npn \@@_initialize:
  {
    \@@_babel_store_lang_id_names:
  }
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}{\@@_babel_store_lang_id_names:}
%   If \pkg{babel} is in use, get language names from \cs{bbl@languages}. Items stored in this macro are quadruples
%   prefixed with \cs{bbl@elt}, so locally redefine this latter macro to an auxiliary function that passes language
%   ID/name pairs to the Lua backend.
%    \begin{macrocode}
\cs_new:Npn \@@_babel_store_lang_id_names:
  {
    \cs_if_exist:NT\bbl@languages
      {
        \group_begin:
        \cs_set_eq:NN
          \bbl@elt
          \@@_babel_store_lang_id_names_elt:nnnn
        \bbl@languages
        \group_end:
      }%
  }
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}{\@@_babel_store_lang_id_names_elt:nnnn}
%   Auxiliary function that takes a quadruple stored in \cs{bbl@languages} and passes a language ID/name pair to the Lua
%   backend.
%    \begin{macrocode}
\cs_new:Npn \@@_babel_store_lang_id_names_elt:nnnn #1#2#3#4
  {
    \lua_now:n{
      lualisthyphen.store_lang_id_name(#2,'#1')
    }
  }
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsection{Processing and writing hyphenation lists}
%
% At \texttt{enddocument/info}, process and output the hyphenations that have been found.
%    \begin{macrocode}
\hook_gput_code:nnn{ enddocument/info }{ lua-list-hyphen } {
  \lua_now:n{
    lualisthyphen.process_write_hyphenation_lists()
  }
}
%    \end{macrocode}
%
%
%
%    \begin{macrocode}
%</package>
%    \end{macrocode}
%
%
%
% \section{Implementation (Lua backend)}
%
%    \begin{macrocode}
%<*lua>
%    \end{macrocode}
%
%
%
% \subsection{Initial set-up}
%
% Module identification/version information.
%    \begin{macrocode}
local module_name = 'lualisthyphen'
local lualisthyphen_module = {
  name        = module_name,
  version     = '0.42.7',
  date        = '2026-07-25',
  description = 'lua-list-hyphen',
  author      = 'Alan J. Cain',
  copyright   = 'Alan J. Cain',
  license     = 'LPPL v1.3c'
}
luatexbase.provides_module(lualisthyphen_module)
%    \end{macrocode}
%
%
%
% \subsection{Message functions}
%
% \begin{macro}[int]{info,warning}
%   Functions for writing info or warning messages. First argument is used as a format string for later arguments.
%    \begin{macrocode}
local function info(s,...)
  luatexbase.module_info(module_name,s:format(...))
end
local function warning(s,...)
  luatexbase.module_warning(module_name,s:format(...))
end
local function error_(s,...)
  luatexbase.module_error(module_name,s:format(...))
end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{debug,debug_dummy,debug_real}
%   Debugging function. \luafunc{debug_dummy} and \luafunc{debug_real} respectively do nothing and write debugging
%   information. \luafunc{debug_real}’s uses its parameters in the same way as \luafunc{info} and \luafunc{warning}.
%   \luafunc{debug} is set equal to the former and can be changed using \luafunc{set_debug}.
%    \begin{macrocode}
local function debug_dummy(s,...)
end

local function debug_real(s,...)
  print(module_name .. ' DEBUG: ' .. s:format(...))
end

local debug = debug_dummy
local debug_bool = false
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{set_debug}
%   If the parameter \luavar{a} is \luavar{true}, set \luafunc{debug} to be \luafunc{debug_real}, otherwise set it to
%   \luafunc{debug_dummy}.
%    \begin{macrocode}
local function set_debug(a)
  if a then
    debug = debug_real
  else
    debug = debug_dummy
  end
  debug_bool = a
end
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsection{Node ID and subtype constants}
%
% Define constants for the node IDs that need to be recognized.
%    \begin{macrocode}
local NODE_ID_HLIST = node.id('hlist')
local NODE_ID_DISC = node.id('disc')
local NODE_ID_GLUE = node.id('glue')
local NODE_ID_KERN = node.id('kern')
local NODE_ID_MARGIN_KERN = node.id('margin_kern')
local NODE_ID_GLYPH = node.id('glyph')
local NODE_ID_MATH = node.id('math')
%    \end{macrocode}
% Define constants for the kern node subtypes that have to be recognized. (There seems to be no automatic way to get
% the numerical value from the subtype text other than searching the \luavar{node.subtype(\meta{node type})} tables.)
%    \begin{macrocode}
local NODE_KERN_SUBTYPE_FONTKERN
local NODE_KERN_SUBTYPE_USERKERN
for k,v in pairs(node.subtypes('kern')) do
  if v == 'fontkern' then
    NODE_KERN_SUBTYPE_FONTKERN = k
  elseif v == 'userkern' then
    NODE_KERN_SUBTYPE_USERKERN = k
  end
end
%    \end{macrocode}
% Define constants for the math node subtypes.
%    \begin{macrocode}
local NODE_MATH_SUBTYPE_BEGIN
local NODE_MATH_SUBTYPE_END
for k,v in pairs(node.subtypes('math')) do
  if v == 'beginmath' then
    NODE_MATH_SUBTYPE_BEGIN = k
  elseif v == 'endmath' then
    NODE_MATH_SUBTYPE_END = k
  end
end
%    \end{macrocode}
%
%
%
% \subsection{List utility functions}
%
% \begin{macro}[int]{list_filter}
%   Take a list \luavar{t} and remove from it any elements for which the function
%   \luavar{f} does not return true. (The index \luavar{j} is always the destination index to which a ‘keep’ element
%   is moved.)\footnote{Code adapted from \url{https://stackoverflow.com/a/53038524}.}
%    \begin{macrocode}
local function list_filter(t, f)
  local j = 1
  local n = #t

  for i=1,n do
    if (f(t[i])) then
      if (i ~= j) then
        t[j] = t[i]
        t[i] = nil
      end
      j = j + 1
    else
      t[i] = nil
    end
  end

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{list_uniq}
%   Take a list \luavar{t} and remove from it any element for which the function \luavar{f} returns \luavar{true} when
%   called with the last ‘kept’ element (which always has index \luavar{j}) and the element being considered.
%    \begin{macrocode}
local function list_uniq(t, f)
  local j = 1
  local n = #t

  for i=2,n do
    if (f(t[i],t[j])) then
      t[i] = nil
    else
      j = i
    end
  end

  list_filter(
    t,
    function(a) return a end
  )
end
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsection{String manipulation functions}
%
% String constants (which will ultimately be output).
%    \begin{macrocode}
local STR_MATH = '[MATH]'
local STR_SPACE = ' '
local STR_SPACE_TWO = '  '
local STR_ARROW = ' -> '
local STR_PAGE_PREFIX = 'p.'
local STR_PAGE_NONE = '<none>'
local STR_QUOTE_OPEN = '"'
local STR_QUOTE_CLOSE = '"'
local STR_VALID = 'Valid'
local STR_INVALID = 'Inva.'
local STR_AMBIGUOUS = 'Ambi.'
local STR_UNKNOWN = 'Unkn.'
%    \end{macrocode}
%
%
%
% \begin{macro}[int]{trim_whitespace_both}
%   Remove characters other than letters and hyphens from both the start and end of a string.
%    \begin{macrocode}
local function trim_whitespace_both(s)

  return unicode.utf8.match(s,'^%s*(.-)%s*$')

end
%    \end{macrocode}
% \end{macro}
%
%
%
%
% \begin{macro}[int]{trim_whitespace_left}
%   Remove characters other than letters and hyphens from the start of a string.
%    \begin{macrocode}
local function trim_whitespace_left(s)

  return unicode.utf8.match(s,'^%s*(.*)$')

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{trim_nonlettershyphens_both}
%   Remove characters other than letters and hyphens from both the start and end of a string.
%    \begin{macrocode}
local function trim_nonlettershyphens_both(s)

  return unicode.utf8.match(s,'^[^%a-]*(.-)[^%a-]*$')

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{trim_nonlettershyphens_start}
%   Remove characters other than letters and hyphens from the start of a string.
%    \begin{macrocode}
local function trim_nonlettershyphens_start(s)

  return unicode.utf8.match(s,'^[^%a-]*(.-)$')

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{trim_nonlettershyphens_end}
%   Remove characters other than letters and hyphens from the end of a string.
%    \begin{macrocode}
local function trim_nonlettershyphens_end(s)

  return unicode.utf8.match(s,'^(.-)[^%a-]*$')

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{rpad}
%   Return string \luavar{s} padded on the right with spaces to length \luavar{n}.
%    \begin{macrocode}
local function rpad(s,n)

   return s .. unicode.utf8.rep(STR_SPACE,n - unicode.utf8.len(s))

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{lpad}
%   Return string \luavar{s} padded on the left with spaces to length \luavar{n}.
%    \begin{macrocode}
local function lpad(s,n)

   return unicode.utf8.rep(STR_SPACE,n - unicode.utf8.len(s)) .. s

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{parenthesize}
%   Return string \luavar{s} enclosed in parentheses.
%    \begin{macrocode}
local function parenthesize(s)

  return '(' .. s .. ')'

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsection{Hyphen utility functions}
%
% \begin{macro}[int]{delete_hyphens}
%   Delete all hyphens.
%    \begin{macrocode}
local function delete_hyphens(s)

  return unicode.utf8.gsub(s,'-','')

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{uniformize_hyphens}
%   Make all appearances of \texttt{\symbol{124}} and \texttt{¦} into hyphens.
%    \begin{macrocode}
local function uniformize_hyphens(s)

  return unicode.utf8.gsub(unicode.utf8.gsub(s,'|','-'),'¦','-')

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{hyphenation_matches_pattern}
%   Test that each hyphen in \luavar{hyphenation} also occurs in \luavar{pattern}.
%    \begin{macrocode}
local function hyphenation_matches_pattern(hyphenation,pattern)

  debug('    Testing "%s" against "%s"',hyphenation,pattern)
%    \end{macrocode}
%   \luavar{i} is the ‘current’ position in \luavar{hyphenation}, just as \luavar{j} is the ‘current’ position in
%   \luavar{pattern}. The ‘current’ characters are respectively \luavar{s} and \luavar{t}. If they are equal, both
%   \luavar{i} and \luavar{j} are incremented. If they are unequal and \luavar{s} is a hyphen, then the there is a
%   mismatch in the hyphenation. If they are unequal and \luavar{t} is not a hyphen, then there is a mismatch as words
%   (which should never happen). Otherwise they are unequal and \luavar{t} is a hyphen that does not appear in
%   \luavar{hyphenation}, and so only \luavar{j} must be incremented.
%    \begin{macrocode}
  local i = 1
  for j=1,#pattern do

    local s
    local t

    s = string.sub(hyphenation,i,i)
    t = string.sub(pattern,j,j)

    if s == t then
      i = i + 1
    elseif s == '-' or t ~= '-' then
      debug('      Fails at i=%d,s=%s,j=%d,t=%s',i,s,j,t)
      return false
    end

  end

  debug('      Matches')
  return true

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsection{Language name tables and utility functions}
%
% The following tables map (respectively) each name for a language to its canonical name (or \defterm{cname}), and
% each cname to a list of names.
%    \begin{macrocode}
local lang_name_cname_table = {}
local lang_cname_name_list_table = {}
%    \end{macrocode}
% Populate these tables from \file{language.dat.lua} (which means that they will contain languages not in use).
%    \begin{macrocode}
for k,v in pairs(require('language.dat.lua')) do
  local t = { k }
  lang_cname_name_list_table[k] = t
  lang_name_cname_table[k] = k
  for _,vv in pairs(v.synonyms) do
    lang_name_cname_table[vv] = k
    table.insert(t,vv)
  end
end

if debug_bool then
  debug('Language name -> cname:')
  for k,v in pairs(lang_name_cname_table) do
    debug('  %s -> %s',k,v)
  end
  debug('Language cname -> name list:')
  for k,v in pairs(lang_cname_name_list_table) do
    local s = ''
    for _,vv in ipairs(v) do
      s = s .. vv .. ','
    end
    debug('  %s -> %s ',k,s)
  end
end
%    \end{macrocode}
%
%
%
% The following tables map (respectively) each language ID to its cname and each cname to the language ID.
%    \begin{macrocode}
local lang_id_cname_table = {}
local lang_cname_id_table = {}
%    \end{macrocode}
%
%
%
% Populating these tables is done differently for \pkg{babel} and \pkg{polyglossia}. If \pkg{babel} is in use, the
% \LaTeX\ frontend iterates through \cs{bbl@languages} and calls \luafunc{store_lang_id_name}. for each item If
% \pkg{polyglossia} is in use, \luavar{lang_id_name_list_table} is populated on the fly.
%
% \begin{macro}[int]{store_lang_id_name}
%   Store the association of a language ID to the cname of the language with the given name (which may be the cname
%   itself or a synonym).
%    \begin{macrocode}
local function store_lang_id_name(lang_id,name)

  local cname = lang_name_cname_table[name]
%    \end{macrocode}
%   The supplied name might not be in \luavar{lang_name_cname_table} (since things like \texttt{dumylang} are not in
%   \file{language.dat.lua}).
%    \begin{macrocode}
  if cname == nil then
    cname = name
    lang_name_cname_table[name] = cname
    lang_cname_name_list_table[cname] = { name }
  end

  local lang_id_cname = lang_id_cname_table[lang_id]
  if lang_id_cname == cname then
    debug(
      'Language name "%s" is a synonym for cname "%s" (lang. ID %s)',
      name,cname,lang_id
    )
    return
  elseif lang_id_cname ~= nil then
    error_(
      'Languages with canonical names "%s" and "%s" both have ID %s',
      cname,lang_id_cname,lang_id
    )
    return
  end

  lang_id_cname_table[lang_id] = cname
  lang_cname_id_table[cname] = lang_id

  debug('Language ID %s has cname "%s"',lang_id,cname)

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{polyglossia_store_lang_id_name}
%   If \pkg{polyglossia} has been loaded, use it to store the association of the given language ID to the cname of the
%   language.
%    \begin{macrocode}
local function polyglossia_store_lang_id_name(lang_id)

  if not polyglossia then
    return
  end

  for name,language in pairs(polyglossia.newloader_loaded_languages) do
    local this_lang_id = lang.id(language)
    if this_lang_id == lang_id then
      store_lang_id_name(lang_id,name)
    end
  end

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsection{Language options}
%
% The following table maps cnames to tables of options for languages. It will be populated via
% \luafunc{set_lang_options} when the package options are processed.
%    \begin{macrocode}
local lang_cname_options_table = {}
%    \end{macrocode}
% The following table maps language IDs to tables of options for languages, and \luavar{LANG_DEFAULT} to a set of
% default options. The table of default options will be created and populated when package options are processed. The
% options for other languages will be added from \luavar{lang_cname_options_table} the first time an option for each
% language is needed, at which time the association of language IDs to cnames will be knowns.
%    \begin{macrocode}
local lang_id_options_table = {}
local LANG_DEFAULT = -1
%    \end{macrocode}
%
%
%
% \begin{macro}[int]{get_lang_option}
%   Return the value associated to \luavar{key} for the language with the given ID, or value for
%   \luavar{LANG_DEFAULT} if no per-language options have been specified for the given. This macro should only be
%   called when \luavar{lang_id_cname_table} is known to have entry for \luavar{lang_id}, which means \emph{after}
%   \luafunc{get_lang_division_dict_validation_preprocessor} has been called during post-linebreak processing.
%    \begin{macrocode}
local function get_lang_option(lang_id,key)
%    \end{macrocode}
%   First look for the language's options table in \luavar{lang_id_options_table}. This will always be set after the
%   first call to this function with this \luavar{lang_id}.
%    \begin{macrocode}
  local options = lang_id_options_table[lang_id]

  if not options then
%    \end{macrocode}
%   If no options table was found, trying lookup a cname for the language and then looking in
%   \luavar{lang_name_cname_table}, where options set from the \LaTeX\ frontend will be found.
%    \begin{macrocode}
    local cname = lang_id_cname_table[lang_id]
    if cname then
      options = lang_cname_options_table[cname]
    end
%    \end{macrocode}
%   If still no options table was found, make a copy of the default options table.
%    \begin{macrocode}
    if not options then
      options = {}
      for k,v in pairs(lang_id_options_table[LANG_DEFAULT]) do
        options[k] = v
      end
    end
    lang_id_options_table[lang_id] = options
  end

  return options[key]

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{set_global_option}
%   Set the global value of \luavar{key} to the given \val{value}. That is, set
%   \luavar{lang_id_options_table[LANG_DEFAULT][key]} to \val{value}.
%    \begin{macrocode}
local function set_global_option(key,value)

  local options = lang_id_options_table[LANG_DEFAULT]
  if not options then
    options = {}
    lang_id_options_table[LANG_DEFAULT] = options
  end

  options[key] = value
  debug('Set global option %s="%s"',key,value)

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{set_lang_option}
%   Set the global value of \luavar{key} to the given \val{value}. That is, set
%   \luavar{lang_cname_options_table[cname][key]} to \val{value}, where \luavar{cname} is the cname of the language
%   with the given \luavar{name}.
%    \begin{macrocode}
local function set_lang_option(name,key,value)

  local cname = lang_name_cname_table[name]
  if not cname then
    cname = name
  end

  local options = lang_cname_options_table[cname]
  if not options then
    options = {}
    lang_cname_options_table[cname] = options
  end

  options[key] = value
  debug('Set lang. "%s" option %s="%s"',cname,key,value)

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsection{Division dictionaries}
%
% The following table maps each language ID to a dictionary (possibly empty) of divisions for that language. Each
% such table maps a word to the same word with division points marked by \texttt{-}, \texttt{\symbol{124}}, or
% \texttt{¦}.
%    \begin{macrocode}
local lang_id_division_dict_table = {}
%    \end{macrocode}
%
% The following table maps each language ID to \luavar{nil}/\luavar{true}/\luavar{false} indicating, respectively, no
% attempt to read a division dictionary/read successfully/failed to read.
%    \begin{macrocode}
local lang_id_division_dict_read_table = {}
%    \end{macrocode}
%
% The following table maps language IDs to the full path (after \pkg{kpathsea} lookup) from which the division
% dictionary is read.
%    \begin{macrocode}
local lang_id_division_dict_path_table = {}
%    \end{macrocode}
%
%
%
% \begin{macro}[int]{load_division_dict}
%   Load the dictionary of valid divisions for a given language ID from a given path, if it points to a file.
%   Return a table containing the division dictionary if the file exists and \luavar{nil} if it does not. Unless
%   \luavar{case_sensitive} is \luavar{true}, convert the loaded divisions to lower case.
%    \begin{macrocode}
local function load_division_dict(path,case_sensitive)

  local count = 0

  local f = io.open(path,'r')
%    \en{macrocode}
%   If the file was not found, return immediately.
%    \begin{macrocode}
  if f == nil then
    return nil
  end

  debug('  Loading divisions from "%s"',path)

  local division_dict = {}

  io.input(f)

  for line in io.lines() do

    line = trim_whitespace_left(line)
%    \en{macrocode}
%   Lines in which the first non-whitespace character is \texttt{\%} are comments.
%    \begin{macrocode}
    if #line == 0 or string.sub(line,1,1) == '%' then
      goto continue
    end

    count = count + 1
%    \end{macrocode}
%   The pattern is between the end of leading whitespace and the next piece of whitespace. (Anything after that can be
%   a note etc.)
%    \begin{macrocode}
    local pattern = unicode.utf8.match(line,'^(.-)%s') or line

    pattern = uniformize_hyphens(pattern)

    if not case_sensitive then
      pattern = unicode.utf8.lower(pattern)
    end

    local word = delete_hyphens(pattern)

    local divisions = division_dict[word]
    if divisions == nil then
      divisions = {}
      division_dict[word] = divisions
    end

    table.insert(divisions,pattern)

    ::continue::
  end

  io.close(f)

  debug('Read %s divisions from "%s"',count,path)

  return division_dict

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsection{Validation preprocessors}
%
% The following table maps a language ID to a functions that is applied to each hyphenated word found for that
% language, before it is validated. It will be populated when the first hyphenation in each language is encountered,
% at the same time as the division dictionary for that language is set.
%    \begin{macrocode}
local lang_id_validation_preprocessor_table = {}
%    \end{macrocode}
%
%
%
% Table to hold preprocessors. When options are processed, this table will be checked first for an the function,
% then the global environment.
%    \begin{macrocode}
local preprocessors = {}
%    \end{macrocode}
%
%
%
% \begin{macro}[int]{identity}
%   A function just returning its parameter, used as a preprocessor when the user has not set one.
%    \begin{macrocode}
local function identity(s)
  return s
end
preprocessors.identity = identity
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{english_strip_possessives}
%   A function removing a possessive ’s from the end of a word.
%    \begin{macrocode}
local function english_strip_possessives(s)
  return string.gsub(s,'’s$','')
end
preprocessors.english_strip_possessives = english_strip_possessives
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsection{Getting the division dictionary and validation preprocessor for a language}
%
% \begin{macro}[int]{get_lang_division_dict_validation_preprocessor}
%   Return the division dictionary and word preprocessor for a given language ID, loading the dictionary if
%   necessary and caching the results. If no division dictionary has been specified for this language, this part of
%   the result is empty dictionary. If no preprocessor has been specified for this languages, this part of the result
%   is the identity function. In the case of \pkg{polyglossia}, also store the association of the language ID to the
%   cname.
%    \begin{macrocode}
local function get_lang_division_dict_validation_preprocessor(lang_id)
%    \end{macrocode}
%   If \luavar{lang_id_division_dict_table} already has an entry for the given language ID, just return it and the
%   preprocessor. Both \luavar{lang_id_division_dict_table} and \luavar{lang_id_validation_preprocessor_table} are set in
%   this function and will be ‘in sync’.
%    \begin{macrocode}
  local division_dict = lang_id_division_dict_table[lang_id]
  if division_dict ~= nil then
    return division_dict,lang_id_validation_preprocessor_table[lang_id]
  end
%    \end{macrocode}
%   Otherwise, it is necessary load a division dictionary. If \pkg{polyglossia} is in use, it is also first of all
%   necessary to set the value in\luavar{lang_id_cname_list_table} for this language ID.
%    \begin{macrocode}
  polyglossia_store_lang_id_name(lang_id)
%    \end{macrocode}
%   See if there is a user-specified word preprocessor. If not, use \luafunc{identity}. In either case, store it it in
%   \luavar{lang_id_validation_preprocessor_table}.
%    \begin{macrocode}
  local cname = lang_id_cname_table[lang_id]
  local preprocessor_name = get_lang_option(lang_id,'validation-preprocessor')
  local preprocessor
  if not preprocessor_name then
    debug(
      'No preprocessor specified for "%s"',cname
    )
    preprocessor = identity
  else
    preprocessor = preprocessors[preprocessor_name]
    if not preprocessor then
       preprocessor = _G[preprocessor_name]
    end
    if not preprocessor then
      debug(
        'Preprocessor specified for "%s" was not found',cname
      )
      preprocessor = identity
    else
      debug(
        'Using preprocessor specified for "%s"',cname
      )
    end
  end
  lang_id_validation_preprocessor_table[lang_id] = preprocessor
%    \end{macrocode}
%   Similarly see if there is a user-specified path for the division dictionary. If so, look it up. If successful,
%   load the division dictionary from it, save it in \luavar{lang_id_division_dict_table} and return it. If any of
%   these conditions fail, save and return an empty dictionary.
%    \begin{macrocode}
  debug(
    'Attempting to load division dictionary for "%s" (lang. ID %s)',
    cname,lang_id
  )
  local file_name = get_lang_option(lang_id,'division-dict-path')
  if file_name then
    debug('  Filename: ' .. file_name)
    local p = kpse.find_file(file_name)
    if p then
      debug('  Path (after kpathsea lookup): %s',p)
      lang_id_division_dict_path_table[lang_id] = p

      local case_sensitive = get_lang_option(lang_id,'validation-case-sensitive')

      division_dict = load_division_dict(p,case_sensitive)
      if division_dict ~= nil then
        info(
          '  Loaded division dict for "%s" (lang. ID %s) from "%s"',
          cname,lang_id,path
        )
        lang_id_division_dict_read_table[lang_id] = true
        lang_id_division_dict_table[lang_id] = division_dict
        return division_dict,preprocessor
      end

    else
      warning(
        '  Specified division dict "%s" not found for "%s" (lang. ID %s)',
        file_name,cname,lang_id
      )
    end
  else
    debug('  No division dict path specified')
  end

  lang_id_division_dict_read_table[lang_id] = false
  division_dict = {}
  lang_id_division_dict_table[lang_id] = division_dict

  return division_dict,preprocessor

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsection{Validating a hyphenation}
%
% Constants for the status of a hyphenation
%    \begin{macrocode}
local STATUS_VALID = 1
local STATUS_INVALID = 2
local STATUS_AMBIGUOUS = 3
local STATUS_UNKNOWN = 4
%    \end{macrocode}
%
%
%
% \begin{macro}[int]{validate_division}
%   Return the appropriate \luavar{STATUS_*} depending on whether \luavar{divided} matches the known pattern for
%   \luavar{word} in the language with the given ID.
%    \begin{macrocode}
local function validate_division(lang_id,word,divided)

  debug(
    '    Validating "%s" for language ID %s',divided,lang_id
  )
  local division_dict,preprocessor
    = get_lang_division_dict_validation_preprocessor(lang_id)
%    \end{macrocode}
%   Use the preprocessor on the original and divided word.
%    \begin{macrocode}
  word = preprocessor(word)
  divided = preprocessor(divided)

  if not get_lang_option(lang_id,'validation-case-sensitive') then
    word = unicode.utf8.lower(word)
    divided = unicode.utf8.lower(divided)
  end

  local divisions = division_dict[word]
  if not divisions then
    return STATUS_UNKNOWN
  end

  local count = 0

  for _,division in ipairs(divisions) do
    if hyphenation_matches_pattern(divided,division) then
      count = count + 1
    end
  end

  if count == 0 then
    return STATUS_INVALID
  elseif count == #divisions then
    return STATUS_VALID
  else
    return STATUS_AMBIGUOUS
  end

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsection{Getting text from nodes}
%
% Getting the components of the ligatures that have Unicode code points can be problematic, at least for some fonts,
% so define a lookup table for these cases.
%    \begin{macrocode}
local LIGATURE_TEXT = {
  [0xfb00] = 'ff',
  [0xfb01] = 'fi',
  [0xfb02] = 'fl',
  [0xfb03] = 'ffi',
  [0xfb04] = 'ffl',
  [0xfb05] = 'ſt',
  [0xfb06] = 'st',
}
%    \end{macrocode}
%
%
%
% Cache to save table lookups when extracting text.
%    \begin{macrocode}
local font_characters = {}
%    \end{macrocode}
%
%
%
% Extracting text from nodes uses two functions that call each other, so the names have to be defined ahead of time.
%    \begin{macrocode}
local get_node_text
local get_nodelist_text
%    \end{macrocode}
%
%
%
% \begin{macro}[int]{get_node_text}
%   Return the text content of a glyph node (which might be a normal glyph, a ligature, etc.).
%    \begin{macrocode}
get_node_text = function(n)

  if n.id == NODE_ID_GLYPH then

    local ligature_text = LIGATURE_TEXT[n.char]
    if ligature_text ~= nil then
      return ligature_text
    elseif n.components then
      return get_nodelist_text(n.components)
    else
      -- See [https://tug.org/pipermail/luatex/2018-March/006786.html]
      local characters = font_characters[n.font]
      if not characters then
        characters = fonts.hashes.identifiers[n.font].characters
        font_characters[n.font] = characters
      end
%    \end{macrocode}
%   In looking up the text, things can go wrong (e.g. the glyph might not be part of the version of unicode than is
%   supported by the underlying \luavar{slnunicode} library). So use \luafunc{pcall} and return an empty string if
%   there is an error.
%    \begin{macrocode}
      local ok,retval = pcall(
        function ()
          return utf8.char(tonumber(characters[n.char].tounicode,16))
        end
      )
      if ok then
        return retval
      else
        return ''
      end
    end

  elseif n.id == NODE_ID_DISC then

    if n.replace then
      return get_nodelist_text(n.replace)
    else
      return ''
    end

  else
    return ''
  end

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{get_nodelist_text}
%   Return the text content of the glyph nodes in the list starting at \luavar{head} up to and including the node
%   \luavar{last}, or up to the end of the list if \luavar{last} is not specified.
%    \begin{macrocode}
get_nodelist_text = function (head,last)

  local text = ''

  for item in node.traverse(head) do

    text = text .. get_node_text(item)

    if item == last then
      break
    end
  end

  return text

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{is_possible_word_node}
%  Return boolean indicating if node \luavar{n} could be part of a word. Assume that \luavar{glyph}, \luavar{disc},
%  and \luavar{margin_kern} nodes could be part of a word, as could a \luavar{kern} node with subtype
%  \luavar{fontkern}.
%    \begin{macrocode}
local function is_possible_word_node(n)

  return (
    n.id == NODE_ID_GLYPH
    or
    n.id == NODE_ID_DISC
    or
    (n.id == NODE_ID_KERN and n.subtype == NODE_KERN_SUBTYPE_FONTKERN)
    or
    n.id == NODE_ID_MARGIN_KERN
  )

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{is_possible_space_node}
%  Return boolean indicating if node \luavar{n} could be part of a space. Assume that \luavar{glue} nodes could be
%  part of a space, as could a \luavar{kern} node with subtype \luavar{userkern}.
%    \begin{macrocode}
local function is_possible_space_node(n)

  return (
    n.id == NODE_ID_GLUE
    or
    (n.id == NODE_ID_KERN and n.subtype == NODE_KERN_SUBTYPE_USERKERN)
  )

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsection{Getting language IDs from nodes}
%
% \begin{macro}[int]{get_first_glyph_lang}
%   Return the lang attribute of the first glyph in the the part of the list starting n that could be part of a word.
%   (Currently unused; see the documentation of \luafunc{get_disc_lang}.)
%    \begin{macrocode}
  -- local function get_first_glyph_lang(n)

  --   local item = n
  --   while item and is_possible_word_node(item) do
  --     if item.id == NODE_ID_GLYPH then
  --       return item.lang
  --     end
  --     item = item.next
  --   end

  --   return nil

  -- end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{get_disc_lang}
%   Try to find the language ID in force at a given disc node by looking at (1)~the last glyph in the word
%   before the disc node; (2)~the first glyph in the word after the disc node. Default to language ID \luavar{0}.
%
%   (Looking at \luavar{replace}, \luavar{pre}, \luavar{post} is possible, but is unreliable and so disabled for the
%   present. The author has encountered the situation where an explicit hyphen results in the hyphen characters in
%   \luavar{replace} and \luavar{pre} having different language IDs. He has not had time to investigate how this
%   arises from the interaction of \pkg{babel}/\pkg{polyglossia} and Lua\LaTeX.)
%    \begin{macrocode}
local function get_disc_lang(n)

  -- lang = get_first_glyph_lang(n.replace)
  -- if lang then
  --   return lang
  -- end

  -- lang = get_first_glyph_lang(n.pre)
  -- if lang then
  --   return lang
  -- end

  -- lang = get_first_glyph_lang(n.post)
  -- if lang then
  --   return lang
  -- end

  local item
%    \end{macrocode}
%   Before the disc node.
%    \begin{macrocode}
  item = n
  while item and is_possible_word_node(item) do
    if item.id == NODE_ID_GLYPH then
      return item.lang
    end
    item = item.prev
  end
%    \end{macrocode}
%   After the disc node.
%    \begin{macrocode}
  item = n
  while item and is_possible_word_node(item) do
    if item.id == NODE_ID_GLYPH then
      return item.lang
    end
    item = item.next
  end

  return 0

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsection{Pre-linebreak processing}
%
% Before each line has been broken, find all potential division points and store the words in which they occur,
% linking each potential break point to the corresponding word.
%
% Declare a new attribute, which will be used to store in each disc node the index of the corresponding word in the
% table \luavar{hlist_segment_list}.
%    \begin{macrocode}
local hyphen_attr = luatexbase.new_attribute('hyphen_attr')
%    \end{macrocode}
%
%
%
% Table to hold segments (word/space/math) in the hlist that will be broken. This table will be cleared after the
% post-linebreak processing.
%    \begin{macrocode}
local hlist_segment_list = {}
%    \end{macrocode}
%
%
%
% Constants for types of ‘segments’ found while scanning an hlist before linebreaking.
%    \begin{macrocode}
local SEGMENT_WORD = 0
local SEGMENT_SPACE = 1
local SEGMENT_MATH = 2
%    \end{macrocode}
%
%
%
% \begin{macro}[int]{pre_linebreak}
%   Extract segments (word/space/math) from the hlist at \luavar{hlist_head} and store appropriate data in
%   \luavar{hlist_segment_list}. For spaces and math, this is just the existence of a segment. For a word, store its
%   text and its language ID (as determined by \luafunc{get_disc_lang}). Also, for each disc node, assign the index
%   of the word in \luavar{hlist_segment_list} to its \luavar{hyphen_attr} attribute (declared above).
%    \begin{macrocode}
local function pre_linebreak(hlist_head,groupcode)

  local word_start_node = nil
  local segment_count = 0
  local lang = nil

  debug('Pre-linebreak processing start')

%    \end{macrocode}
%   Per \S~8.2 of the Lua\TeX\ manual, there can be cases where \luavar{.prev} is invalid, so run
%   \luafunc{node.slide()} to set them correctly.
%    \begin{macrocode}
  node.slide(hlist_head)

  local item = hlist_head
  while item do
%    \end{macrocode}
%   If \luavar{item} is a math node (which must have subtype beginmath, unless something has changed the node list),
%   skip the math and add "[MATH]" to \luavar{hlist_segment_list}.
%    \begin{macrocode}
    if item.id == NODE_ID_MATH then
      assert(item.subtype == NODE_MATH_SUBTYPE_BEGIN)
      while not (
        item.id == NODE_ID_MATH and item.subtype == NODE_MATH_SUBTYPE_END
      ) do
        item = item.next
      end
      item = item.next

      segment_count = segment_count + 1
      hlist_segment_list[segment_count] = {
        type = SEGMENT_MATH
      }

      goto continue
    end
%    \end{macrocode}
%   If \luavar{item} is a possible word node, read the whole word, setting the \luavar{hyphen_attr} of any disc nodes
%   to \luavar{segment_count}, and adding the word to \luavar{hlist_segment_list}.
%    \begin{macrocode}
    if is_possible_word_node(item) then
      word_start_node = item
      segment_count = segment_count + 1
      while item and is_possible_word_node(item) do

%    \end{macrocode}
%   When the first disc node is found, find the language of the word.
%    \begin{macrocode}
        if item.id == NODE_ID_DISC then
          if not lang then
            lang = get_disc_lang(item)
          end
          node.set_attribute(item,hyphen_attr,segment_count)
        end

        item = item.next
      end
%    \end{macrocode}
%   \luavar{item} should be a node, because even after the last word node, the hlist will contain something. But just
%   in case, check and find the last node using \luafunc{node.tail} if necessary. This latter case should be very
%   rare, so it is more efficient to recalculate here if necessary rather than having an extra assignment to store the
%   previous node in the while loop.
%    \begin{macrocode}
      local word_end_node
      if item then
        word_end_node = item.prev
      else
        word_end_node = node.tail(word_start_node)
      end

      local word = get_nodelist_text(word_start_node,word_end_node)
      hlist_segment_list[segment_count] = {
        type = SEGMENT_WORD,
        word = word,
        lang = lang,
      }

      word_start_node = nil
      lang = nil

      goto continue
    end
%    \end{macrocode}
%   If \luavar{item} is a node that could be part of a space, add a space to the segment list.
%    \begin{macrocode}
    if is_possible_space_node(item) then
      segment_count = segment_count + 1

      while item and is_possible_space_node(item) do
        item = item.next
      end

      hlist_segment_list[segment_count] = {
        type = SEGMENT_SPACE
      }

      goto continue
    end
%    \end{macrocode}
%   If \luavar{item} is anything else, just move on.
%    \begin{macrocode}
    item = item.next

    ::continue::
  end

  debug('Pre-linebreak processing finish')

  return true
end
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsection{Post-linebreak processing}
%
% \subsubsection{Node processing}
%
% After linebreaking, look for a discretionary node at the end of each line, which indicates that a word has been
% divided between the end of that line and the start of the next. Extract the two word-pieces from the lines and store
% them, together with the undivided word and its context in the appropriate language table. Also insert a whatsit to
% that will set the page number when the hyphenation is written out.
%
% \begin{macro}[int]{get_used_disc}
%   If at the tail of the hlist at \luavar{hlist_head} (which will be a line) there is a disc node not followed by a
%   glyph node, return that disc node. Otherwise return \luavar{nil}. Only search up to at most
%   \luavar{USED_DISC_SEARCH_LIMIT} nodes from the tail.
%    \begin{macrocode}
local USED_DISC_SEARCH_LIMIT = 20

local function get_used_disc(hlist_head)

  debug('  Looking for disc node at end of line')

  local item = node.tail(hlist_head)

  local count = 0
  while item and item.id ~= NODE_ID_GLYPH and count < USED_DISC_SEARCH_LIMIT do
    if item.id == NODE_ID_DISC then
      return item
    end
    item = item.prev
    count = count + 1
  end

  return nil

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{get_disc_word_start}
%   Return the node starting the word that includes a given disc node \luavar{n}, or \luavar{nil} if there is no such
%   node.
%    \begin{macrocode}
local function get_disc_word_start(hlist_head,n)

  local item = n

  while item do
    local prev = item.prev

    if not (prev and is_possible_word_node(prev)) then
      return item
    end

    item = prev
  end

  return nil
end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{get_next_hlist}
%   Return the next hlist in the list containing the given node \luavar{n}, or \luavar{nil} if there is no such hlist
%   node.
%    \begin{macrocode}
local function get_next_hlist(n)

  local item = n.next

  while item do
    if item.id == NODE_ID_HLIST then
      return item
    end
    item = item.next
  end

  return nil

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{get_line_first_word}
%   Return the first word in the hlist at \luavar{hlist_head}, or \luavar{nil} if there is no such word.
%    \begin{macrocode}
local function get_line_first_word(hlist_head)
%    \end{macrocode}
%   \luavar{word_start_node} is either \luavar{nil} or the (glyph) node that starts the word.
%    \begin{macrocode}
  local word_start_node = nil

  for item in node.traverse(hlist_head) do

    if item.id == NODE_ID_GLYPH then
      if not word_start_node then
        word_start_node = item
      end
    end

    if not is_possible_word_node(item) then
      if word_start_node then
        return get_nodelist_text(word_start_node,item.prev)
      end
    end

  end
%    \end{macrocode}
%   It is possible that the word ends at the end of the hlist, so check if a word has been started.
%    \begin{macrocode}
  if word_start_node then
    return get_nodelist_text(word_start_node,node.tail(hlist_head))
  else
    return nil
  end
end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{get_context}
%   Return a string assembled from the part of \luavar{hlist_segment_list} before or after \luavar{index} according to
%   \luavar{incr} (which must be \(\pm 1\)) up a maximum of \luavar{target_word_count} words.
%    \begin{macrocode}
local function get_context(index,incr,target_word_count)

  local result = ''
  local word_count = 0

  local i = index + incr
  while (
    i > 0 and i <= #hlist_segment_list and word_count < target_word_count
  ) do
    local t = hlist_segment_list[i]

    local item

    if t.type == SEGMENT_WORD then
      item = t.word
      word_count = word_count + 1
    elseif t.type == SEGMENT_SPACE then
      item = STR_SPACE
    elseif t.type == SEGMENT_MATH then
      item = STR_MATH
    end

    if incr > 0 then
      result = result .. item
    else
      result = item .. result
    end

    i = i + incr
  end

  return result

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsubsection{Flags}
%
% Define constant PDF literal nodes for flags.
% \begin{macro}[int]{make_flag_node}
%   Return \luavar{pdf_literal} whatsit node containing \luavar{code} preceded by \luavar{context} (if
%   non-\luavar{nil}) enclosed in a state save/restore.
%    \begin{macrocode}
local function make_flag_node(code,context)
  if context then
    code = context .. ' ' .. code
  end

  local n = node.new('whatsit','pdf_literal')
  n.data = 'q ' .. code .. ' Q'

  return n

end
%    \end{macrocode}
% \end{macro}
%
%
%
% Transformation to shift flags into the margin.
%    \begin{macrocode}
local FLAG_MARGIN_SHIFT = '1 0 0 1 4 -1 cm'
%    \end{macrocode}
%
% Width of flags. (Must match the actual code for flags below.)
%    \begin{macrocode}
local FLAG_WIDTH = tex.sp('6pt')
%    \end{macrocode}
%
% Code for flags.
%    \begin{macrocode}
local FLAG_VALID_CODE
  = '0 0 6 10 re .1 .7 .1 rg f ' -- Green background
  .. '1 w 1 G ' -- Line width 1pt, colour white
  .. '1 5.5 m 2.33333 3 l 5 8 l S' -- "Tick"
local FLAG_INVALID_CODE
  = '0 0 6 10 re .9 0 0 rg f ' -- Red background
  .. '1 w 1 G ' -- Line width 1pt, colour white
  .. '1 3 m 5 7 l 1 7 m 5 3 l S'
local FLAG_AMBIGUOUS_CODE = (
  '0 0 6 10 re .1 .7 .7 rg f ' -- Cyan background
  .. '1 0 0 1 2.85 6.25 cm ' -- Shift
  .. '1 w 1 G ' -- Line width 1pt, colour white
  .. '-1.55885 .9 m -1 1.4 -.65 2 0 2 c 1.3 2 1.8 1.3 1.8 .5 c '
  .. '1.8 -.75 0 -1.25 0 -2.5 c 0 -2.75 l S ' -- Question mark body
  .. '1.4 w 1 J ' -- Line width 1.4pt, line cap round
  .. '0 -4.25 m 0 -4.25 l S' -- Question mark dot
)
local FLAG_UNKNOWN_CODE = (
  '0 0 6 10 re .9 .9 0 rg f ' -- Yellow background
  .. '1 0 0 1 3 5 cm ' -- Shift
  .. '.8 w 1 G ' -- Linec with .8pt, colour white
  .. '-1.41421 -1.41421 m 1.41421 1.41421 l S ' -- Stroke of 0
  .. '2 0 m 2 1.2 1.2 3 0 3 c -1.2 3 -2 1.2 -2 0 c -2 -1.2 -1.2 -3 0 -3 c '
  .. '1.2 -3 2 -1.2 2 0 c h S' -- Outside of 0
)
%    \end{macrocode}
% Nodes made from the preceding code.
%    \begin{macrocode}
local FLAG_VALID_NODE = make_flag_node(FLAG_VALID_CODE,FLAG_MARGIN_SHIFT)
local FLAG_INVALID_NODE = make_flag_node(FLAG_INVALID_CODE,FLAG_MARGIN_SHIFT)
local FLAG_AMBIGUOUS_NODE = make_flag_node(FLAG_AMBIGUOUS_CODE,FLAG_MARGIN_SHIFT)
local FLAG_UNKNOWN_NODE = make_flag_node(FLAG_UNKNOWN_CODE,FLAG_MARGIN_SHIFT)
%    \end{macrocode}
%
%
%
% \begin{macro}[int]{insert_flag}
%   Add a ‘flag’ after the node \luavar{current} in the list at \luavar{head}, with the type of the flag corresponding
%   to \luavar{status}.
%    \begin{macrocode}
local function insert_flag(head,current,status)

  if status == STATUS_VALID then
    node.insert_after(head,current,node.copy(FLAG_VALID_NODE))
  elseif status == STATUS_UNKNOWN then
    node.insert_after(head,current,node.copy(FLAG_UNKNOWN_NODE))
  elseif  status == STATUS_INVALID then
    node.insert_after(head,current,node.copy(FLAG_INVALID_NODE))
  elseif  status == STATUS_AMBIGUOUS then
    node.insert_after(head,current,node.copy(FLAG_AMBIGUOUS_NODE))
  end

end
%    \end{macrocode}
%  \end{macro}
%
%
%
% \begin{macro}[int]{write_flag}
%   Generic function to write a flag to the current \TeX\ list.
%    \begin{macrocode}
local function write_flag(code)

  node.write(make_flag_node(code,'1 0 0 1 0 -1 cm'))

  local kern_n = node.new('kern','userkern')
  kern_n.kern=FLAG_WIDTH
  node.write(kern_n)

end
%    \end{macrocode}
%  \end{macro}
%
%
%
% \begin{macro}[int]{write_flag_valid}
%   Write the valid flag to the current \TeX\ list.
%    \begin{macrocode}
local function write_flag_valid()

  write_flag(FLAG_VALID_CODE)

end
%    \end{macrocode}
%  \end{macro}
%
%
%
% \begin{macro}[int]{write_flag_invalid}
%   Write the invalid flag to the current \TeX\ list.
%    \begin{macrocode}
local function write_flag_invalid()

  write_flag(FLAG_INVALID_CODE)

end
%    \end{macrocode}
%  \end{macro}
%
%
%
% \begin{macro}[int]{write_flag_ambiguous}
%   Write the ambiguous flag to the current \TeX\ list.
%    \begin{macrocode}
local function write_flag_ambiguous()

  write_flag(FLAG_AMBIGUOUS_CODE)

end
%    \end{macrocode}
%  \end{macro}
%
%
%
% \begin{macro}[int]{write_flag_unknown}
%   Write the unknown flag to the current \TeX\ list.
%    \begin{macrocode}
local function write_flag_unknown()

  write_flag(FLAG_UNKNOWN_CODE)

end
%    \end{macrocode}
%  \end{macro}
%
%
%
%
% \subsubsection{Main list of hyphenations}
%
% Count and list for hyphenations. Each entry in the list will be a table containing the original word, the
% hyphenation, the language, the index of the table in the list (which is needed later for stable sorting and sorting
% into the original order), and the context.
%    \begin{macrocode}
local hyphenation_list = {}
local hyphenation_count = 0
%    \end{macrocode}
%
%
%
% \subsubsection{Check single line hyphenation}
%
% \begin{macro}[int]{check_line_hyphenation}
%   Check whether there is a hyphenated word at the end of the given hlist; if so, save the word to
%   \luavar{hyphenation_list}.
%    \begin{macrocode}
local function check_line_hyphenation(hlist)
%    \end{macrocode}
%   First, is there a disc node not followed by a glyph node at the end of the list?
%    \begin{macrocode}
  local last_disc = get_used_disc(hlist.head)
  if not last_disc then
    debug('  No disc node found at end of line')
    return
  end
  debug('  Disc node found at end of line')
%    \end{macrocode}
%   Get the undivided word and its language from \luavar{hlist_segment_list}.
%    \begin{macrocode}
  local hyphenation_index = node.has_attribute(last_disc,hyphen_attr)
  local t = hlist_segment_list[hyphenation_index]
  assert(t)
  assert(t.type == SEGMENT_WORD)
  local word = t.word
  local lang = t.lang
%    \end{macrocode}
%   \luavar{word} might be something other than a genuine word, such as an ISBN (with hyphen separators). So only
%   proceed if it contains at least one letter.
%    \begin{macrocode}
  if not unicode.utf8.match(word,'%a') then
    debug('  Divided "word" contains no letters')
    return
  end
%    \end{macrocode}
%   There should always be a next line, since there is a disc node at the end of \luavar{hlist}, but check anyway.
%    \begin{macrocode}
  local next_line = get_next_hlist(hlist)

  if not next_line then
    debug('  No following line found (which should not happen)')
    return
  end
%    \end{macrocode}
%   For the pre-linebreak part of the word, get the word that ends the line, and trim any leading non-letters. This
%   could leave an empty word; for example, if \(n\)-dimensional is broken at the hyphen, the word ending the line is
%   just the hyphen. If an empty word is left, just use the non-trimmed result.
%    \begin{macrocode}
  local pre = get_nodelist_text(get_disc_word_start(hlist.head,last_disc))
  local pre_temp = trim_nonlettershyphens_start(pre)
  if pre_temp ~= '' then
    pre = pre_temp
  end
%    \end{macrocode}
%   For the post-linebreak part, just get the word at the start of the next line, and trim and trailing non-letters.
%    \begin{macrocode}
  local post = trim_nonlettershyphens_end(get_line_first_word(next_line.head))
%    \end{macrocode}
%   Save the original word to be included in the context later.
%    \begin{macrocode}
  local word_orig = word
%    \end{macrocode}
%   Check if the division is valid/invalid/ambiguous/unknown. (No calls to \luafunc{get_lang_option} should appear
%   before this point in this function, since it is \luafunc{validate_division} that ultimately sets the
%   language--name association when \pkg{polyglossia} is in use.)
%    \begin{macrocode}
  word = trim_nonlettershyphens_both(word)

  local divided = pre .. post
  debug('  Hyphenated word found: %s -> %s (%s|%s)',word,divided,pre,post)
  local status = validate_division(lang,word,divided)
%    \end{macrocode}
%   If specified, insert a flag depending on the flag mode for this language and the status.
%    \begin{macrocode}
  local flag_mode = get_lang_option(lang,'flag-mode')
  if flag_mode == 1 or (flag_mode == 2 and status ~= STATUS_VALID) then
    insert_flag(hlist.head,last_disc,status)
  end
%    \end{macrocode}
%   Compute the context and then trim any unwanted symbols from the word itself.
%    \begin{macrocode}
   local context =
    get_context(
      hyphenation_index,-1,get_lang_option(lang,'context-before')
    )
    .. word_orig ..
    get_context(
      hyphenation_index,1,get_lang_option(lang,'context-after')
    )
%    \end{macrocode}
%   Store everything (except the page number on which the hyphenated word appears, which is not yet known) in the
%   hyphenation list.
%    \begin{macrocode}
  hyphenation_count = hyphenation_count + 1
  hyphenation_list[hyphenation_count] = {
    lang = lang,
    word = word,
    divided = divided,
    index = hyphenation_count,
    context = context,
    status = status,
  }
%    \end{macrocode}
%   Add a whatsit to record the page number when the page with the hyphenation is shipped out. This information also
%   serves to distinguish hyphenations that are written to the page from those that occur in (e.g.) boxes that are
%   discarded without being written to the page.
%    \begin{macrocode}
  late_lua_n = node.new('whatsit','late_lua')
  late_lua_n.data =
    'lualisthyphen.set_hyphenation_page('
    .. hyphenation_count .. ',tex.count["c@page"])'

  node.insert_after(hlist.head,last_disc,late_lua_n)

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{set_hyphenation_page}
%   Set the page on which the hyphenation with the given index appears.
%    \begin{macrocode}
local function set_hyphenation_page(index,page)

  hyphenation_list[index].page = page

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsubsection{Main post-linebreak function}
%
% \begin{macro}[int]{post_linebreak}
%   For every line in the vlist at \luavar{vlist_head}, check whether there is a hyphenated word at the end.
%    \begin{macrocode}
local function post_linebreak(vlist_head,groupcode)

  debug('Post-linebreak processing start')
%    \end{macrocode}
%   Per \S~8.2 of the Lua\TeX\ manual, there can be cases where \luavar{.prev} is invalid, so run
%   \luafunc{node.slide()} on each line to set them correctly.
%    \begin{macrocode}
  for item in node.traverse(vlist_head) do
    if item.id == NODE_ID_HLIST then
      node.slide(item.head)
    end
  end
%    \end{macrocode}
%   Now actually look for hyphenations.
%    \begin{macrocode}
  local line_no = 0

  for item in node.traverse(vlist_head) do

    if item.id == NODE_ID_HLIST then
      line_no = line_no + 1
      debug('  Line no. %d',line_no)
      check_line_hyphenation(item)
    end

  end

  hlist_segment_list = {}

  debug('Post-linebreak processing end')

  return true

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsection{Add callbacks}
%
% Add \luafunc{pre_linebreak} and \luafunc{post_linebreak} to the relevant callbacks.
%    \begin{macrocode}
local LUA_LIST_HYPHEN_PRE_LINEBREAK = 'LUA_LIST_HYPHEN_PRE_LINEBREAK'
luatexbase.add_to_callback(
  'pre_linebreak_filter',
  pre_linebreak,
  LUA_LIST_HYPHEN_PRE_LINEBREAK
)

local LUA_LIST_HYPHEN_POST_LINEBREAK = 'LUA_LIST_HYPHEN_POST_LINEBREAK'
luatexbase.add_to_callback(
  'post_linebreak_filter',
  post_linebreak,
  LUA_LIST_HYPHEN_POST_LINEBREAK
)
%    \end{macrocode}
%
%
%
% \subsection{Processing hyphenation lists}
%
% Before hyphenation lists are written out, they are filtered, deduplicated and/or sorted in accordance with the set
% options.
%
%
%
% \subsubsection{Comparisons and equality checks for stored hyphenations}
%
% \begin{macro}[int]{equal_hyphenation_case_sensitive}
%   Equality check for deduplicating the list of hyphenations case-sensitively.
%    \begin{macrocode}
local function equal_hyphenation_case_sensitive(a,b)
  return (
    a.word == b.word
    and
    a.divided == b.divided
  )
end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{equal_hyphenation_case_insensitive}
%   Equality check for deduplicating the list of hyphenations case-insensitively.
%    \begin{macrocode}
local function equal_hyphenation_case_insensitive(a,b)
  return (
    unicode.utf8.lower(a.word) == unicode.utf8.lower(b.word)
    and
    unicode.utf8.lower(a.divided) == unicode.utf8.lower(b.divided)
  )
end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{lessthan_hyphenation_case_sensitive}
%   Comparison for sorting the list of hyphenations case-sensitively.
%
%   The comparison of \luavar{index} keys ensures that the sorting is stable.
%    \begin{macrocode}
local function lessthan_hyphenation_case_sensitive(a,b)
  return (
    a.word < b.word
    or
    (
      a.word == b.word
      and
      a.divided < b.divided
    )
    or
    (
      a.word == b.word
      and
      a.divided == b.divided
      and
      a.index < b.index
    )
  )
end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{lessthan_hyphenation_case_insensitive}
%   Comparison for sorting the list of hyphenations case-insensitively.
%
%   The comparison of \luavar{index} keys ensures that the sorting is stable.
%    \begin{macrocode}
local function lessthan_hyphenation_case_insensitive(a,b)
  return (
    unicode.utf8.lower(a.word) < unicode.utf8.lower(b.word)
    or
    (
      unicode.utf8.lower(a.word) == unicode.utf8.lower(b.word)
      and
      unicode.utf8.lower(a.divided) < unicode.utf8.lower(b.divided)
    )
    or
    (
      unicode.utf8.lower(a.word) == unicode.utf8.lower(b.word)
      and
      unicode.utf8.lower(a.divided) < unicode.utf8.lower(b.divided)
      and
      a.index < b.index
    )
  )
end
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsubsection{Sorting}
%
% \begin{macro}[int]{sort_hyphenation_list_none}
%   Sort \luavar{hyphenation_list} into its original order of appearance.
%    \begin{macrocode}
local function sort_hyphenation_list_none(hyphenation_list)
  table.sort(
    hyphenation_list,
    function(a,b)
      return a.index < b.index
    end
  )
end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{sort_hyphenation_list_case}
%   Sort \luavar{hyphenation_list} case-sensitively.
%    \begin{macrocode}
local function sort_hyphenation_list_case(hyphenation_list)
  table.sort(
    hyphenation_list,
    lessthan_hyphenation_case_sensitive
  )
end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{sort_hyphenation_list_nocase}
%   Sort \luavar{hyphenation_list} case-insensitively.
%    \begin{macrocode}
local function sort_hyphenation_list_nocase(hyphenation_list)
  table.sort(
    hyphenation_list,
    lessthan_hyphenation_case_insensitive
  )
end
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsubsection{Deduplication}
%
% \begin{macro}[int]{deduplicate_hyphenation_list_case}
%   Remove duplicates from \luavar{hyphenation_list} case-sensitively.
%    \begin{macrocode}
local function deduplicate_hyphenation_list_case(hyphenation_list)
  table.sort(
    hyphenation_list,
    lessthan_hyphenation_case_sensitive
  )
  list_uniq(
    hyphenation_list,
    equal_hyphenation_case_sensitive
  )
end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{deduplicate_hyphenation_list_nocase}
%   Remove duplicates from \luavar{hyphenation_list} case-insensitively.
%    \begin{macrocode}
local function deduplicate_hyphenation_list_nocase(hyphenation_list)
  table.sort(
    hyphenation_list,
    lessthan_hyphenation_case_insensitive
  )
  list_uniq(
    hyphenation_list,
    equal_hyphenation_case_insensitive
  )
end
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsubsection{Combined processing}
%
% \begin{macro}[int]{process_lang_hyphenation_list}
%   As specified in the options for the given language, filter, deduplicates, and sort \luavar{hyphenation_list}.
%    \begin{macrocode}
local function process_lang_hyphenation_list(lang_id,hyphenation_list)

  local output_non_typeset = get_lang_option(lang_id,'output-non-typeset')
  local include_valid = (get_lang_option(lang_id,'output-mode') == 0)

%    \end{macrocode}
%   Filter the list to remove valid hyphenations, if required, and non-typeset hyphenations, if required.
%    \begin{macrocode}
  list_filter(
    hyphenation_list,
    function (a)
      return (
        (a.status ~= STATUS_VALID or include_valid)
        and
        (a.page or output_non_typeset)
      )
    end
  )
%    \end{macrocode}
%   Deduplicate in accordance with the specified options for this language.
%    \begin{macrocode}
  local unique = get_lang_option(lang_id,'unique')
  if unique == 1 then
    deduplicate_hyphenation_list_case(hyphenation_list)
  elseif unique == 2 then
    deduplicate_hyphenation_list_nocase(hyphenation_list)
  end
%    \end{macrocode}
%   Sort in accordance with the specified options for this language.
%    \begin{macrocode}
  local sort_mode = get_lang_option(lang_id,'sort')
  if sort_mode == 1 then
    sort_hyphenation_list_case(hyphenation_list)
  elseif sort_mode == 2 then
    sort_hyphenation_list_nocase(hyphenation_list)
  else
    sort_hyphenation_list_none(hyphenation_list)
  end

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsection{Writing}
%
% \begin{macro}[int]{write_lang_hyphenation_list_standard}
%   Write out just the divided words in \luavar{hyphenation_list} to file handle \luavar{f}.
%    \begin{macrocode}
local function write_lang_hyphenation_list_standard(f,hyphenation_list,widths)

  for i,v in ipairs(hyphenation_list) do

    if v then
      f:write(v.divided .. '\n')
    end

  end

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{write_lang_hyphenation_list_verbose}
%   Write out all hyphenation information in \luavar{hyphenation_list} to file handle \luavar{f}, in columns as
%   specified. The parameter \luavar{show_status} is boolean and controls whether the status is written.
%    \begin{macrocode}
local function write_lang_hyphenation_list_verbose(f,hyphenation_list,show_status)
%    \end{macrocode}
%   Calculate the number of columns required for each output field.
%    \begin{macrocode}
  local cols_word = 0
  local cols_divided = 0
  local cols_page = 0

  for _,h in pairs(hyphenation_list) do

    cols_word = math.max(
      cols_word,
      unicode.utf8.len(h.word)
    )
    cols_divided = math.max(
      cols_divided,
      unicode.utf8.len(h.divided)
    )
    cols_page = math.max(
      cols_page,
      unicode.utf8.len(tostring(h.page))
    )

  end
%    \end{macrocode}
%   Adjust the number of columns required the page output, since there is a prefix and a ‘no page’ indicator to
%   consider.
%    \begin{macrocode}
  cols_page = math.max(
    cols_page + unicode.utf8.len(STR_PAGE_PREFIX),
    unicode.utf8.len(STR_PAGE_NONE)
  )

  local cols_status = math.max(
    unicode.utf8.len(STR_VALID),
    unicode.utf8.len(STR_INVALID),
    unicode.utf8.len(STR_AMBIGUOUS),
    unicode.utf8.len(STR_UNKNOWN)
  ) + 2 -- Add for parentheses

  for i,v in ipairs(hyphenation_list) do

    if v then
%    \end{macrocode}
%   It is possible for the \luavar{page} key not to have been set, for instance if the hyphenation occured in a box
%   that was never output, so prepare the string containing the page accordingly.
%    \begin{macrocode}
      local page = v.page
      if page then
        page = STR_PAGE_PREFIX .. page
      else
        page = STR_PAGE_NONE
      end
%    \end{macrocode}
%   Set the string containing the status depending on \luavar{show_status}.
%    \begin{macrocode}
      local status
      if show_status then
        status = v.status
        if status == STATUS_VALID then
          status = STR_VALID
        elseif status == STATUS_AMBIGUOUS then
          status = STR_AMBIGUOUS
        elseif status == STATUS_INVALID then
          status = STR_INVALID
        else
          status = STR_UNKNOWN
        end
        status = rpad(parenthesize(status),cols_status) .. STR_SPACE_TWO
      else
        status = ''
      end
%    \end{macrocode}
%   Write everything out on a single line.
%    \begin{macrocode}
      f:write(
        rpad(v.word,cols_word)
        .. STR_ARROW
        .. rpad(v.divided,cols_divided)
        .. STR_SPACE_TWO
        .. status
        .. lpad(page,cols_page)
        .. STR_SPACE
        .. STR_QUOTE_OPEN
        .. v.context
        .. STR_QUOTE_CLOSE
        .. '\n'
      )
    end

  end

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{get_settings_desc}
%   Compute a settings description to insert into file headers.
%    \begin{macrocode}
local function get_settings_desc(lang_id)

  local settings_desc

  if get_lang_option(lang_id,'output-verbose') then
    settings_desc = string.format(
      'verbose=true,context-before=%d,context-after=%s',
      get_lang_option(lang_id,'context-before'),
      get_lang_option(lang_id,'context-after')
    )
  else
    settings_desc = 'verbose=false'
  end

  local ALL_NONVALID = {
    [0] = 'all',
    [1] = 'non-valid',
  }
  settings_desc = settings_desc .. string.format(
    ',output-mode=%s',
    ALL_NONVALID[get_lang_option(lang_id,'output-mode')]
  )
  settings_desc = settings_desc .. string.format(
    ',include-non-typeset=%s',
    get_lang_option(lang_id,'output-non-typeset')
  )
  local NONE_CASE_NOCASE = {
    [0] = 'none',
    [1] = 'case',
    [2] = 'nocase'
  }
  settings_desc = settings_desc .. string.format(
    ',sort=%s,unique=%s',
    NONE_CASE_NOCASE[get_lang_option(lang_id,'sort')],
    NONE_CASE_NOCASE[get_lang_option(lang_id,'unique')]
  )

  return settings_desc

end
%    \end{macrocode}
% \end{macro}
%
%
% \begin{macro}[int]{prefix_output_directory}
%   Return the given \luavar{path} with the output directory prefixed, if defined.
%    \begin{macrocode}
local function prefix_output_directory(path)

  local output_directory = status.output_directory
  if not output_directory then
    return path
  end

  if string.sub(output_directory,-1,-1) == '/' then
    return output_directory .. path
  else
    return output_directory .. '/' .. path
  end

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{process_write_lang_hyphenation_list}
%   Process and write out the \luavar{hyphenation_list} (which will be for the language with the numerical
%   \luavar{lang_id}) to a file with the given \luavar{output_prefix} and \luavar{output_ext}, using
%   \luavar{widths} for the ‘columns’ in verbose mode.
%    \begin{macrocode}
local function process_write_lang_hyphenation_list(
    lang_id,hyphenation_list
)
  process_lang_hyphenation_list(lang_id,hyphenation_list)

  local output_path
  local lang_desc

  local cname = lang_id_cname_table[lang_id]

  debug('Processing and writing hyphenation lists for "%s" (lang. ID %d)',cname,lang_id)

  if cname then
    output_path = get_lang_option(lang_id,'output-path')
    if not output_path then
      output_path = get_lang_option(LANG_DEFAULT,'output-prefix')
        .. cname .. get_lang_option(LANG_DEFAULT,'output-extension')
    end
    lang_desc = string.format('language "%s" (ID %s)',cname,lang_id)
  else
    output_path = get_lang_option(LANG_DEFAULT,'output-prefix')
      .. lang_id .. get_lang_option(LANG_DEFAULT,'output-extension')
    lang_desc = 'language with ID ' .. lang_id
  end

  output_path = prefix_output_directory(output_path)
  debug('Output path: "%s"',output_path)

  local output_header = get_lang_option(lang_id,'output-header')
  local output_verbose = get_lang_option(lang_id,'output-verbose')

  local division_dict_read = lang_id_division_dict_read_table[lang_id]

  local f = io.open(output_path,'w')

  if output_header then
    f:write('% This file was generated by lua-list-hyphen\n')
    f:write('% Hyphenations for ' ..  lang_desc .. '\n')
    f:write('% General settings: ' .. get_settings_desc(lang_id) .. '\n')
    local division_dict_path = lang_id_division_dict_path_table[lang_id]
    if division_dict_path and division_dict_read then
      f:write(
        '% Division dictionary read from: "' .. division_dict_path .. '"\n'
      )
    end
  end

  if output_verbose then
    write_lang_hyphenation_list_verbose(f,hyphenation_list,division_dict_read)
  else
    write_lang_hyphenation_list_standard(f,hyphenation_list,division_dict_read)
  end

  f:close()

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \begin{macro}[int]{process_write_hyphenation_lists}
%   Divide \luavar{hyphenation_list} into per-language lists and write them out to separate files.
%    \begin{macrocode}
local function process_write_hyphenation_lists()

  local lang_hyphenation_table = {}
  local lang_widths_table = {}
%    \end{macrocode}
%   Iterate through all the stored hyphenations. Sort them into per-language lists (creating the list the first time
%   each language is encountered).
%    \begin{macrocode}
  for _,h in pairs(hyphenation_list) do

    local lang = h.lang

    local t = lang_hyphenation_table[lang]
    if not t then
      lang_hyphenation_table[lang] = {}
      t = lang_hyphenation_table[lang]
    end

    table.insert(t,h)

  end
%    \end{macrocode}
%   For each language, process and write out its hyphenations to a file.
%    \begin{macrocode}
  for lang_id,hyphenations in pairs(lang_hyphenation_table) do
    process_write_lang_hyphenation_list(lang_id,hyphenations)
  end

end
%    \end{macrocode}
% \end{macro}
%
%
%
% \subsection{Export public functions}
%
% Finally, make available the functions that will be called from the \LaTeX\ frontend using \cs{lua_now:n}
% or \cs{lua_now:e}.
%    \begin{macrocode}
lualisthyphen = lualisthyphen or {}
lualisthyphen.process_write_hyphenation_lists = process_write_hyphenation_lists
lualisthyphen.set_hyphenation_page = set_hyphenation_page
lualisthyphen.store_lang_id_name = store_lang_id_name

lualisthyphen.write_flag_valid = write_flag_valid
lualisthyphen.write_flag_invalid = write_flag_invalid
lualisthyphen.write_flag_ambiguous = write_flag_ambiguous
lualisthyphen.write_flag_unknown = write_flag_unknown

lualisthyphen.set_debug = set_debug
lualisthyphen.set_global_option = set_global_option
lualisthyphen.set_lang_option = set_lang_option

lualisthyphen.preprocessors = preprocessors
%    \end{macrocode}
%
%
%
%    \begin{macrocode}
%</lua>
%    \end{macrocode}

%
%
%
% \clearpage
% \end{implementation}
%
%
%
% \iffalse
%<*metadriver>
\input{lua-list-hyphen.dtx}
%</metadriver>
%
%<*demo>
\documentclass[twocolumn,oneside]{book}

\usepackage[
paperwidth=100mm,
paperheight=77.48mm,
inner=5mm,
width=87mm,
columnsep=7mm,
top=5mm,
height=192pt,
nohead,
nofoot,
]{geometry}

\usepackage{polyglossia}
\setdefaultlanguage[variant=british]{english}

\usepackage[
output-verbose=true,
output-header=false,
sort=nocase,
unique=nocase,
context-before=2,
context-after=2,
flag-mode=all,
output-mode=non-valid,
division-dict-path=division-dict-ukenglish.txt
]{lua-list-hyphen}

\pagestyle{empty}
\setlength{\parfillskip}{0pt}

\begin{document}

In elective monarchies, the vacancy of the throne is a moment
big with danger and mischief. The Roman emperors, desirous to
spare the legions that interval of suspense, and the temptation of an
irregular choice, invested their designed successor with so large a share
of pres\-ent power, as should enable him, after their decease, to assume
the remainder without suffering the empire to perceive the change of
masters. Thus Augustus, after all his fairer prospects had been
snatched from him by untimely deaths, rested his last hopes on
Tiberius, obtained for his adopted son the censorial and tribunitian
powers, and dictated a law, by which the future prince was invested
with an authority equal to his own over the provinces and the armies.
Thus Vespasian subdued the generous mind of his eldest son.

\end{document}
%</demo>
%
%<*demo-division-dict>
de-cease
en-able
fu-ture
gen-er-ous
ir-regu-lar
ob-tained
pres-ent (adj./noun)
pre-sent (verb)
re-main-der
trib-un-itian
un-time-ly
%</demo-division-dict>
% \fi