-
Notifications
You must be signed in to change notification settings - Fork 3
/
Copy pathmodules-tokenized-to-tree.html
executable file
·423 lines (398 loc) · 34.1 KB
/
modules-tokenized-to-tree.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
<?xml version="1.0" encoding="UTF-8"?><html xmlns="http://www.w3.org/1999/xhtml" xml:base="../modules-tokenized-to-tree.html">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<title>tokenized-to-tree</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="theme-color" content="#FF9800" />
<link rel="apple-touch-icon-precomposed" href="icons/apple-touch-icon-152x152.png" />
<meta name="msapplication-TileImage" content="icons/mstile-144x144.png" />
<link rel="icon" href="icons/favicon-32x32.png" sizes="32x32" />
<link type="text/css" rel="stylesheet" media="screen,projection" href="css/prism.css" />
<link type="text/css" rel="stylesheet" media="screen,projection" href="css/materialize.min.css" />
<link type="text/css" rel="stylesheet" media="screen,projection" href="css/le-tex.css" />
</head>
<body id="modules-tokenized-to-tree">
<header>
<div class="container"><a href="#" data-activates="tr-nav" class="button-collapse top-nav circle hide-on-large-only"><i class="mdi-navigation-menu"></i></a></div>
<ul id="tr-nav" class="side-nav fixed" style="width:260px">
<li class="logo"><a id="logo-container" href="#" class="brand-logo"><object id="front-page-logo" type="image/svg+xml" data="icons/logo-transpect.svg">Your browser does not support SVG</object></a></li>
<li class="naventry-1st"><a href="index.html">About</a></li>
<li class="naventry-1st"><a href="getting-started.html">Getting Started</a></li>
<li class="active no-padding">
<ul class="collapsible collapsible-accordion">
<li class="active naventry-1st" id="d42574e418"><a class="active collapsible-header">Modules</a><div style="display:block" class="collapsible-body">
<ul>
<li class="naventry-2nd"><a href="modules-CoCoTeX.html">CoCoTeX</a></li>
<li class="naventry-2nd"><a href="modules-ace-daisy.html">ace-daisy</a></li>
<li class="naventry-2nd"><a href="modules-basex-svn-api.html">basex-svn-api</a></li>
<li class="naventry-2nd"><a href="modules-bib-parser.html">bib-parser</a></li>
<li class="naventry-2nd"><a href="modules-bits2hub.html">bits2hub</a></li>
<li class="naventry-2nd"><a href="modules-calabash-distro.html">calabash-distro</a></li>
<li class="naventry-2nd"><a href="modules-calabash-frontend.html">calabash-frontend</a></li>
<li class="naventry-2nd"><a href="modules-cascade.html">cascade</a></li>
<li class="naventry-2nd"><a href="modules-control.html">control</a></li>
<li class="naventry-2nd"><a href="modules-control-backend.html">control-backend</a></li>
<li class="naventry-2nd"><a href="modules-crossref.html">crossref</a></li>
<li class="naventry-2nd"><a href="modules-css-tools.html">css-tools</a></li>
<li class="naventry-2nd"><a href="modules-docx2hub.html">docx2hub</a></li>
<li class="naventry-2nd"><a href="modules-docx2hub-frontend.html">docx2hub-frontend</a></li>
<li class="naventry-2nd"><a href="modules-docx2jats-demo.html">docx2jats-demo</a></li>
<li class="naventry-2nd"><a href="modules-docx2tex.html">docx2tex</a></li>
<li class="naventry-2nd"><a href="modules-docx_modify-lib.html">docx_modify-lib</a></li>
<li class="naventry-2nd"><a href="modules-epub-font-subsetter.html">epub-font-subsetter</a></li>
<li class="naventry-2nd"><a href="modules-epub2epub.html">epub2epub</a></li>
<li class="naventry-2nd"><a href="modules-epub2hub-lib.html">epub2hub-lib</a></li>
<li class="naventry-2nd"><a href="modules-epubcheck-extension.html">epubcheck-extension</a></li>
<li class="naventry-2nd"><a href="modules-epubcheck-idpf.html">epubcheck-idpf</a></li>
<li class="naventry-2nd"><a href="modules-epubcheck-transpect.html">epubcheck-transpect</a></li>
<li class="naventry-2nd"><a href="modules-epubtools.html">epubtools</a></li>
<li class="naventry-2nd"><a href="modules-epubtools-frontend.html">epubtools-frontend</a></li>
<li class="naventry-2nd"><a href="modules-evolve-hub.html">evolve-hub</a></li>
<li class="naventry-2nd"><a href="modules-font-obfuscate-extension.html">font-obfuscate-extension</a></li>
<li class="naventry-2nd"><a href="modules-fontmaps.html">fontmaps</a></li>
<li class="naventry-2nd"><a href="modules-github-api.html">github-api</a></li>
<li class="naventry-2nd"><a href="modules-html2hub.html">html2hub</a></li>
<li class="naventry-2nd"><a href="modules-html2xlsx.html">html2xlsx</a></li>
<li class="naventry-2nd"><a href="modules-htmlreports.html">htmlreports</a></li>
<li class="naventry-2nd"><a href="modules-htmltables.html">htmltables</a></li>
<li class="naventry-2nd"><a href="modules-htmltemplates.html">htmltemplates</a></li>
<li class="naventry-2nd"><a href="modules-hub2bits.html">hub2bits</a></li>
<li class="naventry-2nd"><a href="modules-hub2dbk.html">hub2dbk</a></li>
<li class="naventry-2nd"><a href="modules-hub2docx-frontend.html">hub2docx-frontend</a></li>
<li class="naventry-2nd"><a href="modules-hub2docx-lib.html">hub2docx-lib</a></li>
<li class="naventry-2nd"><a href="modules-hub2html.html">hub2html</a></li>
<li class="naventry-2nd"><a href="modules-hub2tei.html">hub2tei</a></li>
<li class="naventry-2nd"><a href="modules-idml2xml.html">idml2xml</a></li>
<li class="naventry-2nd"><a href="modules-idml2xml-frontend.html">idml2xml-frontend</a></li>
<li class="naventry-2nd"><a href="modules-idml_docx-to-docbook_epub-demo.html">idml_docx-to-docbook_epub-demo</a></li>
<li class="naventry-2nd"><a href="modules-idmlvalidation.html">idmlvalidation</a></li>
<li class="naventry-2nd"><a href="modules-image-props-extension.html">image-props-extension</a></li>
<li class="naventry-2nd"><a href="modules-image-transform-extension.html">image-transform-extension</a></li>
<li class="naventry-2nd"><a href="modules-jats2html.html">jats2html</a></li>
<li class="naventry-2nd"><a href="modules-javascript-extension.html">javascript-extension</a></li>
<li class="naventry-2nd"><a href="modules-jing-trang-patch.html">jing-trang-patch</a></li>
<li class="naventry-2nd"><a href="modules-kindlegen-amzn.html">kindlegen-amzn</a></li>
<li class="naventry-2nd"><a href="modules-latex-math-images.html">latex-math-images</a></li>
<li class="naventry-2nd"><a href="modules-logo.html">logo</a></li>
<li class="naventry-2nd"><a href="modules-map-style-names.html">map-style-names</a></li>
<li class="naventry-2nd"><a href="modules-markdown2hub.html">markdown2hub</a></li>
<li class="naventry-2nd"><a href="modules-mathtype-extension.html">mathtype-extension</a></li>
<li class="naventry-2nd"><a href="modules-mml-normalize.html">mml-normalize</a></li>
<li class="naventry-2nd"><a href="modules-mml2tex.html">mml2tex</a></li>
<li class="naventry-2nd"><a href="modules-nlm-stylechecker.html">nlm-stylechecker</a></li>
<li class="naventry-2nd"><a href="modules-officeopenxml-validation.html">officeopenxml-validation</a></li>
<li class="naventry-2nd"><a href="modules-pdf2fxl.html">pdf2fxl</a></li>
<li class="naventry-2nd"><a href="modules-proj-eval.html">proj-eval</a></li>
<li class="naventry-2nd"><a href="modules-pygmentize-xpl.html">pygmentize-xpl</a></li>
<li class="naventry-2nd"><a href="modules-rng-extension.html">rng-extension</a></li>
<li class="naventry-2nd"><a href="modules-schema-bits.html">schema-bits</a></li>
<li class="naventry-2nd"><a href="modules-schema-html.html">schema-html</a></li>
<li class="naventry-2nd"><a href="modules-schema-jats.html">schema-jats</a></li>
<li class="naventry-2nd"><a href="modules-schema-onix.html">schema-onix</a></li>
<li class="naventry-2nd"><a href="modules-schematron.html">schematron</a></li>
<li class="naventry-2nd"><a href="modules-sdox2hub.html">sdox2hub</a></li>
<li class="naventry-2nd"><a href="modules-semex.html">semex</a></li>
<li class="naventry-2nd"><a href="modules-split-docx.html">split-docx</a></li>
<li class="naventry-2nd"><a href="modules-stylemapper-frontend.html">stylemapper-frontend</a></li>
<li class="naventry-2nd"><a href="modules-stylemapper-lib.html">stylemapper-lib</a></li>
<li class="naventry-2nd"><a href="modules-svn-extension.html">svn-extension</a></li>
<li class="naventry-2nd"><a href="modules-tei2bits.html">tei2bits</a></li>
<li class="naventry-2nd"><a href="modules-tei2html.html">tei2html</a></li>
<li class="naventry-2nd"><a href="modules-tei2hub.html">tei2hub</a></li>
<li class="active naventry-2nd"><a href="modules-tokenized-to-tree.html">tokenized-to-tree</a></li>
<li class="naventry-2nd"><a href="modules-transpect.github.io.html">transpect.github.io</a></li>
<li class="naventry-2nd"><a href="modules-transpectdoc.html">transpectdoc</a></li>
<li class="naventry-2nd"><a href="modules-unwrap-mml.html">unwrap-mml</a></li>
<li class="naventry-2nd"><a href="modules-unzip-extension.html">unzip-extension</a></li>
<li class="naventry-2nd"><a href="modules-upload-webcomponent.html">upload-webcomponent</a></li>
<li class="naventry-2nd"><a href="modules-use-css-decorator-classes.html">use-css-decorator-classes</a></li>
<li class="naventry-2nd"><a href="modules-xerif.html">xerif</a></li>
<li class="naventry-2nd"><a href="modules-xerif-demo.html">xerif-demo</a></li>
<li class="naventry-2nd"><a href="modules-xerif-latex.html">xerif-latex</a></li>
<li class="naventry-2nd"><a href="modules-xlsx2html-frontend.html">xlsx2html-frontend</a></li>
<li class="naventry-2nd"><a href="modules-xlsx2html-lib.html">xlsx2html-lib</a></li>
<li class="naventry-2nd"><a href="modules-xml2idml.html">xml2idml</a></li>
<li class="naventry-2nd"><a href="modules-xml2tex.html">xml2tex</a></li>
<li class="naventry-2nd"><a href="modules-xproc-util.html">xproc-util</a></li>
<li class="naventry-2nd"><a href="modules-xproc3kit.html">xproc3kit</a></li>
<li class="naventry-2nd"><a href="modules-xslt-util.html">xslt-util</a></li>
</ul>
</div>
</li>
</ul>
</li>
<li class="naventry-1st"><a href="styleguide.html">Styleguide</a></li>
<li class="naventry-1st"><a href="tutorial.html">Tutorials</a></li>
</ul>
</header>
<main>
<div class="section orange" id="index-banner">
<div class="container">
<h1 id="tr-title" class="light header center-on-small-only white-text">tokenized-to-tree</h1>
<h4 id="tr-subtitle" class="light orange-text text-lighten-4 center-on-small-only">Perform text analysis and patch the results back into the source XML</h4>
</div>
</div>
<div class="container">
<div class="row">
<div class="col s12 m9 l10" id="tr-content">
<div class="card">
<div class="card-content"><span class="card-title">Repository</span><table class="informaltable">
<tbody>
<tr>
<td>Git URL</td>
<td><code class="language-markup">https://github.com/transpect/tokenized-to-tree.git</code></td>
</tr>
<tr>
<td>SVN URL</td>
<td><code class="language-markup">https://github.com/transpect/tokenized-to-tree</code></td>
</tr>
<tr>
<td>Base URI</td>
<td><code class="language-markup">http://transpect.io/tokenized-to-tree/</code></td>
</tr>
</tbody>
</table>
<p><a href="https://github.com/transpect/tokenized-to-tree" class="btn-flat orange lighten-4">Source ⬇</a></p>
</div>
</div>
<div id="ttt-line-finder" class="section scrollspy">
<h2 class="header">ttt:line-finder</h2>
<h4>Import</h4>
<pre><code class="language-markup"><p:import href="http://transpect.io/tokenized-to-tree/line-finder-module/xpl/line-finder.xpl"/></code></pre>
<h4>Dependencies</h4>
<ul>
<li>
<p><a href="modules-xproc-util transpect.github.io.html">xproc-util transpect.github.io</a></p>
</li>
</ul>
<h4>Synopsis</h4>
<pre><code class="language-markup"><ttt:line-finder xmlns:ttt="http://transpect.io/tokenized-to-tree">
<p:input port="lines" primary="true"/>
<p:input port="ttt-paras"/>
<p:input port="stylesheet"/>
<p:output port="result" primary="true"/>
<p:option name="ignore-matched-lines" required="false" select="'false'"/>
<p:option name="debug" required="false" select="'no'"/>
<p:option name="debug-dir-uri" required="false" select="'debug'"/>
</ttt:line-finder></code></pre>
</div>
<div id="ppp-postprocess-poppler" class="section scrollspy">
<h2 class="header">ppp:postprocess-poppler</h2>
<h4>Import</h4>
<pre><code class="language-markup"><p:import href="http://transpect.io/tokenized-to-tree/postprocess-poppler-module/xpl/postprocess-poppler.xpl"/></code></pre>
<h4>Dependencies</h4>
<ul>
<li>
<p><a href="modules-tokenized-to-tree.html">tokenized-to-tree</a></p>
</li>
<li>
<p><a href="modules-xproc-util transpect.github.io.html">xproc-util transpect.github.io</a></p>
</li>
</ul>
<h4>Synopsis</h4>
<pre><code class="language-markup"><ppp:postprocess-poppler xmlns:ppp="http://transpect.io/postprocess-poppler">
<p:input port="source" primary="true"/>
<p:input port="stylesheet"/>
<p:input port="param-doc"/>
<p:output port="result" primary="true"/>
<p:option name="debug" required="false" select="'no'"/>
<p:option name="debug-dir-uri" required="false" select="'debug'"/>
</ppp:postprocess-poppler></code></pre>
</div>
<div id="ttt-prepare-input" class="section scrollspy">
<h2 class="header">ttt:prepare-input</h2>
<p>
<h3 xmlns:p="http://www.w3.org/ns/xproc" xmlns:ttt="http://transpect.io/tokenized-to-tree" xmlns:tr="http://transpect.io" xmlns:c="http://www.w3.org/ns/xproc-step" xmlns:j="http://marklogic.com/json" xmlns:cx="http://xmlcalabash.com/ns/extensions" xmlns:xlink="http://www.w3.org/1999/xlink">Ignorable, Normalized and Generated Content</h3>
<p xmlns:p="http://www.w3.org/ns/xproc" xmlns:ttt="http://transpect.io/tokenized-to-tree" xmlns:tr="http://transpect.io" xmlns:c="http://www.w3.org/ns/xproc-step" xmlns:j="http://marklogic.com/json" xmlns:cx="http://xmlcalabash.com/ns/extensions" xmlns:xlink="http://www.w3.org/1999/xlink">Content that should be ignored for string counting purposes will receive a <code>role="ttt:placeholder"</code>
attribute.</p>
<p xmlns:p="http://www.w3.org/ns/xproc" xmlns:ttt="http://transpect.io/tokenized-to-tree" xmlns:tr="http://transpect.io" xmlns:c="http://www.w3.org/ns/xproc-step" xmlns:j="http://marklogic.com/json" xmlns:cx="http://xmlcalabash.com/ns/extensions" xmlns:xlink="http://www.w3.org/1999/xlink">If the content is not an element, it will be wrapped in one of the following special
elements:</p>
<dl xmlns:p="http://www.w3.org/ns/xproc" xmlns:ttt="http://transpect.io/tokenized-to-tree" xmlns:tr="http://transpect.io" xmlns:c="http://www.w3.org/ns/xproc-step" xmlns:j="http://marklogic.com/json" xmlns:cx="http://xmlcalabash.com/ns/extensions" xmlns:xlink="http://www.w3.org/1999/xlink">
<dt>ttt:comment</dt>
<dd>Contains a comment that will be invisible to the string-length counting process but
will be re-inserted after
token tagging has been inserted.</dd>
<dt>ttt:ignorable-text</dt>
<dd>Contains text that will be invisible to the string-length counting process but will
be re-inserted after
token tagging has been inserted.</dd>
<dt>ttt:pi</dt>
<dd>Contains a processing instruction that will be invisible to the string-length counting
process but will be
re-inserted after token tagging has been inserted.</dd>
</dl>
<p xmlns:p="http://www.w3.org/ns/xproc" xmlns:ttt="http://transpect.io/tokenized-to-tree" xmlns:tr="http://transpect.io" xmlns:c="http://www.w3.org/ns/xproc-step" xmlns:j="http://marklogic.com/json" xmlns:cx="http://xmlcalabash.com/ns/extensions" xmlns:xlink="http://www.w3.org/1999/xlink">These placeholder elements will then be emptied for the document that is output on
the <code>result</code> port.
The full elements will be retained in the document that is output on the <code>with-ids</code> port.</p>
<p xmlns:p="http://www.w3.org/ns/xproc" xmlns:ttt="http://transpect.io/tokenized-to-tree" xmlns:tr="http://transpect.io" xmlns:c="http://www.w3.org/ns/xproc-step" xmlns:j="http://marklogic.com/json" xmlns:cx="http://xmlcalabash.com/ns/extensions" xmlns:xlink="http://www.w3.org/1999/xlink">After the result document has been processed (tokenization, analysis, creating token
markup and attaching
the analysis results to the marked-up tokens), the empty placeholder elements will
be replaced with their
original content.</p>
<p xmlns:p="http://www.w3.org/ns/xproc" xmlns:ttt="http://transpect.io/tokenized-to-tree" xmlns:tr="http://transpect.io" xmlns:c="http://www.w3.org/ns/xproc-step" xmlns:j="http://marklogic.com/json" xmlns:cx="http://xmlcalabash.com/ns/extensions" xmlns:xlink="http://www.w3.org/1999/xlink">For this to work, all placeholder elements need to have IDs. A generated <code>xml:id</code> attribute starting
with 'NOID_' will be added if the future placeholder element does not have an ID yet.
The 'NOID_' IDs will
be removed during the merging step, <code>ttt:merge-results</code> (file <code>ttt-5-expand-placeholders.xpl</code>).</p>
<p xmlns:p="http://www.w3.org/ns/xproc" xmlns:ttt="http://transpect.io/tokenized-to-tree" xmlns:tr="http://transpect.io" xmlns:c="http://www.w3.org/ns/xproc-step" xmlns:j="http://marklogic.com/json" xmlns:cx="http://xmlcalabash.com/ns/extensions" xmlns:xlink="http://www.w3.org/1999/xlink">There is an additional type of <code>ttt:placeholder</code> element:</p>
<dl xmlns:p="http://www.w3.org/ns/xproc" xmlns:ttt="http://transpect.io/tokenized-to-tree" xmlns:tr="http://transpect.io" xmlns:c="http://www.w3.org/ns/xproc-step" xmlns:j="http://marklogic.com/json" xmlns:cx="http://xmlcalabash.com/ns/extensions" xmlns:xlink="http://www.w3.org/1999/xlink">
<dt>ttt:normalized-space</dt>
<dd>An element that contains exactly one space character, plus an <code>@original</code> attribute. For counting
and analysis purposes, it will look like an ordinary single space character. The original
space-like content will
be restored from the <code>@original</code> attribute during the merging phase.</dd>
</dl>
<p xmlns:p="http://www.w3.org/ns/xproc" xmlns:ttt="http://transpect.io/tokenized-to-tree" xmlns:tr="http://transpect.io" xmlns:c="http://www.w3.org/ns/xproc-step" xmlns:j="http://marklogic.com/json" xmlns:cx="http://xmlcalabash.com/ns/extensions" xmlns:xlink="http://www.w3.org/1999/xlink">Finally, there is the opposite case of ignored content, which is generated content.
Scenario: In a TEI critical
apparatus, the text-critical notes are rendered in a distinct section. Each note is
preceded by the page and
line(s) that it applies to, plus some words of context. The context will be rendered
via the note’s
<code>@target</code> attribute. If the task is to retrofit PDF line breaks into the source, the PDF lines
must
be matched against the notes. This can be done either by inserting a regex that would
match any page/line numbers,
followed by any context phrase, or by at least including the context phrase in the
note element so that it will
match the PDF line (modulo page/line numbers). Then we need to temporarily augment
the note with the context phrase,
but we need to make sure that it will be removed after the line breaks have been marked
up.</p>
<p xmlns:p="http://www.w3.org/ns/xproc" xmlns:ttt="http://transpect.io/tokenized-to-tree" xmlns:tr="http://transpect.io" xmlns:c="http://www.w3.org/ns/xproc-step" xmlns:j="http://marklogic.com/json" xmlns:cx="http://xmlcalabash.com/ns/extensions" xmlns:xlink="http://www.w3.org/1999/xlink">For this purpose, an element</p>
<dl xmlns:p="http://www.w3.org/ns/xproc" xmlns:ttt="http://transpect.io/tokenized-to-tree" xmlns:tr="http://transpect.io" xmlns:c="http://www.w3.org/ns/xproc-step" xmlns:j="http://marklogic.com/json" xmlns:cx="http://xmlcalabash.com/ns/extensions" xmlns:xlink="http://www.w3.org/1999/xlink">
<dt>ttt:generated</dt>
<dd>Mostly useful for line number retrofitting applications. Contains text that is not
present at this location in
the source XML. It can be a placeholder for a page number or for text from another
place in the text, for example
from a lemma that is repeated at the beginning of a note.</dd>
</dl>
<h3 xmlns:p="http://www.w3.org/ns/xproc" xmlns:ttt="http://transpect.io/tokenized-to-tree" xmlns:tr="http://transpect.io" xmlns:c="http://www.w3.org/ns/xproc-step" xmlns:j="http://marklogic.com/json" xmlns:cx="http://xmlcalabash.com/ns/extensions" xmlns:xlink="http://www.w3.org/1999/xlink">Side-by-Side Format</h3>
<h3 xmlns:p="http://www.w3.org/ns/xproc" xmlns:ttt="http://transpect.io/tokenized-to-tree" xmlns:tr="http://transpect.io" xmlns:c="http://www.w3.org/ns/xproc-step" xmlns:j="http://marklogic.com/json" xmlns:cx="http://xmlcalabash.com/ns/extensions" xmlns:xlink="http://www.w3.org/1999/xlink">Tokenization Markup</h3>
<p xmlns:p="http://www.w3.org/ns/xproc" xmlns:ttt="http://transpect.io/tokenized-to-tree" xmlns:tr="http://transpect.io" xmlns:c="http://www.w3.org/ns/xproc-step" xmlns:j="http://marklogic.com/json" xmlns:cx="http://xmlcalabash.com/ns/extensions" xmlns:xlink="http://www.w3.org/1999/xlink">The tokens will be marked up with <code>ttt:token</code> and <code>ttt:space</code> elements. These elements
cover the whole unexpanded paragraph.</p>
</p>
<h4>Import</h4>
<pre><code class="language-markup"><p:import href="http://transpect.io/tokenized-to-tree/xpl/ttt-1-prepare-input.xpl"/></code></pre>
<h4>Dependencies</h4>
<ul>
<li>
<p><a href="modules-xproc-util transpect.github.io.html">xproc-util transpect.github.io</a></p>
</li>
</ul>
<h4>Synopsis</h4>
<pre><code class="language-markup"><ttt:prepare-input xmlns:ttt="http://transpect.io/tokenized-to-tree">
<p:input port="source" primary="true"/>
<p:input port="stylesheet"/>
<p:output port="result" primary="true"/>
<p:output port="with-ids"/>
<p:option name="map-higher-unicode-planes" select="'no'"/>
<p:option name="debug" required="false" select="'no'"/>
<p:option name="debug-dir-uri" required="false" select="'debug'"/>
</ttt:prepare-input></code></pre>
</div>
<div id="ttt-process-paras" class="section scrollspy">
<h2 class="header">ttt:process-paras</h2>
<h4>Import</h4>
<pre><code class="language-markup"><p:import href="http://transpect.io/tokenized-to-tree/xpl/ttt-3-integrate-tokenizer-results.xpl"/></code></pre>
<h4>Dependencies</h4>
<ul>
<li>
<p><a href="modules-xproc-util transpect.github.io.html">xproc-util transpect.github.io</a></p>
</li>
<li>
<p><a href="modules-tokenized-to-tree.html">tokenized-to-tree</a></p>
</li>
</ul>
<h4>Synopsis</h4>
<pre><code class="language-markup"><ttt:process-paras xmlns:ttt="http://transpect.io/tokenized-to-tree">
<p:input port="source" primary="true"/>
<p:input port="patch-token-stylesheet"/>
<p:input port="rng-schema"/>
<p:output port="result" primary="true"/>
<p:option name="debug" required="false" select="'no'"/>
<p:option name="debug-dir-uri" required="false" select="'debug'"/>
<p:option name="milestones-only" select="'no'"/>
<p:option name="map-higher-unicode-planes" select="'no'"/>
</ttt:process-paras></code></pre>
</div>
<div id="ttt-merge-results" class="section scrollspy">
<h2 class="header">ttt:merge-results</h2>
<h4>Import</h4>
<pre><code class="language-markup"><p:import href="http://transpect.io/tokenized-to-tree/xpl/ttt-5-expand-placeholders.xpl"/></code></pre>
<h4>Dependencies</h4>
<ul>
<li>
<p><a href="modules-xproc-util transpect.github.io.html">xproc-util transpect.github.io</a></p>
</li>
</ul>
<h4>Synopsis</h4>
<pre><code class="language-markup"><ttt:merge-results xmlns:ttt="http://transpect.io/tokenized-to-tree">
<p:input port="source" primary="false"/>
<p:input port="patched-paras" primary="true"/>
<p:input port="stylesheet"/>
<p:input port="params"/>
<p:output port="result" primary="true"/>
<p:option name="debug" required="false" select="'no'"/>
<p:option name="debug-dir-uri" required="false" select="'debug'"/>
</ttt:merge-results></code></pre>
</div>
<div id="d42861e209" class="status">
<h2 class="header status"></h2>
<p class="last-updated">GitHub sync date: 2025-01-08+01:00</p>
</div>
</div>
<div class="col hide-on-small-only m3 l2">
<div class="toc-wrapper pin-top" style="top: 0px;">
<div style="height: 1px;">
<ul id="tr-toc" class="table-of-contents">
<li><a href="#ttt-line-finder" class="active">ttt:line-finder</a></li>
<li><a href="#ppp-postprocess-poppler">ppp:postprocess-poppler</a></li>
<li><a href="#ttt-prepare-input">ttt:prepare-input</a></li>
<li><a href="#ttt-process-paras">ttt:process-paras</a></li>
<li><a href="#ttt-merge-results">ttt:merge-results</a></li>
<li><a href="#d42861e209"></a></li>
</ul>
</div>
</div>
</div>
</div>
</div>
</main>
<footer class="page-footer grey">
<div class="container">
<div class="row">
<div id="tr-footer-1" class="col s12 m6 l4">
<h5 class="white-text">GitHub</h5>
<p class="white-text">Checkout the transpect repositories on GitHub and use transpect in your project.
Give us feedback, create issues and make pull requests.</p>
<p><a target="_blank" href="https://github.com/transpect" class="btn red">GitHub</a></p>
</div>
<div id="tr-footer-2" class="col s12 m6 l4">
<h5 class="white-text">Twitter</h5>
<p class="white-text">News from le-tex’ XML machine room. Mostly related to
XSLT/XPath 2+, XProc, Relax NG, Schematron, and our
open-source framework transpect.
</p>
<p><a target="_blank" href="https://twitter.com/intent/user?screen_name=letexml" class="btn red">Follow @letexml</a></p>
</div>
<div id="tr-footer-3" class="col s12 m6 l4"><a target="_blank" href="http://www.le-tex.de"><object id="letex-logo" type="image/svg+xml" data="icons/logo-le-tex.svg">Your browser does not support SVG</object></a></div>
</div>
</div>
<div class="footer-copyright grey darken-1">
<div class="container">
<div class="row">
<div id="tr-footer-copyright-1" class="col s12 m6 l6">
<p class="white-text left">le-tex publishing services GmbH, <span class="current-year">2025</span></p>
</div>
<div id="tr-footer-copyright-2" class="col s12 m6 l6">
<p>
<a class="white-text right" href="http://www.le-tex.de">http://www.le-tex.de</a>
<a rel="me" href="https://fosstodon.org/@letexml" aria-label="Mastodon"></a>
</p>
</div>
</div>
</div>
</div>
</footer><script type="text/javascript" src="https://code.jquery.com/jquery-2.1.1.min.js"></script><script type="text/javascript" src="js/prism.js"></script><script type="text/javascript" src="js/materialize.min.js"></script><script src="js/init.js"></script></body>
</html>