',
'',
)
notetext = notetext.replace(
"", ""
)
return "{}{}{}".format(tag[0], notetext, tag[1])
text = NOTERE.sub(simplerepl, text, 0)
# process additional footnote tags if present
for _ in [r"\f", r"\x", r"\+f", r"\+x"]:
if _ in text:
text = notefix(text)
# handle fp tags
if r"\fp " in text:
# xmllint says this works and validates.
text = text.replace("\uFDD2", "")
text = text.replace("\uFDD3", "
")
text = text.replace(r"\fp ", r"")
else:
text = text.replace("\uFDD2", "").replace("\uFDD3", "")
# study bible index categories
if r"\cat " in text:
text = text.replace(r"\cat ", r'')
# return our processed text
return text
def c2o_specialfeatures(text):
"""Process special features."""
def simplerepl(match):
"""Simple regex replacement helper function."""
matchtag = match.group("tag")
tag = FEATURETAGS[matchtag]
rawosis = match.group("osis")
attributetext = None
osis, attributetext, attributes, isvalid = {
True: parseattributes(matchtag, rawosis),
False: (rawosis, None, dict(), True),
}["|" in rawosis]
del isvalid # make pylint happy
osis2 = osis
# handle w tag attributes
tag2 = None
if matchtag in [r"\w", r"\+w"]:
if "lemma" in attributes.keys():
osis2 = attributes["lemma"]
if "strong" in attributes.keys():
tag2 = STRONGSTAG
tmp = " ".join(
[
"strong:{}".format(_.strip())
for _ in attributes["strong"].split(",")
]
)
strong1 = 'lemma="{}"'.format(tmp)
strong2 = ""
if "x-morph" in attributes.keys():
if tag2 is not None:
strong2 = ' morph="{}"'.format(attributes["x-morph"])
# TODO: improve processing of tag attributes
if tag[0] == "" and tag2 is None:
# limited filtering of index entry contents...
if '' in osis2:
osis2 = osis2.replace(
'', ""
)
osis2 = osis2.replace("", "")
outtext = "{}{}".format(osis, tag[1].format(osis2))
elif tag2 is not None:
# process strongs
outtext = "{}{}{}".format(
tag2[0].format(strong1, strong2), osis, tag2[1]
)
else:
outtext = "{}{}{}".format(tag[0], osis, tag[1])
if attributetext is not None:
# problems can occur when strongs numbers are present...
# this avoids those problems.
if matchtag not in [r"\w", r"\+w"]:
outtext = "{}{}".format(
outtext,
"".format(attributetext),
)
return outtext
def figtags(text):
"""Process fig tags."""
text = text.replace(r"\fig ", "\n\\fig ")
text = text.replace(r"\fig*", "\\fig*\n")
tlines = text.split("\n")
for i in enumerate(tlines):
if tlines[i[0]].startswith(r"\fig "):
# old style \fig handling
# \fig DESC|FILE|SIZE|LOC|COPY|CAP|REF\fig*
if len(tlines[i[0]][5:-5].split("|")) > 2:
fig = tlines[i[0]][5:-5].split("|")
figref = ""
fig[0] = {
False: "\n".format(fig[0]),
True: fig[0],
}[not fig[0]]
fig[1] = {
False: ' src="{}"'.format(fig[1]),
True: fig[1],
}[not fig[1]]
fig[2] = {
False: ' size="{}"'.format(fig[2]),
True: fig[2],
}[not fig[2]]
fig[3] = {
False: "\n".format(fig[3]),
True: fig[3],
}[not fig[3]]
fig[4] = {
False: ' rights="{}"'.format(fig[4]),
True: fig[4],
}[not fig[4]]
fig[5] = {
False: "{}\n".format(fig[5]),
True: fig[5],
}[not fig[5]]
# this is likely going to be very broken without
# further processing of the references.
if fig[6]:
figref = "{}\n".format(
'type="annotateRef"', fig[6]
)
fig[6] = ' annotateRef="{}"'.format(fig[6])
# new style \fig handling
else:
figattr = parseattributes(r"\fig", tlines[i[0]][5:-5])
fig = []
figparts = {
"alt": "\n",
"src": ' src="{}"',
"size": ' size="{}"',
"loc": "\n",
"copy": ' rights="{}"',
}
for _ in ["alt", "src", "size", "loc", "copy"]:
fig.append(
{
True: figparts[_].format(figattr[2][_]),
False: "",
}[_ in figattr[2]]
)
# caption absent in new style fig attributes
fig.append("")
# this is likely going to be very broken without
# further processing of the references.
if "ref" in figattr[2]:
figref = "{}{}{}\n".format(
'',
figattr[2]["ref"],
"\n",
)
fig.append(
' annotateRef="{}"'.format(figattr[2]["ref"])
)
else:
figref = ""
fig.append("")
# build our osis figure tag
tlines[i[0]] = "".join(
[
fig[0],
fig[3],
"\n",
figref,
fig[5],
"",
]
)
return "".join(tlines)
def milestonequotes(text):
"""Handle usfm milestone quotations."""
for _ in [
r"\qt-",
r"\qt1-",
r"\qt2-",
r"\qt3-",
r"\qt4-",
r"\qt5-",
]:
if _ in text:
text = text.replace(_, "\n{}".format(_))
text = text.replace(r"\*", "\\*\n")
tlines = text.split("\n")
for i in enumerate(tlines):
# make sure we're processing milestone \qt tags
if tlines[i[0]].endswith(r"\*"):
for j in (
r"\qt-",
r"\qt1-",
r"\qt2-",
r"\qt3-",
r"\qt4-",
r"\qt5-",
):
if tlines[i[0]].startswith(j):
# replace with milestone osis tags
newline = ""
tlines[i[0]] = tlines[i[0]].strip().replace(r"\*", "")
tag, _, qttext = tlines[i[0]].partition(" ")
# milestone start tag
if tag.endswith(r"-s"):
(
_,
attributetext,
attributes,
isvalid,
) = parseattributes(r"\qt-s", qttext)
del isvalid, attributetext # make pylint happy
newline = r"",
)
# milestone end tag
elif tag.endswith(r"-e"):
(
_,
attributetext,
attributes,
isvalid,
) = parseattributes(r"\qt-e", qttext)
newline = r"",
)
# replace line with osis milestone tag
if newline != "":
tlines[i[0]] = newline
# rejoin lines
return "".join(tlines)
text = SPECIALFEATURESRE.sub(simplerepl, text, 0)
if r"\fig" in text:
text = figtags(text)
# Process usfm milestone quotation tags... up to 5 levels.
for _ in [r"\qt-", r"\qt1-", r"\qt2-", r"\qt3-", r"\qt4-", r"\qt5-"]:
if _ in text:
text = milestonequotes(text)
return text
def c2o_ztags(text):
"""Process z tags that have both a start and end marker."""
def simplerepl(match):
"""Simple regex replacement helper function."""
return '{}'.format(
match.group("tag").replace(r"\z", ""), match.group("osis")
)
text = ZTAGSRE.sub(simplerepl, text, 0)
# milestone z tags… this may need more work…
if r"\z" in text:
words = text.split(" ")
for i in enumerate(words):
if i[1].startswith(r"\z"):
words[i[0]] = ''.format(
i[1].replace(r"\z", "")
)
text = " ".join(words)
return text
def c2o_chapverse(lines, bookid):
"""Process chapter and verse tags."""
def verserange(text):
"""Generate list for verse ranges."""
low, high = text.split("-")
retval = ""
try:
retval = [str(_) for _ in range(int(low), int(high) + 1)]
except ValueError:
end1 = ""
end2 = ""
if low[-1] in ["a", "b", "A", "B"]:
end1 = low[-1]
low = "".join(low[:-1])
if high[-1] in ["a", "b", "A", "B"]:
end2 = high[-1]
high = "".join(high[:-1])
retval = [str(_) for _ in range(int(low), int(high) + 1)]
if end1 != "":
retval[0] = "!".join([retval[0], end1])
if end2 != "":
retval[-1] = "!".join([retval[-1], end2])
return retval
# chapter and verse numbers
closerlist = [
_ for _ in range(len(lines)) if lines[_].startswith(r"" in tmp[2]:
cnum = cnum.replace(""
]
for _ in enumerate(lines):
# fix SIDEBAR
if "SIDEBAR" in lines[_[0]]:
lines[_[0]] = lines[_[0]].replace(
"", ''
)
lines[_[0]] = lines[_[0]].replace("", "
")
# Convert unhandled vp tags, to milestones...
while r"\vp " in lines[_[0]]:
vpnum = VPRE.search(lines[_[0]]).group("num")
lines[_[0]] = VPRE.sub(
''.format(vpnum),
lines[_[0]],
1,
)
# adjust some tags for postprocessing purposes.
i = len(lines)
while i > 0:
i -= 1
# remove empty l tags if present.
if lines[i] == ' ':
del lines[i]
continue
# move lb to it's own line
if lines[i].endswith('
'):
lines.insert(i + 1, '
')
lines[i] = lines[i].rpartition('
')[0].strip()
# move lg to it's own line
if lines[i].endswith(""):
lines.insert(i + 1, "")
lines[i] = lines[i].rpartition("")[0].strip()
# swap lb and lg end tag when lg end tag follows lb.
i = len(lines)
while i > 0:
i -= 1
try:
if lines[i] == '
' and lines[i + 1] == "":
lines[i], lines[i + 1] = (lines[i + 1], lines[i])
except IndexError:
pass
# adjust placement of some verse end tags...
for i in [
_ for _ in range(len(lines)) if lines[_].startswith("" and lines[i - 2] == "":
lines.insert(i - 2, lines.pop(i))
for i, j in [
(x, y)
for x in [
"",
"",
"",
"",
]
for y in [
_ for _ in range(len(lines)) if lines[_].startswith("",
"",
"",
"",
"",
]
for y in [
_ for _ in range(len(lines)) if lines[_].startswith("", ""]
for y in [
_ for _ in range(len(lines)) if lines[_].startswith(""):
lines[i - 1] = "{}{}".format(
lines[i - 1].rpartition("<")[0], lines[i]
)
lines[i] = ""
for i, j in [
(x, y)
for x in ["")
]:
if (
lines[i + 1].startswith("")
and lines[i + 2].startswith("":
lines.insert(j - 1, lines.pop(j))
except IndexError:
pass
# adjust placement of some chapter start tags
for i in [
_ for _ in range(len(lines)) if lines[_].startswith("" and lines[i + 2].startswith(""
and "chapterLabel" in lines[i + 2]
and lines[i + 3].startswith(""
and lines[i + 2] == ""
and lines[i + 3].startswith("]+> ?", lines[i]) and lines[
i + 1
].startswith("", ""))
lines[i] = ""
elif re.match("]+> ?]+> ?", lines[i]):
lines.insert(i + 1, lines[i].replace(" ", ""))
lines[i] = " "
except IndexError:
pass
lines = [_ for _ in lines if _ != ""]
# -- # -- # -- #
# selah processing sometimes does weird things with l and lg tags
# that needs to be fixed.
for i in [
_ for _ in range(len(lines)) if lines[_].startswith("")
and lines[i + 1] == ""
and lines[i - 1].startswith("")
):
lines[i - 3] = lines[i - 3][:-3]
lines[i - 2] = "{}{}".format(lines[i - 2], "")
lines[i] = lines[i][:-4]
lines[i + 1] = ""
# additional postprocessing for l and lg tags
for i in [
_ for _ in range(len(lines)) if lines[_].startswith("")
and lines[i - 1].startswith(""
):
lines[i - 2] = "{}".format(lines[i - 2])
lines[i] = lines[i][:-4]
lines[i + 1] = ""
# done postprocessing of lines
return lines
def convert_to_osis(text, bookid="TEST"):
"""Convert usfm file to osis."""
# ---------------------------------------------------------------------- #
description = []
# ---------------------------------------------------------------------- #
# split text into lines for processing
lines = text.split("\n")
# mark introduction endings...
for _ in [r"\ib", r"\ie", r"\il", r"\im", r"\io", r"\ip", r"\iq", r"\is"]:
if _ in text:
lines = markintroend(lines)
break
for i in enumerate(lines):
# preprocessing and special spacing... if necessary
for _ in ["&", "<", ">", "~", r"//", r"\pb"]:
if _ in lines[i[0]]:
lines[i[0]] = c2o_preprocess(lines[i[0]])
break
# identification
lines[i[0]], description = c2o_identification(lines[i[0]], description)
# character style formatting
lines[i[0]] = c2o_noterefmarkers(lines[i[0]])
lines[i[0]] = c2o_specialtext(lines[i[0]])
# special features if present, and stray \xt tags that were missed.
for _ in [r"\ndx", r"\pro", r"\w", r"\+w", r"\fig", r"\xt", r"\+xt"]:
if _ in lines[i[0]]:
lines[i[0]] = c2o_specialfeatures(lines[i[0]])
break
for _ in [r"\qt-", r"\qt1-", r"\qt2-", r"\qt3-", r"\qt4-", r"\qt5-"]:
if _ in lines[i[0]]:
lines[i[0]] = c2o_specialfeatures(lines[i[0]])
break
# z tags if present
if r"\z" in lines[i[0]]:
lines[i[0]] = c2o_ztags(lines[i[0]])
# paragraph style formatting.
lines[i[0]] = c2o_titlepar(lines[i[0]], bookid)
# process words of Jesus
if r"\wj" in text:
lines = c2o_processwj2(lines)
# postprocessing of poetry, lists, tables, and sections
# to add missing tags and div's.
lines = c2o_fixgroupings(lines)
# process chapter/verse markers
lines = [_.strip() for _ in " ".join(lines).split("\ufdd0")]
lines = c2o_chapverse(lines, bookid)
# postprocessing to fix some issues that may be present
lines = c2o_postprocess(lines)
descriptiontext = "\n".join(description)
# rejoin lines after processing
return ("\n".join([_ for _ in lines if _ != ""]), descriptiontext)
# -------------------------------------------------------------------------- #
def doconvert(args):
"""Convert our text and return our results."""
text = args
# convert cl lines to form that follows each chapter marker instead of
# form that precedes first chapter.
if r"\cl " in text:
text = convertcl(text)
# decode and reflow our text
newtext = reflow(text)
# get book id. use TEST if none present.
bookid = getbookid(newtext)
if bookid is not None:
if bookid.startswith("* "):
LOG.error("Book id naming issue - %s", bookid.replace("* ", ""))
else:
bookid = "TEST"
# convert file to osis
LOG.info("... Processing %s ...", bookid)
newtext, descriptiontext = convert_to_osis(newtext, bookid)
return (bookid, descriptiontext, newtext)
def processfiles(args):
"""Process usfm files specified on command line."""
books = {}
descriptions = {}
booklist = []
files = []
# get username from operating system
username = {True: os.getenv("LOGNAME"), False: os.getenv("USERNAME")}[
os.getenv("USERNAME") is None
]
# read all files
LOG.info("Reading files... ")
for fname in args.file:
# read our text files
with open(fname, "rb") as ifile:
text = ifile.read()
# strip whitespace from beginning and end of file
text = text.strip()
# get encoding. Abort processing if we don't know the encoding.
# default to utf-8-sig encoding if no encoding is specified.
try:
if args.e is not None:
bookencoding = codecs.lookup(args.e).name
else:
bookencoding = getencoding(text)
if bookencoding is not None:
if bookencoding == "65001 - Unicode (UTF-8)":
bookencoding = "utf-8-sig"
else:
bookencoding = codecs.lookup(bookencoding).name
else:
bookencoding = "utf-8-sig"
# use utf-8-sig in place of utf-8 encoding to eliminate errors that
# may occur if a Byte Order Mark is present in the input file.
if bookencoding == "utf-8":
bookencoding = "utf-8-sig"
except LookupError:
LOG.error("ERROR: Unknown encoding... aborting conversion.")
LOG.error(r" \ide line for %s says --> %s", fname, bookencoding)
sys.exit()
# convert file to unicode and add contents to list for processing...
files.append(text.decode(bookencoding))
# set number of processes to use while processing file contents
numprocesses = 1
if not args.d and HAVEMULTIPROCESSING:
try:
numprocesses = multiprocessing.cpu_count()
except NotImplementedError:
numprocesses = 1
# process file contents
filelist = files
results = []
LOG.info("Processing files...")
if numprocesses == 1:
results = [doconvert(_) for _ in filelist]
else:
try:
with multiprocessing.Pool(numprocesses) as pool:
results = pool.imap(doconvert, filelist)
pool.close()
pool.join()
except AttributeError:
# pylint: disable=no-member
with closing(multiprocessing.Pool(numprocesses)) as pool:
results = pool.imap(doconvert, filelist)
pool.close()
pool.join()
# store results
for bookid, descriptiontext, newtext in results:
# store our converted text for output
if bookid != "TEST":
if bookid in NONCANONICAL:
books[bookid] = '\n{}\n \n\n'.format(
NONCANONICAL[bookid], newtext
)
else:
books[
bookid
] = '\n{}\n \n\n'.format(
bookid, 'canonical="true"', newtext
)
descriptions[bookid] = descriptiontext
booklist.append(bookid)
else:
if bookid in books.keys():
books[bookid] = "{}\n{}".format(books[bookid], newtext)
descriptions[bookid] = "{}\n{}".format(
books[bookid], descriptiontext
)
else:
books[bookid] = newtext
descriptions[bookid] = descriptiontext
if "TEST" not in booklist:
booklist.append("TEST")
# ## Get order for books...
if args.s == "none":
tmp = "\n".join([books[_] for _ in booklist])
tmp2 = [descriptions[_] for _ in booklist]
elif args.s == "canonical":
tmp = "\n".join(
[books[_] for _ in CANONICALORDER if _ in books.keys()]
)
tmp2 = [descriptions[_] for _ in CANONICALORDER if _ in books.keys()]
else:
with open("order-{}.txt".format(args.s), "r") as order:
bookorder = order.read()
bookorder = [
_
for _ in bookorder.split("\n")
if _ != "" and not _.startswith("#")
]
tmp = "\n".join([books[_] for _ in bookorder if _ in books.keys()])
tmp2 = [descriptions[_] for _ in bookorder if _ in books.keys()]
# check for strongs presence in osis
strongsheader = {True: STRONGSWORK, False: ""}["", " "),
(" ", ""),
(" ", ""),
(" ` at the end of the reference text to make it easy to locate where the problem occurred.
## Example config file
The following is an example config file I used when I was testing this script. NOTE: All parts of the config file are required.
---
```markdown
[DEFAULT]
#
# WARNING! Make sure that all of these default values are different. If any of
# them are duplicated, it will cause problems for the reference parsing.
#
#
# SEPM specifies the character which separates multiple references
# SEPC specifies the character which separates chapters from verses
# SEPP specifies the character which separates multiple parts of a reference.
# That is, multiple verses or verse ranges.
# SEPR specifies the character which separates the first part of a verse
# range from the 2nd part of a verse range
# SEPRNORM is a list of characters that will be converted to SEPR during
# processing of verse ranges to make things easier.
#
SEPM = ;
SEPC = :
SEPP = ,
SEPR = -
SEPRNORM = –—
[ABBR]
#
# This section contains comma separated book names and abbreviations used for
# processing cross references.
#
Gen = Genesis, Gen
Exod = Exodus, Exo
Lev = Leviticus, Lev
Num = Numbers, Num
Deut = Deuteronomy, Deu
Josh = Joshua, Jos
Judg = Judges, Jdg
Ruth = Ruth, Rut
1Sam = 1 Samuel, 1Sa
2Sam = 2 Samuel, 2Sa
1Kgs = 1 Kings, 1Ki
2Kgs = 2 Kings, 2Ki
1Chr = 1 Chronicles, 1Ch
2Chr = 2 Chronicles, 2Ch
PrMan = Prayer of Manasses, Man
Jub =
1En =
Ezra = Ezra, Ezr
Neh = Nehemiah, Neh
Tob = Tobit, Tob
Jdt = Judith, Jdt
Esth = Esther, Est
EsthGr = Esther Greek, ESG
1Meq =
2Meq =
3Meq =
Job = Job, Job
Ps = Psalms, Psalm
AddPs = Psalm 151, Ps151
5ApocSyrPss =
Odes =
Prov = Proverbs, Pro
Reproof =
Eccl = Ecclesiastes, Ecc
Song = Song of Solomon, Sng
Wis = Wisdom of Solomon, Wisdom, Wis
Sir = Sirach, Sir
PssSol =
Isa = Isaiah, Isa
Jer = Jeremiah, Jer
Lam = Lamentations, Lam
Bar = Baruch, Bar
EpJer =
2Bar =
EpBar =
4Bar =
Ezek = Ezekiel, Ezk
Dan = Daniel, Dan
DanGr = Daniel (Greek), DanielG
PrAzar =
Sus =
Bel =
Hos = Hosea, Hos
Joel = Joel, Jol
Amos = Amos, Amo
Obad = Obadiah, Oba
Jonah = Jonah, Jon
Mic = Micah, Mic
Nah = Nahum, Nam
Hab = Habakkuk, Hab
Zeph = Zephaniah, Zep
Hag = Haggai, Hag
Zech = Zechariah, Zec
Mal = Malachi, Mal
1Esd = 1 Esdras, 1Es
2Esd = 2 Esdras, 2Es
4Ezra =
5Ezra =
6Ezra =
1Macc = 1 Maccabees, 1Ma
2Macc = 2 Maccabees, 2Ma
3Macc = 3 Maccabees, 3Ma
4Macc = 4 Maccabees, 4Ma
Matt = Matthew, Mat
Mark = Mark, Mrk
Luke = Luke, Luk
John = John, Jhn
Acts = Acts, Act
Rom = Romans, Rom
1Cor = 1 Corinthians, 1Co
2Cor = 2 Corinthians, 2Co
Gal = Galatians, Gal
Eph = Ephesians, Eph
Phil = Philippians, Php
Col = Colossians, Col
1Thess = 1 Thessalonians, 1Th
2Thess = 2 Thessalonians, 2Th
1Tim = 1 Timothy, 1Ti
2Tim = 2 Timothy, 2Ti
Titus = Titus, Tit
Phlm = Philemon, Phm
Heb = Hebrews, Heb
Jas = James, Jas
1Pet = 1 Peter, 1Pe
2Pet = 2 Peter, 2Pe
1John = 1 John, 1Jn
2John = 2 John, 2Jn
3John = 3 John, 3Jn
Jude = Jude, Jud
Rev = Revelation, Rev
EpLao =
```
u2o-0.7/unmaintained/orefs.py 0000775 0000000 0000000 00000044763 14077653007 0016263 0 ustar 00root root 0000000 0000000 #!/usr/bin/python3
# -*- coding: utf-8 -*-
"""Helper script to add osisRef attributes to reference tags in osis files."""
from __future__ import print_function, unicode_literals
import sys
import argparse
import os
import os.path
import datetime
import configparser
from collections import OrderedDict
import re
# -------------------------------------------------------------------------- #
# NOTES AND KNOWN LIMITATIONS:
#
# For automatic determination of book abbreviations, the presence
# of toc2 and toc3 tags is required. If these are not present, there
# won't be any way to determine the book part for the osisRef attribute.
#
# ALTERNATIVELY, a separate config file can be used to manually specify
# book names and abbreviations. See the README-orefs.md file for more
# information and example config file contents.
#
# A beginning config file can be automatically generated now using the
# following command:
#
# orefs.py -i OSISFILE -o CONFIGFILE -c create
#
# SPECIAL CHARACTERS USED DURING PROCESSING:
# uFDEA - marks start of book abbreviation during processing
# uFDEB - marks end of book abbreviation during processing
# VARIABLES THAT CONTROL HOW VERSE REFERENCES ARE PARSED
SEPM = ";" # separates multiple references
SEPC = ":" # separates chapter from verse
SEPP = "," # separates multiple verses or verse ranges
SEPR = "-" # separates verse ranges
SEPRNORM = ["–", "—"] # used to normalize verse ranges
DIGITS = "0123456789" # default set of decimal digits to use
DIGITTABLE = str.maketrans(DIGITS, DIGITS) # digit translation table
# book tag format string
BTAG = "\uFDEA{}\uFDEB"
# list of OSIS bible books
BOOKLIST = [
'Gen', 'Exod', 'Lev', 'Num', 'Deut', 'Josh', 'Judg', 'Ruth', '1Sam',
'2Sam', '1Kgs', '2Kgs', '1Chr', '2Chr', 'PrMan', 'Jub', '1En', 'Ezra',
'Neh', 'Tob', 'Jdt', 'Esth', 'EsthGr', '1Meq', '2Meq', '3Meq', 'Job',
'Ps', 'AddPs', '5ApocSyrPss', 'Odes', 'Prov', 'Reproof', 'Eccl', 'Song',
'Wis', 'Sir', 'PssSol', 'Isa', 'Jer', 'Lam', 'Bar', 'EpJer', '2Bar',
'EpBar', '4Bar', 'Ezek', 'Dan', 'DanGr', 'PrAzar', 'Sus', 'Bel', 'Hos',
'Joel', 'Amos', 'Obad', 'Jonah', 'Mic', 'Nah', 'Hab', 'Zeph', 'Hag',
'Zech', 'Mal',
'1Esd', '2Esd', '4Ezra', '5Ezra', '6Ezra', '1Macc', '2Macc', '3Macc',
'4Macc',
'Matt', 'Mark', 'Luke', 'John', 'Acts', 'Rom', '1Cor', '2Cor', 'Gal',
'Eph', 'Phil', 'Col', '1Thess', '2Thess', '1Tim', '2Tim', 'Titus', 'Phlm',
'Heb', 'Jas', '1Pet', '2Pet', '1John', '2John', '3John', 'Jude', 'Rev',
'EpLao'
]
# list of books with one chapter
ONECHAP = ['Obad', 'Phlm', '2John', '3John', 'Jude']
# revisionDesc
REVISIONDESC = '''
{}
osisRef attributes added using orefs.py
'''
def getabbrevs(text):
"""Get book abbreviations from toc2 and toc3 usfm tags."""
abbrevs = OrderedDict()
abbrevs2 = OrderedDict()
book = ""
lines = text.split("\n")
num = 0
for i in [_ for _ in lines if 'type="book"' in _]:
num += 1
bkline = lines.index(i)
book = i.partition('osisID="')[2].split('"')[0]
abbrevs[book] = ["{:03}".format(num)]
abbrevs2["{:03}".format(num)] = book
for j in range(bkline, bkline + 10):
if "x-usfm-toc2" in lines[j]:
abbr = lines[j].partition('n="')[2].split('"')[0]
abbrevs[book].append(abbr)
elif "x-usfm-toc3" in lines[j]:
abbr = lines[j].partition('n="')[2].split('"')[0]
abbrevs[book].append(abbr)
return abbrevs, abbrevs2
def readconfig(fname):
"""Read a config file and return the values."""
# set up config parser and read our config file...
config = configparser.ConfigParser(allow_no_value=True)
config.optionxform = str
config.read(fname)
# variables used while reading config file
abbr = OrderedDict()
# get default part of config...
for i in ["SEPM", "SEPP", "SEPC", "SEPR", "SEPRNORM"]:
globals()[i] = config["DEFAULT"][i]
if "DIGITS" in config["DEFAULT"]:
globals()["DIGITS"] = config["DEFAULT"]["DIGITS"]
globals()["DIGITTABLE"] = str.maketrans("0123456789",
globals()["DIGITS"])
globals()["SEPRNORM"] = {
True: list(globals()["SEPR"]),
False: list(globals()["SEPRNORM"])
}[globals()["SEPRNORM"] == ""]
# get abbreviations
for i in BOOKLIST:
if i in config["ABBR"]:
abbr[i] = {
True: config["ABBR"][i],
False: None
}[config["ABBR"][i] != ""]
# fix abbreviation lists
abbrevs = OrderedDict()
abbrevs2 = OrderedDict()
num = 0
for i in abbr:
num += 1
if abbr[i] is not None:
abbr[i] = [i.strip() for i in abbr[i].split(",")]
abbr[i] = sorted(abbr[i], key=len, reverse=True)
abbr[i].insert(0, "{:03}".format(num))
abbrevs[i] = abbr[i]
abbrevs2[abbr[i][0]] = i
# return our abbreviations and configuration
return abbrevs, abbrevs2
def genconf(text):
"""Generate contents of a config file."""
# set up config parser
config = configparser.ConfigParser(allow_no_value=True)
config.optionxform = str
for i in (("SEPM", SEPM),
("SEPP", SEPP),
("SEPC", SEPC),
("SEPR", SEPR),
("SEPRNORM", "".join(SEPRNORM)),
("DIGITS", DIGITS)):
config["DEFAULT"][i[0]] = i[1]
# create ABBR section of config…
config["ABBR"] = {}
for i in BOOKLIST:
config["ABBR"][i] = ""
abbr1, _ = getabbrevs(text)
for i in abbr1.keys():
config["ABBR"][i] = ", ".join(abbr1[i][1:])
# return our generate configuration
return config
def processreferences(text, abbr, abbr2):
"""Process cross references in osis text."""
crossrefnote = re.compile(
r'()(.*?)()', re.U)
reftag = re.compile(
r'(]*>)(.*?)()', re.U)
lines = text.split('\n')
currentbook = ""
def simplerepl(match):
"""Simple regex replacement helper function."""
errortext = ""
text = match.group(2).strip()
if text.startswith("(") and text.endswith(")"):
text = text[1:-1].strip()
osisrefs, oreferror = getosisrefs(text, currentbook, abbr, abbr2)
if oreferror:
errortext = ''
# process reference tags
if match.group(1).startswith('', ' {}>').format(
'osisRef="{}"'.format(osisrefs))
outtext = '{}{}{}'.format(
reftagstart, text, errortext)
else:
# only process references if no reference tag is present in text.
if '{}'.format(
'{}{}'.format(
osisrefs,
text,
errortext))
else:
outtext = r'{}'.format(
text)
return outtext
for i in (_ for _ in range(len(lines))):
if 'div type="book"' in lines[i]:
currentbook = lines[i].split('osisID="')[1].split('"')[0]
if " 1:
# split 2nd part of verse range at SEPC
vrsrange2 = vrsrange[1].split(SEPC)
if len(vrsrange2) > 1:
# additional chapter specified
vrsrange2[0] = chapchk(vrsrange2[0])
vrsrange2[1] = vrschk(vrsrange2[1])
if False in vrsrange2:
referror(" ".join([abbr2[bkref], j]), abbr2)
oreferror = True
continue
refs.append("{}.{}.{}-{}.{}.{}".format(
abbr2[bkref],
chap,
vrsrange[0],
abbr2[bkref],
vrsrange2[0],
vrsrange2[1]))
else:
# no additional chapter specified
vrsrange[0] = vrschk(vrsrange[0])
vrsrange[1] = vrschk(vrsrange[1])
if False in vrsrange:
referror(" ".join([abbr2[bkref], j]), abbr2)
oreferror = True
continue
refs.append("{}.{}.{}-{}.{}.{}".format(
abbr2[bkref],
chap,
vrsrange[0],
abbr2[bkref],
chap,
vrsrange[1]))
# not a verse range
else:
if SEPC in j:
chapverse2 = j.split(SEPC)
if " " in chapverse2[1]:
chapverse2[1] = chapverse2[1].split(" ")[0]
chapverse2[0] = chapchk(chapverse2[0])
chapverse2[1] = vrschk(chapverse2[1])
if False in chapverse2:
referror(" ".join([abbr2[bkref], j]), abbr2)
oreferror = True
continue
refs.append("{}.{}.{}".format(abbr2[bkref],
chapverse2[0],
chapverse2[1]))
else:
if " " in j:
j = j.split(" ")[0]
tmp = vrschk(j)
if tmp is False:
referror(" ".join([abbr2[bkref], j]), abbr2)
oreferror = True
continue
refs.append("{}.{}.{}".format(abbr2[bkref], chap, tmp))
# --- return joined references
return " ".join(refs), oreferror
# -------------------------------------------------------------------------- #
def processfile(args):
"""Process osis file."""
with open(args.i, "r") as ifile:
if args.v:
print("Reading input file {} ...".format(args.i))
text = ifile.read()
if args.v:
print("Getting book names and abbreviations...")
if args.c is not None:
if args.c == "create":
if args.v:
print("Creating config file...")
generatedconf = genconf(text)
else:
if args.v:
print("Using config file for abbreviations...")
bookabbrevs, bookabbrevs2 = readconfig(args.c)
else:
if args.v:
print("Extracting book names and abbreviations from osis file...")
bookabbrevs, bookabbrevs2 = getabbrevs(text)
if args.c != "create":
if args.v:
print("Processing cross references...")
text = processreferences(text, bookabbrevs, bookabbrevs2)
# add new revisionDesc to osis file...
username = {
True: os.getenv("LOGNAME"),
False: os.getenv("USERNAME")
}[os.getenv("USERNAME") is None]
textsplit = text.partition(" 0:
# -- only add if there is already a revisionDesc tag present
# -- just in case this was run on something processed by
# -- another usfm to osis converter that didn't add a
# -- revisionDesc tag to the osis document.
text = "".join([textsplit[0],
REVISIONDESC.format(
username,
datetime.datetime.now().strftime(
"%Y.%m.%dT%H.%M.%S")),
textsplit[1],
textsplit[2]])
if args.v:
print("Writing output to {} ".format(args.o))
if args.c == "create":
with open(args.o, "w") as ofile:
generatedconf.write(ofile)
else:
with open(args.o, "wb") as ofile:
ofile.write(text.encode("utf8"))
if args.v:
print("Done.")
# -------------------------------------------------------------------------- #
def main():
"""
Main routine.
Process our command line arguments and pass the options
along to subroutine that processes the file.
"""
parser = argparse.ArgumentParser(
description="""
Process cross references.
"""
)
parser.add_argument("-v",
help="verbose output",
action="store_true")
parser.add_argument("-i",
help="name of input file",
required=True, metavar="FILE")
parser.add_argument("-o",
help="name of output file",
required=True, metavar="FILE")
parser.add_argument("-c",
help="\n".join([
"config file to use.",
"create means to generate a config file."]),
metavar="FILE|create")
args = parser.parse_args()
if not os.path.isfile(args.i):
print("ERROR: input file is not present or is not a normal file.",
file=sys.stderr)
sys.exit()
processfile(args)
if __name__ == "__main__":
main()
u2o-0.7/utaglist.py 0000775 0000000 0000000 00000017606 14077653007 0014321 0 ustar 00root root 0000000 0000000 #!/usr/bin/python3
# -*- coding: utf-8 -*-
"""
List ufm tags.
A simple script to generate a list of usfm tags that were used in one
or more utf8 encoded usfm files. Compatible with python2 and python3.
This script is public domain.
"""
from __future__ import print_function, unicode_literals
import argparse
import re
import collections
import glob
# -------------------------------------------------------------------------- #
VERSION = "1.1.1"
# -------------------------------------------------------------------------- #
# set of known usfm tags in version 3.0 of usfm specification
# I think I have all the tags here.
KNOWNTAGS = {
# identification
r"\id",
r"\ide",
r"\sts",
r"\rem",
r"\h",
r"\h1",
r"\h2",
r"\h3",
r"\toc1",
r"\toc2",
r"\toc3",
# introductions
r"\imt",
r"\imt1",
r"\imt2",
r"\imt3",
r"\imt4",
r"\is",
r"\is1",
r"\is2",
r"\ip",
r"\ipi",
r"\im",
r"\imi",
r"\ipq",
r"\imq",
r"\ipr",
r"\iq",
r"\iq1",
r"\iq2",
r"\iq3",
r"\ib",
r"\ili",
r"\ili1",
r"\ili2",
r"\iot",
r"\io",
r"\io1",
r"\io2",
r"\io3",
r"\io4",
r"\ior",
r"\ior*",
r"\iex",
r"\iqt",
r"\iqt*",
r"\imte",
r"\imte1",
r"\imte2",
r"\ie",
# titles, headings, labels
r"\mt",
r"\mt1",
r"\mt2",
r"\mt3",
r"\mt4",
r"\mte",
r"\mte1",
r"\mte2",
r"\ms",
r"\ms1",
r"\ms2",
r"\ms3",
r"\mr",
r"\s",
r"\s1",
r"\s2",
r"\s3",
r"\s4",
r"\sr",
r"\r",
r"\rq",
r"\rq*",
r"\d",
r"\sp",
# chapters and verses
r"\c",
r"\ca",
r"\ca*",
r"\cl",
r"\cp",
r"\cd",
r"\v",
r"\va",
r"\va*",
r"\vp",
r"\vp*",
# paragraphs/poetry
r"\p",
r"\po",
r"\m",
r"\pmo",
r"\pm",
r"\pmc",
r"\pmr",
r"\pi",
r"\pi1",
r"\pi2",
r"\pi3",
r"\mi",
r"\nb",
r"\cls",
r"\lh",
r"\lf",
r"\li",
r"\li1",
r"\li2",
r"\li3",
r"\li4",
r"\lim",
r"\lim1",
r"\lim2",
r"\lim3",
r"\lim4",
r"\lim5",
r"\pc",
r"\pr",
r"\ph",
r"\ph1",
r"\ph2",
r"\ph3",
r"\q",
r"\q1",
r"\q2",
r"\q3",
r"\q4",
r"\qr",
r"\qc",
r"\qs",
r"\qs*",
r"\qa",
r"\qac",
r"\qac*",
r"\qm",
r"\qm1",
r"\qm2",
r"\qm3",
r"\qd",
r"\lik",
r"\lik*",
r"\litl",
r"\litl*",
r"\ts",
r"\sd",
r"\sd1",
r"\sd2",
r"\sd3",
r"\sd4",
r"\sd5",
r"\b",
# tables
r"\tr",
r"\th",
r"\th1",
r"\th2",
r"\th3",
r"\th4",
r"\th5",
r"\thr",
r"\thr1",
r"\thr2",
r"\thr3",
r"\thr4",
r"\thr5",
r"\tc",
r"\tc1",
r"\tc2",
r"\tc3",
r"\tc4",
r"\tc5",
r"\tcr",
r"\tcr1",
r"\tcr2",
r"\tcr3",
r"\tcr4",
r"\tcr5",
# footnotes
r"\f",
r"\f*",
r"\fe",
r"\fe*",
r"\fr",
r"\fk",
r"\fq",
r"\fqa",
r"\fl",
r"\fp",
r"\fv",
r"\fv*",
r"\ft",
r"\fdc",
r"\fdc*",
r"\fm",
r"\fm*",
# cross references
r"\x",
r"\x*",
r"\xo",
r"\xk",
r"\xq",
r"\xt",
r"\xot",
r"\xot*",
r"\xnt",
r"\xnt*",
r"\xdc",
r"\xdc*",
r"\xta",
r"\xta*",
r"\xop",
r"\xop*",
# special text
r"\add",
r"\add*",
r"\bk",
r"\bk*",
r"\dc",
r"\dc*",
r"\k",
r"\k*",
r"\lit",
r"\nd",
r"\nd*",
r"\ord",
r"\ord*",
r"\pn",
r"\pn*",
r"\qt",
r"\qt*",
r"\sig",
r"\sig*",
r"\sls",
r"\sls*",
r"\tl",
r"\tl*",
r"\wj",
r"\wj*",
r"\addpn",
r"\addpn*",
r"\+add",
r"\+add*",
r"\+bk",
r"\+bk*",
r"\+dc",
r"\+dc*",
r"\+k",
r"\+k*",
r"\+nd",
r"\+nd*",
r"\+ord",
r"\+ord*",
r"\+pn",
r"\+pn*",
r"\+qt",
r"\+qt*",
r"\+sig",
r"\+sig*",
r"\+sls",
r"\+sls*",
r"\+tl",
r"\+tl*",
r"\+wj",
r"\+wj*",
r"\+addpn",
r"\+addpn*",
# character styles
r"\em",
r"\em*",
r"\bd",
r"\bd*",
r"\it",
r"\it*",
r"\bdit",
r"\bdit*",
r"\no",
r"\no*",
r"\sc",
r"\sc*",
r"\+em",
r"\+em*",
r"\+bd",
r"\+bd*",
r"\+it",
r"\+it*",
r"\+bdit",
r"\+bdit*",
r"\+no",
r"\+no*",
r"\+sc",
r"\+sc*",
# spacing
r"\pb",
# special features
r"\fig",
r"\fig*",
r"\ndx",
r"\ndx*",
r"\pro",
r"\pro*",
r"\w",
r"\w*",
r"\wg",
r"\wg*",
r"\wh",
r"\wh*",
r"\wa",
r"\wa*",
r"\png",
r"\png*",
r"\jmp",
r"\jmp*",
r"\rb",
r"\rb*",
r"\rt",
r"\rt*",
# peripherals
r"\periph",
# study bible content
r"\ef",
r"\ef*",
r"\ex",
r"\ex*",
r"\esb",
r"\esbe",
r"\cat",
r"\cat*",
# milestone quotations
r"\qt-s",
r"\qt1-s",
r"\qt2-s",
r"\qt3-s",
r"\qt4-s",
r"\qt5-s",
r"\qt-e",
r"\qt1-e",
r"\qt2-e",
r"\qt3-e",
r"\qt4-e",
r"\qt5-e",
r"\*",
}
# regex for finding usfm tags
USFMRE = re.compile(
r"""
# handle usfm tags other than special spacing...
(?:
# the first character of a usfm tag is always a backslash
\\
# a plus symbol marks the start of a nested character style.
# this may or may not be present.
\+?
# tag names are ascii letters
[A-Za-z]+
# tags may or may not be numbered
[0-9]?
# a word boundary to mark the end of our tags.
\b
# character style closing tags end with an asterisk.
\*?
)
# OR...
|
# one of the two special spacing tags.
(?:
(?:~|//)
)
""",
re.U + re.VERBOSE,
)
# -------------------------------------------------------------------------- #
def processtags(args):
"""Process usfm tags in all files."""
count = 0
counttags = collections.Counter()
knownset = set()
filenames = []
for _ in args.file:
if "*" in _:
filenames.extend(glob.glob(_))
else:
filenames.append(_)
for fname in filenames:
with open(fname, "rb") as infile:
intext = infile.read()
# build tag set
knownset.update(USFMRE.findall(intext.decode("utf8")))
# build usage counts
for i in USFMRE.findall(intext.decode("utf8")):
count += 1
counttags[i] += 1
# split tags into known and unknown sets
unknownset = knownset.difference(KNOWNTAGS)
knownset = knownset.intersection(KNOWNTAGS)
# output results.
print()
if knownset:
print("Known USFM Tags: {}\n".format(", ".join(sorted(knownset))))
if unknownset:
print("Unknown USFM Tags: {}\n".format(", ".join(sorted(unknownset))))
# print tag usage counts
if args.c:
print("\nTag usage count:\n")
for i in sorted(counttags):
print("{: 8} - {}".format(counttags[i], i))
print("\nTotal number of usfm tags found: {}\n".format(count))
def main():
"""Process command line arguments and then process files."""
parser = argparse.ArgumentParser(
description="""
A simple script to generate a list of usfm tags that were used in
one or more utf8 encoded usfm files.
""",
epilog="""
* Version: {} * This script is public domain *
""".format(
VERSION
),
)
parser.add_argument(
"-c", help="include usage counts for tags", action="store_true"
)
parser.add_argument(
"file", help="name of file to process (wildcards allowed)", nargs="+"
)
args = parser.parse_args()
processtags(args)
# -------------------------------------------------------------------------- #
if __name__ == "__main__":
main()
|