Hey Brian.
Tom Morris
fake
+44 1234 567890
mf2py-2.0.1/test/examples/value_name_whitespace.html 0000664 0000000 0000000 00000002311 14534511526 0022565 0 ustar 00root root 0000000 0000000
Hello World
one
two
three
Correct name
Correct summary
mf2py-2.0.1/test/test_dom_addins.py 0000664 0000000 0000000 00000000451 14534511526 0017246 0 ustar 00root root 0000000 0000000 from mf2py.parser import Parser
def test_getElementsByClassName():
p = Parser(doc=open("test/examples/person_with_url.html"))
dom = p.__doc__
assert len(dom.find_all(class_="u-url")) == 1
expected_el = dom.find_all(class_="u-url")[0]
assert expected_el["class"] == ["u-url"]
mf2py-2.0.1/test/test_parser.py 0000664 0000000 0000000 00000122731 14534511526 0016447 0 ustar 00root root 0000000 0000000 import os
import re
import sys
from unittest import TestCase
import bs4
import mock
from bs4 import BeautifulSoup
from mf2py import Parser
TestCase.maxDiff = None
TEST_DIR = "test/examples/"
def parse_fixture(path, **kwargs):
with open(os.path.join(TEST_DIR, path)) as f:
p = Parser(doc=f, html_parser="html5lib", **kwargs)
return p.to_dict()
def test_empty():
p = Parser()
assert type(p) is not None
assert type(p.to_dict()) is dict
def test_open_file():
with open(os.path.join(TEST_DIR, "empty.html")) as f:
p = Parser(doc=f)
assert p.__doc__ is not None
assert type(p) is not None
assert type(p.to_dict()) is dict
def test_doc_tag():
# test that strings, BS doc and BS tags are all parsed
doc = """
"""
soup = BeautifulSoup(doc, "html5lib")
parse_string = Parser(doc).to_dict()
assert "h-entry" in parse_string["items"][0]["type"]
parse_doc = Parser(soup).to_dict()
assert "h-entry" in parse_doc["items"][0]["type"]
parse_tag = Parser(soup.article).to_dict()
assert "h-entry" in parse_tag["items"][0]["type"]
@mock.patch("requests.get")
def test_user_agent(getter):
ua_expect = "mf2py - microformats2 parser for python"
assert Parser.useragent.startswith(ua_expect)
resp = mock.MagicMock()
resp.content = b""
resp.text = ""
resp.headers = {}
getter.return_value = resp
Parser(url="http://example.com")
getter.assert_called_with(
"http://example.com", headers={"User-Agent": Parser.useragent}
)
Parser.useragent = "something else"
assert Parser.useragent == "something else"
# set back to default. damn stateful classes
Parser.useragent = "mf2py - microformats2 parser for python"
def test_base():
with open(os.path.join(TEST_DIR, "base.html")) as f:
p = Parser(doc=f)
assert p.__url__ == "http://tantek.com/"
def test_simple_parse():
result = parse_fixture("simple_person_reference.html")
assert result["items"][0]["properties"] == {"name": ["Frances Berriman"]}
def test_simple_person_reference_same_element():
result = parse_fixture("simple_person_reference_same_element.html")
assert result["items"][0]["properties"] == {"name": ["Frances Berriman"]}
def test_person_with_url():
result = parse_fixture("person_with_url.html")
assert result["items"][0]["properties"]["name"] == ["Tom Morris"]
assert result["items"][0]["properties"]["url"] == ["http://tommorris.org/"]
def test_vcp():
result = parse_fixture("value_class_person.html")
assert result["items"][0]["properties"]["tel"] == ["+44 1234 567890"]
def test_multiple_root_classnames():
result = parse_fixture("nested_multiple_classnames.html")
# order does not matter
assert len(result["items"]) == 1
assert set(result["items"][0]["type"]) == set(["h-entry", "h-as-note"])
def test_property_nested_microformat():
result = parse_fixture("nested_multiple_classnames.html")
assert len(result["items"]) == 1
assert "author" in result["items"][0]["properties"]
assert (
result["items"][0]["properties"]["author"][0]["properties"]["name"][0]
== "Tom Morris"
)
assert (
result["items"][0]["properties"]["reviewer"][0]["properties"]["name"][0]
== "Tom Morris"
)
assert (
result["items"][0]["properties"]["author"][0]["properties"]["adr"][0][
"properties"
]["city"][0]
== "London"
)
def test_plain_child_microformat():
result = parse_fixture("nested_multiple_classnames.html")
assert len(result["items"]) == 1
assert "children" in result["items"][0]
assert len(result["items"][0]["children"]) == 1
assert result["items"][0]["children"][0]["properties"]["name"][0] == "Some Citation"
def test_datetime_parsing():
result = parse_fixture("datetimes.html")
assert result["items"][0]["properties"]["start"][0] == "2014-01-01T12:00:00+0000"
assert result["items"][0]["properties"]["end"][0] == "3014-01-01T18:00:00+0000"
assert result["items"][0]["properties"]["duration"][0] == "P1000Y"
assert result["items"][0]["properties"]["updated"][0] == "2011-08-26T00:01:21+0000"
assert result["items"][0]["properties"]["updated"][1] == "2011-08-26T00:01:21+0000"
def test_datetime_vcp_parsing():
result = parse_fixture("datetimes.html")
assert len(result["items"]) == 16
assert result["items"][1]["properties"]["published"][0] == "3014-01-01 01:21Z"
assert result["items"][2]["properties"]["updated"][0] == "2014-03-11 09:55"
assert result["items"][3]["properties"]["published"][0] == "2014-01-30 15:28"
assert result["items"][4]["properties"]["published"][0] == "9999-01-14T11:52+0800"
assert result["items"][5]["properties"]["published"][0] == "2014-06-01 12:30-0600"
assert result["items"][8]["properties"]["start"][0] == "2014-06-01 12:30-0600"
assert result["items"][9]["properties"]["start"][0] == "2014-06-01 12:30-0600"
assert result["items"][10]["properties"]["start"][0] == "2014-06-01 00:30-0600"
assert result["items"][10]["properties"]["end"][0] == "2014-06-01 12:15"
assert result["items"][10]["properties"]["start"][1] == "2014-06-01 00:30-0600"
assert result["items"][10]["properties"]["end"][1] == "2014-06-01 12:15"
assert result["items"][11]["properties"]["start"][0] == "2016-03-02 00:30-0600"
assert result["items"][12]["properties"]["start"][0] == "2014-06-01 12:30-600"
assert result["items"][13]["properties"]["start"][0] == "2014-06-01 12:30+600"
assert result["items"][14]["properties"]["start"][0] == "2014-06-01 12:30Z"
assert result["items"][15]["properties"]["start"][0] == "2014-06-01 12:30-600"
def test_dt_end_implied_date():
"""Test that events with dt-start and dt-end use the implied date rule
http://microformats.org/wiki/value-class-pattern#microformats2_parsers
for times without dates"""
result = parse_fixture("datetimes.html")
event_wo_tz = result["items"][6]
assert event_wo_tz["properties"]["start"][0] == "2014-05-21 18:30"
assert event_wo_tz["properties"]["end"][0] == "2014-05-21 19:30"
event_w_tz = result["items"][7]
assert event_w_tz["properties"]["start"][0] == "2014-06-01 12:30-0600"
assert event_w_tz["properties"]["end"][0] == "2014-06-01 19:30-0600"
def test_embedded_parsing():
result = parse_fixture("embedded.html")
assert (
result["items"][0]["properties"]["content"][0]["html"]
== "
Blah blah blah blah blah.
\n
Blah.
\n
Blah blah blah.
"
)
assert (
result["items"][0]["properties"]["content"][0]["value"]
== "Blah blah blah blah blah.\n\nBlah.\n\nBlah blah blah."
)
def test_embedded_exposed_dom():
result = parse_fixture("embedded.html", expose_dom=True)
content = result["items"][0]["properties"]["content"][0]
assert "html" not in content
assert isinstance(content["dom"], bs4.element.Tag)
def test_hoisting_nested_hcard():
result = parse_fixture("nested_hcards.html")
expected = [
{
"properties": {
"author": [
{
"properties": {"name": ["KP1"]},
"type": ["h-card"],
"value": "KP1",
}
],
"in-reply-to": [
{"properties": {"name": ["KP"]}, "type": ["h-cite"], "value": "KP"}
],
},
"type": ["h-entry"],
}
]
assert expected == result["items"]
def test_html_tag_class():
result = parse_fixture("hfeed_on_html_tag.html")
assert ["h-feed"] == result["items"][0]["type"]
assert ["entry1"] == result["items"][0]["children"][0]["properties"]["name"]
assert ["entry2"] == result["items"][0]["children"][1]["properties"]["name"]
def test_string_strip():
result = parse_fixture("string_stripping.html")
assert "Tom Morris" == result["items"][0]["properties"]["name"][0]
def test_template_parse():
result = parse_fixture("template_tag.html")
assert 0 == len(result["items"])
def test_template_tag_inside_e_value():
result = parse_fixture("template_tag_inside_e_value.html")
assert (
"This is a Test with a
template tag after this:"
== result["items"][0]["properties"]["content"][0]["html"]
)
assert (
"This is a Test with a template tag after this:"
== result["items"][0]["properties"]["content"][0]["value"]
)
def test_ordering_dedup():
"""test that classes are dedeuped and alphabetically ordered"""
result = parse_fixture("ordering_dedup.html")
item = result["items"][0]
assert ["h-entry", "h-feed", "h-product", "h-x-test"] == item["type"]
assert ["example.com", "example.com/2"] == item["properties"]["url"]
assert ["name", "URL name"] == item["properties"]["name"]
assert ["author", "bookmark", "me"] == result["rel-urls"]["example.com/rel"]["rels"]
assert "de" == result["rel-urls"]["example.com/lang"]["hreflang"]
def test_class_names_format():
"""test that only classes with letters and possibly numbers in the vendor prefix part are used"""
result = parse_fixture("class_names_format.html")
item = result["items"][0]
assert ["h-feed", "h-p3k-entry", "h-x-test"] == item["type"]
assert "url" in item["properties"]
assert "p3k-url" in item["properties"]
assert "Url" not in item["properties"]
assert "-url" not in item["properties"]
assert "url-" not in item["properties"]
assert "name" in item["properties"]
assert "p3k-name" in item["properties"]
assert "nAme" not in item["properties"]
assert "-name" not in item["properties"]
assert "name-" not in item["properties"]
def test_area_uparsing():
result = parse_fixture("area.html")
assert {"url": ["http://suda.co.uk"], "name": ["Brian Suda"]} == result["items"][0][
"properties"
]
assert "shape" in result["items"][0]
assert "coords" in result["items"][0]
def test_src_equiv():
result = parse_fixture("test_src_equiv.html")
for item in result["items"]:
assert "x-example" in item["properties"]
assert "http://example.org/" == item["properties"]["x-example"][0]
def test_rels():
result = parse_fixture("rel.html")
assert {
"in-reply-to": ["http://example.com/1", "http://example.com/2"],
"author": ["http://example.com/a", "http://example.com/b"],
"alternate": ["http://example.com/fr"],
"home": ["http://example.com/fr"],
} == result["rels"]
assert {
"http://example.com/1": {"text": "post 1", "rels": ["in-reply-to"]},
"http://example.com/2": {"text": "post 2", "rels": ["in-reply-to"]},
"http://example.com/a": {"text": "author a", "rels": ["author"]},
"http://example.com/b": {"text": "author b", "rels": ["author"]},
"http://example.com/fr": {
"text": "French mobile homepage",
"media": "handheld",
"rels": ["alternate", "home"],
"hreflang": "fr",
},
} == result["rel-urls"]
def test_alternates():
result = parse_fixture("rel.html")
assert [
{
"url": "http://example.com/fr",
"media": "handheld",
"text": "French mobile homepage",
"rel": "home",
"hreflang": "fr",
}
] == result["alternates"]
def test_enclosures():
result = parse_fixture("rel_enclosure.html")
assert {"enclosure": ["http://example.com/movie.mp4"]} == result["rels"]
assert {
"http://example.com/movie.mp4": {
"rels": ["enclosure"],
"text": "my movie",
"type": "video/mpeg",
}
} == result["rel-urls"]
def test_empty_href():
result = parse_fixture("hcard_with_empty_url.html", url="http://foo.com")
for hcard in result["items"]:
assert ["http://foo.com"] == hcard["properties"]["url"]
def test_link_with_u_url():
result = parse_fixture("link_with_u-url.html", url="http://foo.com")
assert {
"type": ["h-card"],
"properties": {
"name": [""],
"url": ["http://foo.com/"],
},
} == result["items"][0]
def test_broken_url():
result = parse_fixture("broken_url.html", url="http://example.com")
assert (
result["items"][0]["properties"]["relative"][0] == "http://example.com/foo.html"
)
assert result["items"][0]["properties"]["url"][0] == "http://www.[w3.org/"
assert (
result["items"][0]["properties"]["photo"][0]
== "http://www.w3].org/20[08/site/images/logo-w3c-mobile-lg"
)
def test_complex_e_content():
"""When parsing h-* e-* properties, we should fold {"value":..., "html":...}
into the parsed microformat object, instead of nesting it under an
unnecessary second layer of "value":
"""
result = parse_fixture("complex_e_content.html")
assert {
"type": ["h-entry"],
"properties": {
"content": [
{
"type": ["h-card"],
"properties": {"name": ["Hello"]},
"html": "
Hello
",
"value": "Hello",
}
],
},
} == result["items"][0]
def test_relative_url_in_e():
"""When parsing e-* properties, make relative URLs absolute."""
result = parse_fixture("relative_url_in_e.html")
assert (
'
Cat '
'
'
) == result["items"][0]["properties"]["content"][0]["html"]
def test_nested_values():
"""When parsing nested microformats, check that value is the value of
the simple property element"""
result = parse_fixture("nested_values.html")
entry = result["items"][0]
assert {
"properties": {
"name": ["Kyle"],
"url": ["http://about.me/kyle"],
},
"value": "Kyle",
"type": ["h-card"],
} == entry["properties"]["author"][0]
assert {
"properties": {
"name": ["foobar"],
"url": ["http://example.com/foobar"],
},
"value": "http://example.com/foobar",
"type": ["h-cite"],
} == entry["properties"]["like-of"][0]
assert {
"properties": {
"name": ["George"],
"url": ["http://people.com/george"],
},
"type": ["h-card"],
} == entry["children"][0]
# implied properties tests
def test_implied_name():
result = parse_fixture("implied_properties/implied_properties.html")
for i in range(7):
assert result["items"][i]["properties"]["name"][0] == "Tom Morris"
def test_implied_url():
result = parse_fixture(
"implied_properties/implied_properties.html", url="http://foo.com/"
)
assert result["items"][1]["properties"]["url"][0] == "http://tommorris.org/"
# img should not have a "url" property
assert "url" not in result["items"][4]["properties"]
# href="" is relative to the base url
assert result["items"][5]["properties"]["url"][0] == "http://foo.com/"
def test_implied_photo():
result = parse_fixture("implied_properties/implied_photo.html")
for i in range(12):
photos = result["items"][i]["properties"]["photo"]
assert len(photos) == 1
assert photos[0] == "http://example.com/photo.jpg"
# tests for no photo
for i in range(12, 23):
assert "photo" not in result["items"][i]["properties"]
result = parse_fixture("implied_properties/implied_photo_relative_url.html")
assert (
result["items"][0]["properties"]["photo"][0]["value"]
== "http://example.com/jane-img.jpeg"
)
assert (
result["items"][1]["properties"]["photo"][0]
== "http://example.com/jane-object.jpeg"
)
def test_implied_url():
result = parse_fixture("implied_properties/implied_url.html")
for i in range(12):
urls = result["items"][i]["properties"]["url"]
assert len(urls) == 1
assert urls[0] == "http://example.com"
# tests for no url
for i in range(12, 23):
assert "url" not in result["items"][i]["properties"]
def test_stop_implied_url():
"""testing that explicit properties case implied url-parsing to be aborted"""
result = parse_fixture("implied_properties/stop_implied_url.html")
assert "url" not in result["items"][0]["properties"]
assert "url" not in result["items"][1]["properties"]
assert "url" not in result["items"][2]["properties"]
assert "url" not in result["items"][3]["properties"]
assert "url" not in result["items"][4]["properties"]
assert "url" not in result["items"][5]["properties"]
assert result["items"][6]["properties"]["url"] == ["http://example.com/"]
assert result["items"][7]["properties"]["url"] == ["http://example.com/"]
assert result["items"][8]["properties"]["url"] == ["http://example.com/"]
assert result["items"][9]["properties"]["url"] == ["http://example.com/"]
def test_implied_nested_photo():
result = parse_fixture(
"implied_properties/implied_properties.html", url="http://bar.org"
)
assert result["items"][2]["properties"]["photo"][0] == {
"alt": "",
"value": "http://tommorris.org/photo.png",
}
assert (
result["items"][3]["properties"]["photo"][0] == "http://tommorris.org/photo.png"
)
assert result["items"][4]["properties"]["photo"][0] == {
"alt": "Tom Morris",
"value": "http://tommorris.org/photo.png",
}
# src="" is relative to the base url
assert result["items"][6]["properties"]["photo"][0] == "http://bar.org"
def test_implied_nested_photo_alt_name():
result = parse_fixture("implied_properties/implied_properties.html")
assert result["items"][3]["properties"]["name"][0] == "Tom Morris"
def test_implied_image():
result = parse_fixture("implied_properties/implied_properties.html")
assert result["items"][4]["properties"]["photo"][0] == {
"alt": "Tom Morris",
"value": "http://tommorris.org/photo.png",
}
assert result["items"][4]["properties"]["name"][0] == "Tom Morris"
def test_implied_name_empty_alt():
"""An empty alt text should not prevent us from including other
children in the implied name.
"""
result = parse_fixture("implied_properties/implied_name_empty_alt.html")
hcard = result["items"][0]
assert {
"type": ["h-card"],
"properties": {
"name": ["@kylewmahan"],
"url": ["https://twitter.com/kylewmahan"],
"photo": [{"alt": "", "value": "https://example.org/test.jpg"}],
},
} == hcard
def test_relative_datetime():
result = parse_fixture("implied_properties/implied_relative_datetimes.html")
assert result["items"][0]["properties"]["updated"][0] == "2015-01-02 05:06"
def test_stop_implied_name_nested_h():
result = parse_fixture("implied_properties/stop_implied_name_nested_h.html")
assert "name" not in result["items"][0]["properties"]
def test_stop_implied_name_e_content():
result = parse_fixture("implied_properties/stop_implied_name_e_content.html")
assert "name" not in result["items"][0]["properties"]
def test_stop_implied_name_p_content():
result = parse_fixture("implied_properties/stop_implied_name_p_content.html")
assert "name" not in result["items"][0]["properties"]
def test_implied_properties_silo_pub():
result = parse_fixture("implied_properties/implied_properties_silo_pub.html")
item = result["items"][0]
# implied_name = item['properties']['name'][0]
# implied_name = re.sub('\s+', ' ', implied_name).strip()
# assert '@kylewmahan on Twitter', implied_name)
# no implied name expected under new rules
assert "name" not in item["properties"]
def test_simple_person_reference_implied():
result = parse_fixture("implied_properties/simple_person_reference_implied.html")
assert result["items"][0]["properties"] == {"name": ["Frances Berriman"]}
def test_implied_name_alt():
result = parse_fixture("implied_properties/implied_name_alt.html")
assert result["items"][0]["children"][0] == {
"type": ["h-card"],
"properties": {
"name": ["Avatar of Stephen"],
"photo": [{"alt": "Avatar of", "value": "avatar.jpg"}],
},
}
def test_value_name_whitespace():
result = parse_fixture("value_name_whitespace.html")
for i in range(3):
assert result["items"][i]["properties"]["content"][0]["value"] == "Hello World"
assert result["items"][i]["properties"]["name"][0] == "Hello World"
for i in range(3, 7):
assert result["items"][i]["properties"]["content"][0]["value"] == "Hello\nWorld"
assert result["items"][i]["properties"]["name"][0] == "Hello\nWorld"
assert result["items"][7]["properties"]["content"][0]["value"] == "Hello\n\nWorld"
assert result["items"][7]["properties"]["name"][0] == "Hello\n\nWorld"
assert result["items"][8]["properties"]["content"][0]["value"] == "One\nTwo\nThree"
assert result["items"][8]["properties"]["name"][0] == "One\nTwo\nThree"
assert (
result["items"][9]["properties"]["content"][0]["value"] == "One\n\nTwo\n\nThree"
)
assert result["items"][9]["properties"]["name"][0] == "One\n\nTwo\n\nThree"
assert (
result["items"][10]["properties"]["content"][0]["value"]
== "Hello World one\n two\n three\n "
)
assert (
result["items"][10]["properties"]["name"][0]
== "Hello World one\n two\n three\n "
)
assert (
result["items"][11]["properties"]["content"][0]["value"]
== "Correct name Correct summary"
)
assert result["items"][11]["properties"]["name"][0] == "Correct name"
# backcompat tests
def test_backcompat_hentry():
result = parse_fixture("backcompat/hentry.html")
assert "h-entry" in result["items"][0]["type"]
assert (
"Tom Morris"
== result["items"][0]["properties"]["author"][0]["properties"]["name"][0]
)
assert "A Title" == result["items"][0]["properties"]["name"][0]
assert "Some Content" == result["items"][0]["properties"]["content"][0]["value"]
def test_backcompat_hproduct():
result = parse_fixture("backcompat/hproduct.html")
assert 1 == len(result["items"])
assert ["h-product"] == result["items"][0]["type"]
assert ["bullshit"] == result["items"][0]["properties"]["category"]
assert ["Quacktastic Products"] == result["items"][0]["properties"]["brand"]
assert ["#BULLSHIT-001"] == result["items"][0]["properties"]["identifier"]
assert (
"Magical tasty sugar pills that don't do anything."
== result["items"][0]["properties"]["description"][0]
)
assert ["Tom's Magical Quack Tincture"] == result["items"][0]["properties"]["name"]
def test_backcompat_hproduct_nested_hreview():
result = parse_fixture("backcompat/hproduct_hreview_nested.html")
assert ["h-review"] == result["items"][0]["children"][0]["type"]
def test_backcompat_hreview_nested_card_event_product():
result = parse_fixture("backcompat/hreview_nested_card_event_product.html")
assert ["h-review"] == result["items"][0]["type"]
items = result["items"][0]["properties"]["item"]
assert 3 == len(items)
event = items[0]
assert ["h-event"] == event["type"]
assert ["http://example.com/event-url"] == event["properties"]["url"]
assert ["event name"] == event["properties"]["name"]
card = items[1]
assert ["h-card"] == card["type"]
assert ["http://example.com/card-url"] == card["properties"]["url"]
assert ["card name"] == card["properties"]["name"]
product = items[2]
assert ["h-product"] == product["type"]
assert ["http://example.com/product-url"] == product["properties"]["url"]
assert ["product name"] == product["properties"]["name"]
def test_backcompat_rel_bookmark():
"""Confirm that rel=bookmark inside of an h-entry is converted
to u-url.
"""
result = parse_fixture("backcompat/feed_with_rel_bookmark.html")
for ii, url in enumerate(
(
"/2014/11/24/jump-rope",
"/2014/11/23/graffiti",
"/2014/11/21/earth",
"/2014/11/19/labor",
)
):
assert ["h-entry"] == result["items"][ii]["type"]
assert [url] == result["items"][ii]["properties"]["url"]
def test_backcompat_rel_bookmark():
"""Confirm that rel=bookmark inside of an hentry and hreview is converted
to a u-url and original u-url is ignored
"""
tests = [
"backcompat/hentry_with_rel_bookmark.html",
"backcompat/hreview_with_rel_tag_bookmark.html",
]
results = [parse_fixture(x) for x in tests]
for result in results:
assert [
"https://example.com/bookmark",
"https://example.com/bookmark-url",
] == result["items"][0]["properties"]["url"]
def test_backcompat_rel_tag():
"""Confirm that rel=tag inside of an hentry is converted
to a p-category and the last path segment of the href is used.
"""
tests = [
"backcompat/hentry_with_rel_tag.html",
"backcompat/hfeed_with_rel_tag.html",
"backcompat/hrecipe_with_rel_tag.html",
"backcompat/hreview_with_rel_tag_bookmark.html",
]
results = [parse_fixture(x) for x in tests]
for result in results:
assert ["cat", "dog", "mountain lion", "mouse", "meerkat"] == result["items"][
0
]["properties"]["category"]
def test_backcompat_rel_tag_entry_title():
"""Confirm that other backcompat properties on a rel=tag are parsed"""
result = parse_fixture("backcompat/hentry_with_rel_tag_entry_title.html")
assert ["cat"] == result["items"][0]["properties"]["category"]
assert ["rhinoceros"] == result["items"][0]["properties"]["name"]
def test_backcompat_rel_multiple_root():
"""Confirm that rel=tag and rel=bookmark inside of an hentry+hreview is parsed correctly"""
result = parse_fixture("backcompat/hreview_hentry_with_rel_tag_bookmark.html")
assert len(result["items"]) == 1
assert "h-entry" in result["items"][0]["type"]
assert "h-review" in result["items"][0]["type"]
assert ["cat", "dog", "mountain lion", "mouse", "meerkat"] == result["items"][0][
"properties"
]["category"]
assert [
"https://example.com/bookmark",
"https://example.com/bookmark-url",
] == result["items"][0]["properties"]["url"]
def test_backcompat_ignore_mf1_root_if_mf2_present():
"""Confirm that mf1 root class is ignored if another mf2 root class is present."""
result = parse_fixture("backcompat/ignore_mf1_root_if_mf2_present.html")
assert "h-entry" not in result["items"][0]["type"]
assert "h-event" in result["items"][0]["type"]
def test_backcompat_no_implied_properties_mf1_root():
"""Confirm that mf1 root class does not have implied properties"""
result = parse_fixture("backcompat/ignore_mf1_root_if_mf2_present.html")
assert "h-entry" not in result["items"][0]["properties"]
assert "name" not in result["items"][0]["type"]
assert "url" not in result["items"][0]["properties"]
assert "photo" not in result["items"][0]["properties"]
def test_backcompat_ignore_mf2_properties_in_mf1_root():
"""Confirm that mf2 properties are ignored in mf1 root class"""
result = parse_fixture("backcompat/ignore_mf2_properties_in_mf1_root.html")
assert "Correct name" == result["items"][0]["properties"]["name"][0]
assert "Correct summary" == result["items"][0]["properties"]["summary"][0]
def test_backcompat_ignore_mf1_properties_in_mf2_root():
"""Confirm that mf1 properties are ignored in mf2 root class"""
result = parse_fixture("backcompat/ignore_mf1_properties_in_mf2_root.html")
assert "Correct name" == result["items"][0]["properties"]["name"][0]
assert "Correct summary" == result["items"][0]["properties"]["summary"][0]
def test_backcompat_nested_mf2_in_mf1():
"""Confirm that mf2 roots nested inside mf1 root are parsed"""
result = parse_fixture("backcompat/nested_mf2_in_mf1.html")
assert "h-feed" == result["items"][0]["type"][0]
assert "h-entry" == result["items"][0]["children"][0]["type"][0]
assert "Correct name" == result["items"][0]["children"][0]["properties"]["name"][0]
assert (
"Correct summary"
== result["items"][0]["children"][0]["properties"]["summary"][0]
)
def test_backcompat_nested_mf1_in_mf2():
"""Confirm that mf1 roots nested inside mf2 root are parsed"""
result = parse_fixture("backcompat/nested_mf1_in_mf2.html")
assert "h-feed" == result["items"][0]["type"][0]
assert "h-entry" == result["items"][0]["children"][0]["type"][0]
assert "Correct name" == result["items"][0]["children"][0]["properties"]["name"][0]
assert (
"Correct summary"
== result["items"][0]["children"][0]["properties"]["summary"][0]
)
def test_backcompat_nested_mf1_in_mf2_e_content():
"""Confirm that mf1 roots nested inside mf2 root e-content are parsed as authored"""
result = parse_fixture("backcompat/nested_mf1_in_mf2_e_content.html")
mf2_entry = result["items"][0]
mf1_entry = mf2_entry["children"][0]
assert (
'
\nCorrect name\n\nCorrect summary\n
'
== mf2_entry["properties"]["content"][0]["html"]
)
assert (
"Correct name Correct summary" == mf2_entry["properties"]["content"][0]["value"]
)
assert "h-entry" == mf1_entry["type"][0]
assert "Correct name" == mf1_entry["properties"]["name"][0]
assert "Correct summary" == mf1_entry["properties"]["summary"][0]
def test_backcompat_hentry_content_html():
"""Confirm that mf1 entry-content html is parsed as authored without mf2 replacements"""
result = parse_fixture("backcompat/hentry_content_html.html")
entry = result["items"][0]
assert (
'
This is a summary
\n
This is mytag inside content.
'
== entry["properties"]["content"][0]["html"]
)
def test_whitespace_with_tags_inside_property():
"""Whitespace should only be trimmed at the ends of textContent, not inside.
https://github.com/microformats/mf2py/issues/112
"""
result = parse_fixture("tag_whitespace_inside_p_value.html")
assert result["items"][0]["properties"] == {"name": ["foo bar"]}
def test_plaintext_p_whitespace():
result = parse_fixture("plaintext_p_whitespace.html")
assert result["items"][0]["properties"]["content"][0]["value"] == "foo\nbar baz"
assert result["items"][1]["properties"]["content"][0]["value"] == "foo\nbar baz"
assert result["items"][2]["properties"]["content"][0]["value"] == "foo bar\nbaz"
def test_plaintext_img_whitespace():
result = parse_fixture("plaintext_img_whitespace.html")
assert (
result["items"][0]["properties"]["content"][0]["value"]
== "selfie At some tourist spot"
)
assert (
result["items"][1]["properties"]["content"][0]["value"]
== "At another tourist spot"
)
assert (
result["items"][2]["properties"]["content"][0]["value"]
== "https://example.com/photo.jpg At yet another tourist spot"
)
def test_photo_with_alt():
"""Confirm that alt text in img is parsed as a u-* property and implied photo"""
path = "img_with_alt.html"
result = parse_fixture(path)
with open(os.path.join(TEST_DIR, path)) as f:
exp_result = Parser(doc=f, html_parser="html5lib").to_dict()
# simple img with u-*
assert "/photo.jpg" == result["items"][0]["properties"]["photo"][0]
assert "/photo.jpg" == exp_result["items"][0]["properties"]["photo"][0]
assert {"alt": "alt text", "value": "/photo.jpg"} == result["items"][1][
"properties"
]["url"][0]
assert "/photo.jpg" == exp_result["items"][1]["properties"]["url"][0]["value"]
assert "alt text" == exp_result["items"][1]["properties"]["url"][0]["alt"]
assert {"alt": "", "value": "/photo.jpg"} == result["items"][2]["properties"][
"in-reply-to"
][0]
assert (
"/photo.jpg" == exp_result["items"][2]["properties"]["in-reply-to"][0]["value"]
)
assert "" == exp_result["items"][2]["properties"]["in-reply-to"][0]["alt"]
# img with u-* and h-* example
assert "h-cite" in result["items"][3]["properties"]["in-reply-to"][0]["type"]
assert (
"/photo.jpg"
== result["items"][3]["properties"]["in-reply-to"][0]["properties"]["photo"][0]
)
assert "/photo.jpg" == result["items"][3]["properties"]["in-reply-to"][0]["value"]
assert "alt" not in result["items"][3]["properties"]["in-reply-to"][0]
assert "h-cite" in exp_result["items"][3]["properties"]["in-reply-to"][0]["type"]
assert (
"/photo.jpg"
== exp_result["items"][3]["properties"]["in-reply-to"][0]["properties"][
"photo"
][0]
)
assert (
"/photo.jpg" == exp_result["items"][3]["properties"]["in-reply-to"][0]["value"]
)
assert "alt" not in exp_result["items"][3]["properties"]["in-reply-to"][0]
assert "h-cite" in result["items"][4]["properties"]["in-reply-to"][0]["type"]
assert {"alt": "alt text", "value": "/photo.jpg"} == result["items"][4][
"properties"
]["in-reply-to"][0]["properties"]["photo"][0]
assert "/photo.jpg" == result["items"][4]["properties"]["in-reply-to"][0]["value"]
assert "alt" in result["items"][4]["properties"]["in-reply-to"][0]
assert "h-cite" in exp_result["items"][4]["properties"]["in-reply-to"][0]["type"]
assert (
"/photo.jpg"
== exp_result["items"][4]["properties"]["in-reply-to"][0]["properties"][
"photo"
][0]["value"]
)
assert (
"/photo.jpg" == exp_result["items"][4]["properties"]["in-reply-to"][0]["value"]
)
assert (
"alt text"
== exp_result["items"][4]["properties"]["in-reply-to"][0]["properties"][
"photo"
][0]["alt"]
)
assert "alt text" == exp_result["items"][4]["properties"]["in-reply-to"][0]["alt"]
assert "h-cite" in result["items"][5]["properties"]["in-reply-to"][0]["type"]
assert {"alt": "", "value": "/photo.jpg"} == result["items"][5]["properties"][
"in-reply-to"
][0]["properties"]["photo"][0]
assert "/photo.jpg" == result["items"][5]["properties"]["in-reply-to"][0]["value"]
assert "alt" in result["items"][5]["properties"]["in-reply-to"][0]
assert "h-cite" in exp_result["items"][5]["properties"]["in-reply-to"][0]["type"]
assert (
"/photo.jpg"
== exp_result["items"][5]["properties"]["in-reply-to"][0]["properties"][
"photo"
][0]["value"]
)
assert (
"/photo.jpg" == exp_result["items"][5]["properties"]["in-reply-to"][0]["value"]
)
assert (
""
== exp_result["items"][5]["properties"]["in-reply-to"][0]["properties"][
"photo"
][0]["alt"]
)
assert "" == exp_result["items"][5]["properties"]["in-reply-to"][0]["alt"]
def test_photo_with_srcset():
result = parse_fixture("img_with_srcset.html")
assert result["items"][0]["properties"]["photo"][0]["srcset"] == {
"480w": "elva-fairy-480w.jpg",
"800w": "elva-fairy-800w.jpg",
}
assert result["items"][1]["properties"]["photo"][0]["srcset"] == {
"1x": "elva-fairy-320w.jpg",
"1.5x": "elva-fairy-480w.jpg",
"2x": "elva-fairy-640w.jpg",
}
assert (
result["items"][1]["properties"]["photo"][0]["srcset"]["2x"]
!= "elva-fairy-2w.jpg"
)
for i in range(2, 7):
assert result["items"][i]["properties"]["photo"][0]["srcset"] == {
"1x": "elva-fairy,320w.jpg",
"1.5x": "elva-fairy,480w.jpg",
}
assert result["items"][7]["properties"]["photo"][0]["srcset"] == {
"1x": "elva-fairy,320w.jpg",
}
assert result["items"][8]["properties"]["photo"][0]["srcset"] == {
"1x": "elva-fairy,320w.jpg",
"1.5x": "elva-fairy,480w.jpg",
"2x": "elva-fairy,640w.jpg",
}
result = parse_fixture("img_with_srcset_with_base.html")
assert result["items"][0]["properties"]["photo"][0]["srcset"] == {
"480w": "https://example.com/elva-fairy-480w.jpg",
"800w": "https://example.com/elva-fairy-800w.jpg",
}
def test_parse_id():
result = parse_fixture("parse_id.html")
assert "recentArticles" == result["items"][0]["id"]
assert "article" == result["items"][0]["children"][0]["id"]
assert "id" not in result["items"][0]["children"][1]
assert "theAuthor" == result["items"][0]["properties"]["author"][0]["id"]
# unicode tests
def get_all_files():
all_files = []
for dir_, _, files in os.walk(TEST_DIR):
for filename in files:
rel_dir = os.path.relpath(dir_, TEST_DIR)
all_files.append(os.path.join(rel_dir, filename))
return all_files
def assert_unicode_everywhere(obj):
if isinstance(obj, dict):
for k, v in obj.items():
assert not isinstance(k, bytes), "key=%r; type=%r" % (k, type(k))
assert_unicode_everywhere(v)
elif isinstance(obj, list):
for v in obj:
assert_unicode_everywhere(v)
assert not isinstance(obj, bytes), "value=%r; type=%r" % (obj, type(obj))
def check_unicode(filename, jsonblob):
assert_unicode_everywhere(jsonblob)
def test_unicode_everywhere():
"""make sure everything is unicode"""
for h in get_all_files():
result = parse_fixture(h)
check_unicode(h, result)
def test_input_tree_integrity():
"""make sure that if we parse a BS4 soup, our modifications do not leak into the document represented by it"""
for path in get_all_files():
with open(os.path.join(TEST_DIR, path)) as f:
soup = BeautifulSoup(f, features="lxml")
html1 = soup.prettify()
p = Parser(doc=soup, html_parser="lxml")
html2 = soup.prettify()
make_labelled_cmp("tree_integrity_" + path)(html1, html2)
def make_labelled_cmp(label):
def f(html1, html2):
assert html1 == html2
f.description = label
return f
def test_all_u_cases():
"""test variations of u- parsing and that relative urls are always resolved"""
URL_COUNT = 28
result = parse_fixture("u_all_cases.html")
assert URL_COUNT == len(result["items"][0]["properties"]["url"])
for i in range(URL_COUNT):
make_labelled_cmp("all_u_cases_" + str(i))(
"http://example.com/test", result["items"][0]["properties"]["url"][i]
)
def test_filtered_roots():
result = parse_fixture("filter_roots.html")
assert len(result["items"]) == 8
result = parse_fixture("filter_roots.html", filter_roots=True)
assert len(result["items"]) == 1
result = parse_fixture(
"filter_roots_custom.html", filter_roots={"foo", "bar", "bat", "baz"}
)
assert len(result["items"]) == 1
def test_metaformats_flag_false():
result = parse_fixture("metaformats_ogp.html")
assert result["items"] == []
def test_metaformats_title_only():
result = parse_fixture("base.html", metaformats=True)
assert result["items"] == [
{
"type": ["h-entry"],
"properties": {
"name": ["Hello World"],
},
"source": "metaformats",
}
]
def test_metaformats_ogp():
result = parse_fixture("metaformats_ogp.html", metaformats=True)
assert result["items"] == [
{
"type": ["h-entry"],
"properties": {
"name": ["Titull foo"],
"summary": ["Descrypshun bar"],
"photo": ["http://example.com/baz.jpg"],
"audio": ["http://example.com/biff.mp3"],
"video": ["http://example.com/boff.mov"],
"author": ["http://tantek.com/me"],
"published": ["2023-01-02T03:04Z"],
"updated": ["2023-01-02T05:06Z"],
},
"source": "metaformats",
}
]
def test_metaformats_twitter():
result = parse_fixture("metaformats_twitter.html", metaformats=True)
assert result["items"] == [
{
"type": ["h-entry"],
"properties": {
"name": ["Titull foo"],
"summary": ["Descrypshun bar"],
"photo": ["http://tantek.com/baz.jpg"],
},
"source": "metaformats",
}
]
def test_metaformats_html_meta():
result = parse_fixture("metaformats_html_meta.html", metaformats=True)
assert result["items"] == [
{
"type": ["h-entry"],
"properties": {
"name": ["Hello World"],
"summary": ["Descrypshun bar"],
},
"source": "metaformats",
}
]
def test_language():
result = parse_fixture("language.html")
assert result["items"][0]["lang"] == "it"
assert result["items"][1]["lang"] == "it"
assert result["items"][1]["properties"]["content"][0]["lang"] == "en"
assert result["items"][1]["properties"]["content"][1]["lang"] == "it"
assert result["items"][2]["lang"] == "sv"
assert result["items"][2]["properties"]["content"][0]["lang"] == "en"
assert result["items"][2]["properties"]["content"][1]["lang"] == "sv"
def test_parser_object():
with open(os.path.join(TEST_DIR, "festivus.html")) as f:
p = Parser(doc=f)
assert len(p.to_dict(filter_by_type="h-card")) == 3
assert len(p.to_dict(filter_by_type="h-entry")) == 4
assert (
p.to_json(filter_by_type="h-card")
== '[{"type": ["h-card"], "properties": {"name": ["Jerry"]}}, {"type": '
'["h-card"], "properties": {"name": ["Frank"]}}, {"type": ["h-card"], '
'"properties": {"name": ["Cosmo"]}}]'
)
mf2py-2.0.1/test/test_suite.py 0000664 0000000 0000000 00000001333 14534511526 0016276 0 ustar 00root root 0000000 0000000 import glob
import json
import os.path
import sys
from test_parser import check_unicode
import mf2py
def test_mf2tests():
allfiles = glob.glob(os.path.join(".", "testsuite", "tests", "*", "*", "*.json"))
for jsonfile in allfiles:
htmlfile = jsonfile[:-4] + "html"
with open(htmlfile) as f:
p = mf2py.parse(doc=f, url="http://example.com")
check_unicode(htmlfile, p)
with open(jsonfile) as jsonf:
try:
s = json.load(jsonf)
except:
s = "bad file: " + jsonfile + sys.exc_info()[0]
check_mf2(htmlfile, p, s)
def check_mf2(htmlfile, p, s):
# TODO ignore extra keys in p that are not in s
assert p == s