diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..3ba8d1ab --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,15 @@ +# Agent guidelines + +Read [CONTRIBUTING.rst](CONTRIBUTING.rst) for code style, linting (e.g. `make lint`, `make fmt`), and release process. + +## Testing + +- Prefer **function-based** pytest tests (module-level `def test_...`) over class-based tests (`class Test...`). +- Use descriptive test names that reflect behavior (e.g. `test_remote_context_via_link_alternate`). + +## Committing + +- Prefer one file per commit +- Base the commit message on the diff of the file that you are going to commit +- Prefer one line commit messages +- Do not co-author commits diff --git a/Makefile b/Makefile index b11f362e..c3e3a211 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ upgrade-submodules: git submodule update --remote --init --recursive # TODO: Expand to lib/ and tests/ as linting issues are resolved. -RUFF_TARGET = lib/pyld/context_resolver.py lib/pyld/identifier_issuer.py lib/pyld/iri_resolver.py lib/pyld/nquads.py lib/pyld/resolved_context.py +RUFF_TARGET = lib/pyld/context_resolver.py lib/pyld/identifier_issuer.py lib/pyld/iri_resolver.py lib/pyld/nquads.py lib/pyld/resolved_context.py tests/test_document_loader.py lint: ruff check $(RUFF_TARGET) diff --git a/lib/pyld/documentloader/aiohttp.py b/lib/pyld/documentloader/aiohttp.py index 123f6668..cf02f0a8 100644 --- a/lib/pyld/documentloader/aiohttp.py +++ b/lib/pyld/documentloader/aiohttp.py @@ -82,9 +82,6 @@ async def async_loader(url, headers): async with session.get(url, headers=headers, **kwargs) as response: - # Allow any content_type in trying to parse json - # similar to requests library - json_body = await response.json(content_type=None) content_type = response.headers.get('content-type') if not content_type: content_type = 'application/octet-stream' @@ -92,7 +89,6 @@ async def async_loader(url, headers): 'contentType': content_type, 'contextUrl': None, 'documentUrl': response.url.human_repr(), - 'document': json_body } link_header = response.headers.get('link') if link_header: @@ -100,15 +96,15 @@ async def async_loader(url, headers): LINK_HEADER_REL) # only 1 related link header permitted if linked_context and content_type != 'application/ld+json': - if isinstance(linked_context, list): - raise JsonLdError( - 'URL could not be dereferenced, ' - 'it has more than one ' - 'associated HTTP Link Header.', - 'jsonld.LoadDocumentError', - {'url': url}, - code='multiple context link headers') - doc['contextUrl'] = linked_context['target'] + if isinstance(linked_context, list): + raise JsonLdError( + 'URL could not be dereferenced, ' + 'it has more than one ' + 'associated HTTP Link Header.', + 'jsonld.LoadDocumentError', + {'url': url}, + code='multiple context link headers') + doc['contextUrl'] = linked_context['target'] linked_alternate = parse_link_header(link_header).get('alternate') # if not JSON-LD, alternate may point there if (linked_alternate and @@ -116,7 +112,8 @@ async def async_loader(url, headers): not re.match(r'^application\/(\w*\+)?json$', content_type)): doc['contentType'] = 'application/ld+json' doc['documentUrl'] = iri_resolver.resolve(linked_alternate['target'], url) - + return await async_loader(doc['documentUrl'], headers) + doc['document'] = await response.json(content_type=None) return doc except JsonLdError as e: raise e diff --git a/lib/pyld/documentloader/requests.py b/lib/pyld/documentloader/requests.py index 0015bdf4..5ee30fd5 100644 --- a/lib/pyld/documentloader/requests.py +++ b/lib/pyld/documentloader/requests.py @@ -14,7 +14,7 @@ import urllib.parse as urllib_parse from pyld import iri_resolver -from pyld.jsonld import (JsonLdError, parse_link_header, LINK_HEADER_REL) +from pyld.jsonld import JsonLdError, parse_link_header, LINK_HEADER_REL def requests_document_loader(secure=False, **kwargs): @@ -71,7 +71,6 @@ def loader(url, options={}): 'contentType': content_type, 'contextUrl': None, 'documentUrl': response.url, - 'document': response.json() } link_header = response.headers.get('link') if link_header: @@ -79,15 +78,15 @@ def loader(url, options={}): LINK_HEADER_REL) # only 1 related link header permitted if linked_context and content_type != 'application/ld+json': - if isinstance(linked_context, list): - raise JsonLdError( - 'URL could not be dereferenced, ' - 'it has more than one ' - 'associated HTTP Link Header.', - 'jsonld.LoadDocumentError', - {'url': url}, - code='multiple context link headers') - doc['contextUrl'] = linked_context['target'] + if isinstance(linked_context, list): + raise JsonLdError( + 'URL could not be dereferenced, ' + 'it has more than one ' + 'associated HTTP Link Header.', + 'jsonld.LoadDocumentError', + {'url': url}, + code='multiple context link headers') + doc['contextUrl'] = linked_context['target'] linked_alternate = parse_link_header(link_header).get('alternate') # if not JSON-LD, alternate may point there if (linked_alternate and @@ -95,13 +94,15 @@ def loader(url, options={}): not re.match(r'^application\/(\w*\+)?json$', content_type)): doc['contentType'] = 'application/ld+json' doc['documentUrl'] = iri_resolver.resolve(linked_alternate['target'], url) + return loader(doc['documentUrl'], options=options) + doc['document'] = response.json() return doc except JsonLdError as e: raise e except Exception as cause: raise JsonLdError( 'Could not retrieve a JSON-LD document from the URL.', - 'jsonld.LoadDocumentError', + 'jsonld.LoadDocumentError', code='loading document failed') from cause return loader diff --git a/tests/test_document_loader.py b/tests/test_document_loader.py index fccfc55a..33b041a2 100644 --- a/tests/test_document_loader.py +++ b/tests/test_document_loader.py @@ -1,9 +1,15 @@ """ -Tests for load_document function, specifically Accept header content negotiation. +Tests for document loaders: Accept header content negotiation and Link rel=alternate. -See: https://github.com/digitalbazaar/pyld/pull/170 +- PR #170: load_document and Accept header content negotiation (e.g. ActivityStreams). +- Issue #128: When a context URL returns non-JSON-LD (e.g. text/html), the loader must + follow a Link header with rel="alternate" and type="application/ld+json" to get the + actual JSON-LD. Example: https://schema.org responds with text/html and provides the + JSON-LD context via Link: ; rel="alternate"; type="application/ld+json" """ + import pytest + from pyld import jsonld @@ -11,20 +17,68 @@ def test_activitystreams_context_loads_as_json(): """ The ActivityStreams context URL should return JSON-LD, not the HTML spec. - + This was the original bug report for PR #170. - + This test requires network access and may be slow or flaky. """ - options = { - 'documentLoader': jsonld.requests_document_loader() - } - - result = jsonld.load_document( - 'https://www.w3.org/ns/activitystreams', - options - ) - + options = {'documentLoader': jsonld.requests_document_loader()} + + result = jsonld.load_document('https://www.w3.org/ns/activitystreams', options) + # Should be JSON-LD context, not HTML spec assert isinstance(result['document'], dict) assert '@context' in result['document'] + + +# Minimal JSON-LD document that uses https://schema.org as context. +# schema.org serves text/html with Link rel="alternate" to the JSON-LD context. +_DOC_CONTEXT_VIA_LINK_ALTERNATE = { + "@context": "https://schema.org", + "@type": "Person", + "name": "Jane Doe", + "jobTitle": "Professor", + "telephone": "(425) 123-4567", + "url": "http://www.janedoe.com", +} + + +def _expand_with_loader(loader): + """Expand _DOC_CONTEXT_VIA_LINK_ALTERNATE with the given loader.""" + return jsonld.expand( + _DOC_CONTEXT_VIA_LINK_ALTERNATE, + options={"documentLoader": loader}, + ) + + +def _assert_expanded_link_alternate_result(result): + """Verify expansion produced the expected structure (context loaded via Link rel=alternate).""" + assert isinstance(result, list) + assert len(result) == 1 + item = result[0] + assert "http://schema.org/name" in item + assert item["http://schema.org/name"] == [{"@value": "Jane Doe"}] + assert "@type" in item + assert "http://schema.org/Person" in item["@type"] + + +_LOADERS = { + "requests": jsonld.requests_document_loader, + "aiohttp": jsonld.aiohttp_document_loader, +} + + +@pytest.fixture(params=["requests", "aiohttp"]) +def document_loader(request): + """Parametrizing fixture: yields requests and aiohttp document loaders.""" + return _LOADERS[request.param]() + + +@pytest.mark.network +def test_remote_context_via_link_alternate(document_loader): + """ + When context URL returns non-JSON-LD (e.g. text/html) with Link rel=alternate + type=application/ld+json, the loader follows that link to load the context. + """ + result = _expand_with_loader(document_loader) + _assert_expanded_link_alternate_result(result) diff --git a/tests/test_jsonld.py b/tests/test_jsonld.py index a80154a3..9026ed46 100644 --- a/tests/test_jsonld.py +++ b/tests/test_jsonld.py @@ -433,6 +433,7 @@ def test_remote_context_half_context_local_and_half_remote(self): assert framed == expected # Issue 59 - PR: https://github.com/digitalbazaar/pyld/pull/60 + @pytest.mark.network def test_do_not_compact_dates_without_datatype(self): """ Dates without explicit datatype should not be compacted during framing, @@ -443,10 +444,10 @@ def test_do_not_compact_dates_without_datatype(self): "http://schema.org/deathDate": "2015-02-25", } - frame = {"@context": "https://schema.org/docs/jsonldcontext.jsonld"} + frame = {"@context": "https://schema.org/"} expected = { - "@context": "https://schema.org/docs/jsonldcontext.jsonld", + "@context": "https://schema.org/", "name": "Buster the Cat", "schema:birthDate": "2012", "schema:deathDate": "2015-02-25", @@ -455,6 +456,7 @@ def test_do_not_compact_dates_without_datatype(self): framed = jsonld.frame(input, frame) assert framed == expected + @pytest.mark.network def test_compact_dates_with_datatype(self): """ Dates with explicit datatype should be compacted during framing. @@ -471,10 +473,10 @@ def test_compact_dates_with_datatype(self): }, } - frame = {"@context": "https://schema.org/docs/jsonldcontext.jsonld"} + frame = {"@context": "https://schema.org/"} expected = { - "@context": "https://schema.org/docs/jsonldcontext.jsonld", + "@context": "https://schema.org/", "name": "Buster the Cat", "birthDate": "2012", "deathDate": "2015-02-25",