{
"event": "PreToolUse",
"tool_name": "Write",
"tool_input": {
"file_path": "\/var\/www\/dev.campus.systemische-tools.de\/crawler\/src\/application\/use_cases\/process_url.py",
"content": "from __future__ import annotations\n\nfrom dataclasses import dataclass\nfrom datetime import datetime\n\nfrom src.application.use_cases.extract_seo_data import ExtractInput, ExtractSeoDataUseCase\nfrom src.domain.entities.image import Image\nfrom src.domain.entities.link import Link\nfrom src.domain.entities.page import Page\nfrom src.domain.entities.structured_data_item import StructuredDataItem\nfrom src.domain.entities.url import Url\nfrom src.domain.ports.clock_port import ClockPort\nfrom src.domain.ports.hreflang_repository_port import HreflangRepositoryPort\nfrom src.domain.ports.html_parser_port import HtmlParserPort, ParsedDocument\nfrom src.domain.ports.http_fetcher_port import HttpFetcherPort\nfrom src.domain.ports.image_repository_port import ImageRepositoryPort\nfrom src.domain.ports.link_repository_port import LinkRepositoryPort\nfrom src.domain.ports.page_repository_port import PageRepositoryPort\nfrom src.domain.ports.structured_data_parser_port import StructuredDataParserPort\nfrom src.domain.ports.structured_data_repository_port import StructuredDataRepositoryPort\nfrom src.domain.ports.url_queue_port import UrlQueuePort\nfrom src.domain.ports.url_repository_port import UrlRepositoryPort\nfrom src.domain.services.link_classifier import LinkClassifier\nfrom src.domain.services.url_normalizer import UrlNormalizer\nfrom src.domain.value_objects.enums import RenderMode\nfrom src.domain.value_objects.normalized_url import NormalizedUrl\n\n\n@dataclass(frozen=True, slots=True)\nclass ProcessUrlDeps:\n fetcher: HttpFetcherPort\n html_parser: HtmlParserPort\n sd_parser: StructuredDataParserPort\n pages: PageRepositoryPort\n urls: UrlRepositoryPort\n links: LinkRepositoryPort\n images: ImageRepositoryPort\n structured: StructuredDataRepositoryPort\n hreflang: HreflangRepositoryPort\n queue: UrlQueuePort\n classifier: LinkClassifier\n normalizer: UrlNormalizer\n extractor: ExtractSeoDataUseCase\n clock: ClockPort\n\n\n@dataclass(frozen=True, slots=True)\nclass _LinksContext:\n page_id: int\n source_url_id: int\n crawl_id: int\n now: datetime\n\n\nclass ProcessUrlUseCase:\n def __init__(self, deps: ProcessUrlDeps) -> None:\n self._d = deps\n\n async def execute(self, *, crawl_id: int, url: NormalizedUrl) -> Page:\n now = self._d.clock.now()\n fetch = await self._d.fetcher.fetch(url)\n parsed = self._d.html_parser.parse(\n fetch.body.decode(\"utf-8\", errors=\"replace\"), base_url=url.url,\n )\n url_entity = self._upsert_url(url, now=now, is_internal=True)\n extracted = self._d.extractor.execute(\n ExtractInput(\n crawl_id=crawl_id, url_id=_id(url_entity), fetched_at=now,\n render_mode=RenderMode.HTTP, fetch=fetch, parsed=parsed, page_url=url.url,\n ),\n )\n page = self._d.pages.save(extracted.page)\n ctx = _LinksContext(page_id=_id(page), source_url_id=_id(url_entity),\n crawl_id=crawl_id, now=now)\n self._persist_links(ctx, parsed=parsed)\n self._persist_images(page_id=_id(page), parsed=parsed)\n self._d.hreflang.save_many(_id(page), parsed.hreflang)\n self._persist_structured_data(page_id=_id(page), html=fetch.body, base_url=url.url)\n self._d.queue.mark_done(crawl_id, url)\n return page\n\n def _upsert_url(self, normalized: NormalizedUrl, *, now: datetime, is_internal: bool) -> Url:\n return self._d.urls.upsert(Url(\n id=None, normalized=normalized, is_internal=is_internal,\n first_seen_at=now, last_seen_at=now,\n ))\n\n def _persist_links(self, ctx: _LinksContext, *, parsed: ParsedDocument) -> None:\n links: list[Link] = []\n for raw in parsed.internal_links:\n normalized = self._d.normalizer.normalize(raw)\n target = self._upsert_url(normalized, now=ctx.now, is_internal=True)\n links.append(_make_link(ctx.page_id, ctx.source_url_id, normalized.url,\n target_url_id=_id(target), is_internal=True))\n self._d.queue.enqueue(ctx.crawl_id, normalized)\n for raw in parsed.external_links:\n normalized = self._d.normalizer.normalize(raw)\n links.append(_make_link(ctx.page_id, ctx.source_url_id, normalized.url,\n target_url_id=None, is_internal=False))\n self._d.links.save_many(links)\n\n def _persist_images(self, *, page_id: int, parsed: ParsedDocument) -> None:\n imgs = [Image(id=None, page_id=page_id, src=s, alt=a, title=t,\n width=w, height=h, loading_attr=None, decoding_attr=None,\n http_status=None, content_length=None)\n for s, a, t, w, h in parsed.image_specs]\n self._d.images.save_many(imgs)\n\n def _persist_structured_data(self, *, page_id: int, html: bytes, base_url: str) -> None:\n items = self._d.sd_parser.parse(html.decode(\"utf-8\", errors=\"replace\"), base_url=base_url)\n sd = [StructuredDataItem(id=None, page_id=page_id, format=f, schema_type=t, payload=p)\n for f, t, p in items]\n self._d.structured.save_many(sd)\n\n\ndef _id(entity: Page | Url) -> int:\n if entity.id is None:\n raise RuntimeError(f\"{type(entity).__name__} has no id after save\")\n return entity.id\n\n\ndef _make_link(page_id: int, source_url_id: int, target_url: str,\n *, target_url_id: int | None, is_internal: bool) -> Link:\n return Link(id=None, page_id=page_id, source_url_id=source_url_id, target_url=target_url,\n target_url_id=target_url_id, anchor_text=None, rel=frozenset(),\n is_internal=is_internal, is_external=not is_internal, position_hint=None)\n"
}
}
{
"tool_response": {
"type": "update",
"filePath": "\/var\/www\/dev.campus.systemische-tools.de\/crawler\/src\/application\/use_cases\/process_url.py",
"content": "from __future__ import annotations\n\nfrom dataclasses import dataclass\nfrom datetime import datetime\n\nfrom src.application.use_cases.extract_seo_data import ExtractInput, ExtractSeoDataUseCase\nfrom src.domain.entities.image import Image\nfrom src.domain.entities.link import Link\nfrom src.domain.entities.page import Page\nfrom src.domain.entities.structured_data_item import StructuredDataItem\nfrom src.domain.entities.url import Url\nfrom src.domain.ports.clock_port import ClockPort\nfrom src.domain.ports.hreflang_repository_port import HreflangRepositoryPort\nfrom src.domain.ports.html_parser_port import HtmlParserPort, ParsedDocument\nfrom src.domain.ports.http_fetcher_port import HttpFetcherPort\nfrom src.domain.ports.image_repository_port import ImageRepositoryPort\nfrom src.domain.ports.link_repository_port import LinkRepositoryPort\nfrom src.domain.ports.page_repository_port import PageRepositoryPort\nfrom src.domain.ports.structured_data_parser_port import StructuredDataParserPort\nfrom src.domain.ports.structured_data_repository_port import StructuredDataRepositoryPort\nfrom src.domain.ports.url_queue_port import UrlQueuePort\nfrom src.domain.ports.url_repository_port import UrlRepositoryPort\nfrom src.domain.services.link_classifier import LinkClassifier\nfrom src.domain.services.url_normalizer import UrlNormalizer\nfrom src.domain.value_objects.enums import RenderMode\nfrom src.domain.value_objects.normalized_url import NormalizedUrl\n\n\n@dataclass(frozen=True, slots=True)\nclass ProcessUrlDeps:\n fetcher: HttpFetcherPort\n html_parser: HtmlParserPort\n sd_parser: StructuredDataParserPort\n pages: PageRepositoryPort\n urls: UrlRepositoryPort\n links: LinkRepositoryPort\n images: ImageRepositoryPort\n structured: StructuredDataRepositoryPort\n hreflang: HreflangRepositoryPort\n queue: UrlQueuePort\n classifier: LinkClassifier\n normalizer: UrlNormalizer\n extractor: ExtractSeoDataUseCase\n clock: ClockPort\n\n\n@dataclass(frozen=True, slots=True)\nclass _LinksContext:\n page_id: int\n source_url_id: int\n crawl_id: int\n now: datetime\n\n\nclass ProcessUrlUseCase:\n def __init__(self, deps: ProcessUrlDeps) -> None:\n self._d = deps\n\n async def execute(self, *, crawl_id: int, url: NormalizedUrl) -> Page:\n now = self._d.clock.now()\n fetch = await self._d.fetcher.fetch(url)\n parsed = self._d.html_parser.parse(\n fetch.body.decode(\"utf-8\", errors=\"replace\"), base_url=url.url,\n )\n url_entity = self._upsert_url(url, now=now, is_internal=True)\n extracted = self._d.extractor.execute(\n ExtractInput(\n crawl_id=crawl_id, url_id=_id(url_entity), fetched_at=now,\n render_mode=RenderMode.HTTP, fetch=fetch, parsed=parsed, page_url=url.url,\n ),\n )\n page = self._d.pages.save(extracted.page)\n ctx = _LinksContext(page_id=_id(page), source_url_id=_id(url_entity),\n crawl_id=crawl_id, now=now)\n self._persist_links(ctx, parsed=parsed)\n self._persist_images(page_id=_id(page), parsed=parsed)\n self._d.hreflang.save_many(_id(page), parsed.hreflang)\n self._persist_structured_data(page_id=_id(page), html=fetch.body, base_url=url.url)\n self._d.queue.mark_done(crawl_id, url)\n return page\n\n def _upsert_url(self, normalized: NormalizedUrl, *, now: datetime, is_internal: bool) -> Url:\n return self._d.urls.upsert(Url(\n id=None, normalized=normalized, is_internal=is_internal,\n first_seen_at=now, last_seen_at=now,\n ))\n\n def _persist_links(self, ctx: _LinksContext, *, parsed: ParsedDocument) -> None:\n links: list[Link] = []\n for raw in parsed.internal_links:\n normalized = self._d.normalizer.normalize(raw)\n target = self._upsert_url(normalized, now=ctx.now, is_internal=True)\n links.append(_make_link(ctx.page_id, ctx.source_url_id, normalized.url,\n target_url_id=_id(target), is_internal=True))\n self._d.queue.enqueue(ctx.crawl_id, normalized)\n for raw in parsed.external_links:\n normalized = self._d.normalizer.normalize(raw)\n links.append(_make_link(ctx.page_id, ctx.source_url_id, normalized.url,\n target_url_id=None, is_internal=False))\n self._d.links.save_many(links)\n\n def _persist_images(self, *, page_id: int, parsed: ParsedDocument) -> None:\n imgs = [Image(id=None, page_id=page_id, src=s, alt=a, title=t,\n width=w, height=h, loading_attr=None, decoding_attr=None,\n http_status=None, content_length=None)\n for s, a, t, w, h in parsed.image_specs]\n self._d.images.save_many(imgs)\n\n def _persist_structured_data(self, *, page_id: int, html: bytes, base_url: str) -> None:\n items = self._d.sd_parser.parse(html.decode(\"utf-8\", errors=\"replace\"), base_url=base_url)\n sd = [StructuredDataItem(id=None, page_id=page_id, format=f, schema_type=t, payload=p)\n for f, t, p in items]\n self._d.structured.save_many(sd)\n\n\ndef _id(entity: Page | Url) -> int:\n if entity.id is None:\n raise RuntimeError(f\"{type(entity).__name__} has no id after save\")\n return entity.id\n\n\ndef _make_link(page_id: int, source_url_id: int, target_url: str,\n *, target_url_id: int | None, is_internal: bool) -> Link:\n return Link(id=None, page_id=page_id, source_url_id=source_url_id, target_url=target_url,\n target_url_id=target_url_id, anchor_text=None, rel=frozenset(),\n is_internal=is_internal, is_external=not is_internal, position_hint=None)\n",
"structuredPatch": [
{
"oldStart": 1,
"oldLines": 8,
"newStart": 1,
"newLines": 9,
"lines": [
" from __future__ import annotations",
" ",
" from dataclasses import dataclass",
"+from datetime import datetime",
" ",
"-from src.application.use_cases.extract_seo_data import ExtractSeoDataUseCase",
"+from src.application.use_cases.extract_seo_data import ExtractInput, ExtractSeoDataUseCase",
" from src.domain.entities.image import Image",
" from src.domain.entities.link import Link",
" from src.domain.entities.page import Page"
]
},
{
"oldStart": 10,
"oldLines": 7,
"newStart": 11,
"newLines": 7,
"lines": [
" from src.domain.entities.url import Url",
" from src.domain.ports.clock_port import ClockPort",
" from src.domain.ports.hreflang_repository_port import HreflangRepositoryPort",
"-from src.domain.ports.html_parser_port import HtmlParserPort",
"+from src.domain.ports.html_parser_port import HtmlParserPort, ParsedDocument",
" from src.domain.ports.http_fetcher_port import HttpFetcherPort",
" from src.domain.ports.image_repository_port import ImageRepositoryPort",
" from src.domain.ports.link_repository_port import LinkRepositoryPort"
]
},
{
"oldStart": 43,
"oldLines": 6,
"newStart": 44,
"newLines": 14,
"lines": [
" clock: ClockPort",
" ",
" ",
"+@dataclass(frozen=True, slots=True)",
"+class _LinksContext:",
"+ page_id: int",
"+ source_url_id: int",
"+ crawl_id: int",
"+ now: datetime",
"+",
"+",
" class ProcessUrlUseCase:",
" def __init__(self, deps: ProcessUrlDeps) -> None:",
" self._d = deps"
]
},
{
"oldStart": 50,
"oldLines": 57,
"newStart": 59,
"newLines": 53,
"lines": [
" async def execute(self, *, crawl_id: int, url: NormalizedUrl) -> Page:",
" now = self._d.clock.now()",
" fetch = await self._d.fetcher.fetch(url)",
"- parsed = self._d.html_parser.parse(fetch.body.decode(\"utf-8\", errors=\"replace\"),",
"- base_url=url.url)",
"+ parsed = self._d.html_parser.parse(",
"+ fetch.body.decode(\"utf-8\", errors=\"replace\"), base_url=url.url,",
"+ )",
" url_entity = self._upsert_url(url, now=now, is_internal=True)",
" extracted = self._d.extractor.execute(",
"- crawl_id=crawl_id, url_id=_id_of(url_entity), fetched_at=now,",
"- render_mode=RenderMode.HTTP, fetch=fetch, parsed=parsed, page_url=url.url,",
"+ ExtractInput(",
"+ crawl_id=crawl_id, url_id=_id(url_entity), fetched_at=now,",
"+ render_mode=RenderMode.HTTP, fetch=fetch, parsed=parsed, page_url=url.url,",
"+ ),",
" )",
" page = self._d.pages.save(extracted.page)",
"- self._persist_links(page_id=_id_of(page), source_url_id=_id_of(url_entity),",
"- parsed_internal=parsed.internal_links,",
"- parsed_external=parsed.external_links,",
"+ ctx = _LinksContext(page_id=_id(page), source_url_id=_id(url_entity),",
" crawl_id=crawl_id, now=now)",
"- self._persist_images(page_id=_id_of(page), parsed=parsed)",
"- self._persist_hreflang(page_id=_id_of(page), parsed=parsed)",
"- self._persist_structured_data(page_id=_id_of(page), html=fetch.body, base_url=url.url)",
"+ self._persist_links(ctx, parsed=parsed)",
"+ self._persist_images(page_id=_id(page), parsed=parsed)",
"+ self._d.hreflang.save_many(_id(page), parsed.hreflang)",
"+ self._persist_structured_data(page_id=_id(page), html=fetch.body, base_url=url.url)",
" self._d.queue.mark_done(crawl_id, url)",
" return page",
" ",
"- def _upsert_url(self, normalized: NormalizedUrl, *, now: object, is_internal: bool) -> Url:",
"+ def _upsert_url(self, normalized: NormalizedUrl, *, now: datetime, is_internal: bool) -> Url:",
" return self._d.urls.upsert(Url(",
" id=None, normalized=normalized, is_internal=is_internal,",
"- first_seen_at=now, last_seen_at=now, # type: ignore[arg-type]",
"+ first_seen_at=now, last_seen_at=now,",
" ))",
" ",
"- def _persist_links(self, *, page_id: int, source_url_id: int,",
"- parsed_internal: tuple[str, ...], parsed_external: tuple[str, ...],",
"- crawl_id: int, now: object) -> None:",
"+ def _persist_links(self, ctx: _LinksContext, *, parsed: ParsedDocument) -> None:",
" links: list[Link] = []",
"- for raw in parsed_internal:",
"+ for raw in parsed.internal_links:",
" normalized = self._d.normalizer.normalize(raw)",
"- target = self._upsert_url(normalized, now=now, is_internal=True)",
"- links.append(_make_link(page_id, source_url_id, normalized.url,",
"- target_url_id=_id_of(target), is_internal=True))",
"- self._d.queue.enqueue(crawl_id, normalized)",
"- for raw in parsed_external:",
"+ target = self._upsert_url(normalized, now=ctx.now, is_internal=True)",
"+ links.append(_make_link(ctx.page_id, ctx.source_url_id, normalized.url,",
"+ target_url_id=_id(target), is_internal=True))",
"+ self._d.queue.enqueue(ctx.crawl_id, normalized)",
"+ for raw in parsed.external_links:",
" normalized = self._d.normalizer.normalize(raw)",
"- links.append(_make_link(page_id, source_url_id, normalized.url,",
"+ links.append(_make_link(ctx.page_id, ctx.source_url_id, normalized.url,",
" target_url_id=None, is_internal=False))",
" self._d.links.save_many(links)",
" ",
"- def _persist_images(self, *, page_id: int, parsed: object) -> None:",
"- specs = parsed.image_specs",
"+ def _persist_images(self, *, page_id: int, parsed: ParsedDocument) -> None:",
" imgs = [Image(id=None, page_id=page_id, src=s, alt=a, title=t,",
" width=w, height=h, loading_attr=None, decoding_attr=None,",
" http_status=None, content_length=None)",
"- for s, a, t, w, h in specs]",
"+ for s, a, t, w, h in parsed.image_specs]",
" self._d.images.save_many(imgs)",
" ",
"- def _persist_hreflang(self, *, page_id: int, parsed: object) -> None:",
"- self._d.hreflang.save_many(page_id, parsed.hreflang)",
"-",
" def _persist_structured_data(self, *, page_id: int, html: bytes, base_url: str) -> None:",
" items = self._d.sd_parser.parse(html.decode(\"utf-8\", errors=\"replace\"), base_url=base_url)",
" sd = [StructuredDataItem(id=None, page_id=page_id, format=f, schema_type=t, payload=p)"
]
},
{
"oldStart": 108,
"oldLines": 10,
"newStart": 113,
"newLines": 10,
"lines": [
" self._d.structured.save_many(sd)",
" ",
" ",
"-def _id_of(entity: object) -> int:",
"- v = entity.id",
"- assert isinstance(v, int)",
"- return v",
"+def _id(entity: Page | Url) -> int:",
"+ if entity.id is None:",
"+ raise RuntimeError(f\"{type(entity).__name__} has no id after save\")",
"+ return entity.id",
" ",
" ",
" def _make_link(page_id: int, source_url_id: int, target_url: str,"
]
}
],
"originalFile": "from __future__ import annotations\n\nfrom dataclasses import dataclass\n\nfrom src.application.use_cases.extract_seo_data import ExtractSeoDataUseCase\nfrom src.domain.entities.image import Image\nfrom src.domain.entities.link import Link\nfrom src.domain.entities.page import Page\nfrom src.domain.entities.structured_data_item import StructuredDataItem\nfrom src.domain.entities.url import Url\nfrom src.domain.ports.clock_port import ClockPort\nfrom src.domain.ports.hreflang_repository_port import HreflangRepositoryPort\nfrom src.domain.ports.html_parser_port import HtmlParserPort\nfrom src.domain.ports.http_fetcher_port import HttpFetcherPort\nfrom src.domain.ports.image_repository_port import ImageRepositoryPort\nfrom src.domain.ports.link_repository_port import LinkRepositoryPort\nfrom src.domain.ports.page_repository_port import PageRepositoryPort\nfrom src.domain.ports.structured_data_parser_port import StructuredDataParserPort\nfrom src.domain.ports.structured_data_repository_port import StructuredDataRepositoryPort\nfrom src.domain.ports.url_queue_port import UrlQueuePort\nfrom src.domain.ports.url_repository_port import UrlRepositoryPort\nfrom src.domain.services.link_classifier import LinkClassifier\nfrom src.domain.services.url_normalizer import UrlNormalizer\nfrom src.domain.value_objects.enums import RenderMode\nfrom src.domain.value_objects.normalized_url import NormalizedUrl\n\n\n@dataclass(frozen=True, slots=True)\nclass ProcessUrlDeps:\n fetcher: HttpFetcherPort\n html_parser: HtmlParserPort\n sd_parser: StructuredDataParserPort\n pages: PageRepositoryPort\n urls: UrlRepositoryPort\n links: LinkRepositoryPort\n images: ImageRepositoryPort\n structured: StructuredDataRepositoryPort\n hreflang: HreflangRepositoryPort\n queue: UrlQueuePort\n classifier: LinkClassifier\n normalizer: UrlNormalizer\n extractor: ExtractSeoDataUseCase\n clock: ClockPort\n\n\nclass ProcessUrlUseCase:\n def __init__(self, deps: ProcessUrlDeps) -> None:\n self._d = deps\n\n async def execute(self, *, crawl_id: int, url: NormalizedUrl) -> Page:\n now = self._d.clock.now()\n fetch = await self._d.fetcher.fetch(url)\n parsed = self._d.html_parser.parse(fetch.body.decode(\"utf-8\", errors=\"replace\"),\n base_url=url.url)\n url_entity = self._upsert_url(url, now=now, is_internal=True)\n extracted = self._d.extractor.execute(\n crawl_id=crawl_id, url_id=_id_of(url_entity), fetched_at=now,\n render_mode=RenderMode.HTTP, fetch=fetch, parsed=parsed, page_url=url.url,\n )\n page = self._d.pages.save(extracted.page)\n self._persist_links(page_id=_id_of(page), source_url_id=_id_of(url_entity),\n parsed_internal=parsed.internal_links,\n parsed_external=parsed.external_links,\n crawl_id=crawl_id, now=now)\n self._persist_images(page_id=_id_of(page), parsed=parsed)\n self._persist_hreflang(page_id=_id_of(page), parsed=parsed)\n self._persist_structured_data(page_id=_id_of(page), html=fetch.body, base_url=url.url)\n self._d.queue.mark_done(crawl_id, url)\n return page\n\n def _upsert_url(self, normalized: NormalizedUrl, *, now: object, is_internal: bool) -> Url:\n return self._d.urls.upsert(Url(\n id=None, normalized=normalized, is_internal=is_internal,\n first_seen_at=now, last_seen_at=now, # type: ignore[arg-type]\n ))\n\n def _persist_links(self, *, page_id: int, source_url_id: int,\n parsed_internal: tuple[str, ...], parsed_external: tuple[str, ...],\n crawl_id: int, now: object) -> None:\n links: list[Link] = []\n for raw in parsed_internal:\n normalized = self._d.normalizer.normalize(raw)\n target = self._upsert_url(normalized, now=now, is_internal=True)\n links.append(_make_link(page_id, source_url_id, normalized.url,\n target_url_id=_id_of(target), is_internal=True))\n self._d.queue.enqueue(crawl_id, normalized)\n for raw in parsed_external:\n normalized = self._d.normalizer.normalize(raw)\n links.append(_make_link(page_id, source_url_id, normalized.url,\n target_url_id=None, is_internal=False))\n self._d.links.save_many(links)\n\n def _persist_images(self, *, page_id: int, parsed: object) -> None:\n specs = parsed.image_specs\n imgs = [Image(id=None, page_id=page_id, src=s, alt=a, title=t,\n width=w, height=h, loading_attr=None, decoding_attr=None,\n http_status=None, content_length=None)\n for s, a, t, w, h in specs]\n self._d.images.save_many(imgs)\n\n def _persist_hreflang(self, *, page_id: int, parsed: object) -> None:\n self._d.hreflang.save_many(page_id, parsed.hreflang)\n\n def _persist_structured_data(self, *, page_id: int, html: bytes, base_url: str) -> None:\n items = self._d.sd_parser.parse(html.decode(\"utf-8\", errors=\"replace\"), base_url=base_url)\n sd = [StructuredDataItem(id=None, page_id=page_id, format=f, schema_type=t, payload=p)\n for f, t, p in items]\n self._d.structured.save_many(sd)\n\n\ndef _id_of(entity: object) -> int:\n v = entity.id\n assert isinstance(v, int)\n return v\n\n\ndef _make_link(page_id: int, source_url_id: int, target_url: str,\n *, target_url_id: int | None, is_internal: bool) -> Link:\n return Link(id=None, page_id=page_id, source_url_id=source_url_id, target_url=target_url,\n target_url_id=target_url_id, anchor_text=None, rel=frozenset(),\n is_internal=is_internal, is_external=not is_internal, position_hint=None)\n",
"userModified": false
}
}