{
"event": "PreToolUse",
"tool_name": "Edit",
"tool_input": {
"file_path": "\/var\/www\/dev.campus.systemische-tools.de\/crawler\/src\/infrastructure\/parsers\/spacy_tokenizer.py",
"old_string": "import spacy\nfrom spacy.language import Language",
"new_string": "import spacy # type: ignore[import-untyped]\nfrom spacy.language import Language # type: ignore[import-untyped]",
"replace_all": false
}
}
{
"tool_response": {
"filePath": "\/var\/www\/dev.campus.systemische-tools.de\/crawler\/src\/infrastructure\/parsers\/spacy_tokenizer.py",
"oldString": "import spacy\nfrom spacy.language import Language",
"newString": "import spacy # type: ignore[import-untyped]\nfrom spacy.language import Language # type: ignore[import-untyped]",
"originalFile": "\"\"\"spaCy-based tokenizer with lemmatization (German de_core_news_sm).\"\"\"\n\nfrom __future__ import annotations\n\nfrom collections import Counter\nfrom typing import Self\n\nimport spacy\nfrom spacy.language import Language\n\nfrom src.domain.ports.tokenizer_port import TokenizerPort\nfrom src.domain.services.text_tokenizer import TokenFrequency\n\n_MODEL = \"de_core_news_sm\"\n_MIN_LEN = 3\n_MAX_LEN = 80\n\n\nclass SpacyTokenizer(TokenizerPort):\n def __init__(self, nlp: Language) -> None:\n self._nlp = nlp\n\n def tokenize(self, text: str) -> list[TokenFrequency]:\n doc = self._nlp(text or \"\", disable=[\"parser\", \"ner\", \"attribute_ruler\"])\n per_lemma: dict[str, Counter[str]] = {}\n for spacy_token in doc:\n if not spacy_token.is_alpha or spacy_token.is_stop:\n continue\n surface = spacy_token.text.lower()\n if not (_MIN_LEN <= len(surface) <= _MAX_LEN):\n continue\n lemma = (spacy_token.lemma_ or surface).lower()\n per_lemma.setdefault(lemma, Counter())[surface] += 1\n return [\n TokenFrequency(\n token=_dominant(surface_counts), count=sum(surface_counts.values()), lemma=lemma\n )\n for lemma, surface_counts in per_lemma.items()\n ]\n\n @classmethod\n def create(cls) -> Self:\n return cls(spacy.load(_MODEL, disable=[\"parser\", \"ner\", \"attribute_ruler\"]))\n\n\ndef _dominant(counter: Counter[str]) -> str:\n return counter.most_common(1)[0][0]\n",
"structuredPatch": [
{
"oldStart": 5,
"oldLines": 8,
"newStart": 5,
"newLines": 8,
"lines": [
" from collections import Counter",
" from typing import Self",
" ",
"-import spacy",
"-from spacy.language import Language",
"+import spacy # type: ignore[import-untyped]",
"+from spacy.language import Language # type: ignore[import-untyped]",
" ",
" from src.domain.ports.tokenizer_port import TokenizerPort",
" from src.domain.services.text_tokenizer import TokenFrequency"
]
}
],
"userModified": false,
"replaceAll": false
}
}