Protokoll #19168

{
    "event": "PreToolUse",
    "tool_name": "Edit",
    "tool_input": {
        "file_path": "\/var\/www\/scripts\/pipeline\/step_load.py",
        "old_string": "    def store_chunks(self, doc_id, chunks):\n        \"\"\"\n        Store chunks in database.\n\n        Args:\n            doc_id: Document ID\n            chunks: List of chunk dictionaries\n\n        Returns:\n            list: Chunks with db_id added\n        \"\"\"\n        for i, chunk in enumerate(chunks):\n            chunk_id = self.db.insert_chunk(\n                doc_id=doc_id,\n                chunk_index=i,\n                content=chunk[\"content\"],\n                heading_path=json.dumps(chunk.get(\"heading_path\", [])),\n                position_start=chunk.get(\"position_start\", 0),\n                position_end=chunk.get(\"position_end\", 0),\n                metadata=json.dumps(chunk.get(\"metadata\", {})),\n            )\n            chunk[\"db_id\"] = chunk_id\n\n        return chunks",
        "new_string": "    def store_chunks(self, doc_id, chunks, page_map=None):\n        \"\"\"\n        Store chunks in database.\n\n        Args:\n            doc_id: Document ID\n            chunks: List of chunk dictionaries\n            page_map: Optional dict mapping page_number to page_id\n\n        Returns:\n            list: Chunks with db_id added\n        \"\"\"\n        page_map = page_map or {}\n\n        for i, chunk in enumerate(chunks):\n            # Determine page_id from chunk metadata\n            page_id = None\n            metadata = chunk.get(\"metadata\", {})\n            page_number = metadata.get(\"page\")\n            if page_number and page_map:\n                page_id = page_map.get(page_number)\n\n            chunk_id = self.db.insert_chunk(\n                doc_id=doc_id,\n                chunk_index=i,\n                content=chunk[\"content\"],\n                heading_path=json.dumps(chunk.get(\"heading_path\", [])),\n                position_start=chunk.get(\"position_start\", 0),\n                position_end=chunk.get(\"position_end\", 0),\n                metadata=json.dumps(metadata),\n                page_id=page_id,\n            )\n            chunk[\"db_id\"] = chunk_id\n\n        return chunks"
    }
}

{
    "tool_response": {
        "filePath": "\/var\/www\/scripts\/pipeline\/step_load.py",
        "oldString": "    def store_chunks(self, doc_id, chunks):\n        \"\"\"\n        Store chunks in database.\n\n        Args:\n            doc_id: Document ID\n            chunks: List of chunk dictionaries\n\n        Returns:\n            list: Chunks with db_id added\n        \"\"\"\n        for i, chunk in enumerate(chunks):\n            chunk_id = self.db.insert_chunk(\n                doc_id=doc_id,\n                chunk_index=i,\n                content=chunk[\"content\"],\n                heading_path=json.dumps(chunk.get(\"heading_path\", [])),\n                position_start=chunk.get(\"position_start\", 0),\n                position_end=chunk.get(\"position_end\", 0),\n                metadata=json.dumps(chunk.get(\"metadata\", {})),\n            )\n            chunk[\"db_id\"] = chunk_id\n\n        return chunks",
        "newString": "    def store_chunks(self, doc_id, chunks, page_map=None):\n        \"\"\"\n        Store chunks in database.\n\n        Args:\n            doc_id: Document ID\n            chunks: List of chunk dictionaries\n            page_map: Optional dict mapping page_number to page_id\n\n        Returns:\n            list: Chunks with db_id added\n        \"\"\"\n        page_map = page_map or {}\n\n        for i, chunk in enumerate(chunks):\n            # Determine page_id from chunk metadata\n            page_id = None\n            metadata = chunk.get(\"metadata\", {})\n            page_number = metadata.get(\"page\")\n            if page_number and page_map:\n                page_id = page_map.get(page_number)\n\n            chunk_id = self.db.insert_chunk(\n                doc_id=doc_id,\n                chunk_index=i,\n                content=chunk[\"content\"],\n                heading_path=json.dumps(chunk.get(\"heading_path\", [])),\n                position_start=chunk.get(\"position_start\", 0),\n                position_end=chunk.get(\"position_end\", 0),\n                metadata=json.dumps(metadata),\n                page_id=page_id,\n            )\n            chunk[\"db_id\"] = chunk_id\n\n        return chunks",
        "originalFile": "\"\"\"\nLoad Step Module\nHandles database persistence operations for documents and chunks.\n\nPart of modularized pipeline architecture.\n\"\"\"\n\nimport json\n\n\nclass LoadStep:\n    \"\"\"Step: Load documents and chunks into database.\"\"\"\n\n    def __init__(self, db, progress=None):\n        \"\"\"\n        Initialize load step.\n\n        Args:\n            db: Database instance\n            progress: Optional PipelineProgress instance\n        \"\"\"\n        self.db = db\n        self.progress = progress\n\n    def create_document(self, file_info):\n        \"\"\"\n        Create document record in database.\n\n        Args:\n            file_info: Dict with path, name, type, size, hash\n\n        Returns:\n            int: Document ID\n        \"\"\"\n        doc_id = self.db.insert_document(\n            file_path=file_info[\"path\"],\n            title=file_info[\"name\"],\n            file_type=file_info[\"type\"],\n            file_size=file_info[\"size\"],\n            file_hash=file_info[\"hash\"],\n        )\n\n        self.db.log(\"INFO\", f\"Created document: {doc_id}\")\n\n        return doc_id\n\n    def store_pages(self, doc_id, extraction):\n        \"\"\"\n        Store document pages in database.\n\n        Args:\n            doc_id: Document ID\n            extraction: Extraction result with content list (for PDFs)\n\n        Returns:\n            dict: Mapping of page_number to page_id\n        \"\"\"\n        page_map = {}\n        content = extraction.get(\"content\", [])\n\n        if not isinstance(content, list):\n            # Not a multi-page document\n            return page_map\n\n        for i, page_content in enumerate(content):\n            page_number = i + 1\n            text = page_content.get(\"text\", \"\") if isinstance(page_content, dict) else str(page_content)\n\n            page_id = self.db.insert_page(\n                doc_id=doc_id,\n                page_number=page_number,\n                text_content=text,\n            )\n            page_map[page_number] = page_id\n\n        if self.progress and page_map:\n            self.progress.add_log(f\"{len(page_map)} Seiten gespeichert\")\n\n        self.db.log(\"INFO\", f\"Stored {len(page_map)} pages for document {doc_id}\")\n\n        return page_map\n\n    def store_chunks(self, doc_id, chunks):\n        \"\"\"\n        Store chunks in database.\n\n        Args:\n            doc_id: Document ID\n            chunks: List of chunk dictionaries\n\n        Returns:\n            list: Chunks with db_id added\n        \"\"\"\n        for i, chunk in enumerate(chunks):\n            chunk_id = self.db.insert_chunk(\n                doc_id=doc_id,\n                chunk_index=i,\n                content=chunk[\"content\"],\n                heading_path=json.dumps(chunk.get(\"heading_path\", [])),\n                position_start=chunk.get(\"position_start\", 0),\n                position_end=chunk.get(\"position_end\", 0),\n                metadata=json.dumps(chunk.get(\"metadata\", {})),\n            )\n            chunk[\"db_id\"] = chunk_id\n\n        return chunks\n\n    def update_document_status(self, doc_id, status):\n        \"\"\"\n        Update document processing status.\n\n        Args:\n            doc_id: Document ID\n            status: New status (processing, done, failed)\n        \"\"\"\n        self.db.update_document_status(doc_id, status)\n\n    def check_document_exists(self, file_path):\n        \"\"\"\n        Check if document already exists.\n\n        Args:\n            file_path: Path to file\n\n        Returns:\n            int|None: Document ID if exists, None otherwise\n        \"\"\"\n        return self.db.document_exists(file_path)\n\n    def check_document_done(self, file_path):\n        \"\"\"\n        Check if document is already fully processed.\n\n        Args:\n            file_path: Path to file\n\n        Returns:\n            bool: True if document is done\n        \"\"\"\n        return self.db.document_is_done(file_path)\n",
        "structuredPatch": [
            {
                "oldStart": 80,
                "oldLines": 18,
                "newStart": 80,
                "newLines": 28,
                "lines": [
                    " ",
                    "         return page_map",
                    " ",
                    "-    def store_chunks(self, doc_id, chunks):",
                    "+    def store_chunks(self, doc_id, chunks, page_map=None):",
                    "         \"\"\"",
                    "         Store chunks in database.",
                    " ",
                    "         Args:",
                    "             doc_id: Document ID",
                    "             chunks: List of chunk dictionaries",
                    "+            page_map: Optional dict mapping page_number to page_id",
                    " ",
                    "         Returns:",
                    "             list: Chunks with db_id added",
                    "         \"\"\"",
                    "+        page_map = page_map or {}",
                    "+",
                    "         for i, chunk in enumerate(chunks):",
                    "+            # Determine page_id from chunk metadata",
                    "+            page_id = None",
                    "+            metadata = chunk.get(\"metadata\", {})",
                    "+            page_number = metadata.get(\"page\")",
                    "+            if page_number and page_map:",
                    "+                page_id = page_map.get(page_number)",
                    "+",
                    "             chunk_id = self.db.insert_chunk(",
                    "                 doc_id=doc_id,",
                    "                 chunk_index=i,"
                ]
            },
            {
                "oldStart": 99,
                "oldLines": 7,
                "newStart": 109,
                "newLines": 8,
                "lines": [
                    "                 heading_path=json.dumps(chunk.get(\"heading_path\", [])),",
                    "                 position_start=chunk.get(\"position_start\", 0),",
                    "                 position_end=chunk.get(\"position_end\", 0),",
                    "-                metadata=json.dumps(chunk.get(\"metadata\", {})),",
                    "+                metadata=json.dumps(metadata),",
                    "+                page_id=page_id,",
                    "             )",
                    "             chunk[\"db_id\"] = chunk_id",
                    " "
                ]
            }
        ],
        "userModified": false,
        "replaceAll": false
    }
}