{
"event": "PreToolUse",
"tool_name": "Edit",
"tool_input": {
"file_path": "\/var\/www\/scripts\/pipeline\/run_demo.py",
"old_string": " # 2. PDF laden\n log(\"2. PDF laden...\")\n start = time.time()\n from loaders.pdf_loader import load_pdf\n file_path = \"\/var\/www\/nextcloud\/data\/root\/files\/Documents\/demo.pdf\"\n text, pages = load_pdf(file_path)\n log(f\" OK: {len(text)} chars, {len(pages)} pages ({time.time()-start:.1f}s)\")\n\n # 3. Document erstellen\n log(\"3. Document in DB erstellen...\")\n start = time.time()\n cursor = db.execute(\n \"INSERT INTO documents (title, source_path, status, created_at) VALUES (%s, %s, 'processing', NOW())\",\n (\"demo.pdf\", file_path)\n )\n db.commit()\n doc_id = cursor.lastrowid\n cursor.close()\n log(f\" OK: doc_id={doc_id} ({time.time()-start:.1f}s)\")\n\n # 4. Chunking\n log(\"4. Text chunken...\")\n start = time.time()\n from chunkers.semantic_chunker import semantic_chunk\n chunks = semantic_chunk(text, max_tokens=500)\n log(f\" OK: {len(chunks)} chunks ({time.time()-start:.1f}s)\")\n\n # 5. Chunks speichern\n log(\"5. Chunks in DB speichern...\")\n start = time.time()\n chunk_ids = []\n for i, chunk in enumerate(chunks):\n cursor = db.execute(\n \"INSERT INTO chunks (document_id, chunk_index, content, char_count, created_at) VALUES (%s, %s, %s, %s, NOW())\",\n (doc_id, i, chunk, len(chunk))\n )\n db.commit()\n chunk_ids.append(cursor.lastrowid)\n cursor.close()\n log(f\" OK: {len(chunk_ids)} chunks gespeichert ({time.time()-start:.1f}s)\")\n\n # 6. YAML Prompt laden\n log(\"6. YAML Prompt aus DB laden...\")\n start = time.time()\n prompt_data = db.get_prompt_by_use_case(\"entity_extraction\")\n if prompt_data:\n log(f\" OK: Prompt geladen ({time.time()-start:.1f}s)\")\n # Zeige generierten Prompt\n sample_prompt = _build_prompt_from_yaml(prompt_data[\"content\"], \"SAMPLE TEXT\")\n log(f\" Prompt-Struktur:\\n{sample_prompt[:300]}...\")\n else:\n log(\" WARNUNG: Kein Prompt gefunden!\")\n\n # 7. Entity Extraction pro Chunk\n log(\"7. Entity Extraction (Ollama)...\")\n all_entities = []\n for i, (chunk_id, chunk_text) in enumerate(zip(chunk_ids, chunks)):\n log(f\" Chunk {i+1}\/{len(chunks)}: {len(chunk_text)} chars...\")\n start = time.time()\n entities = extract_entities_ollama(chunk_text)\n log(f\" -> {len(entities)} entities ({time.time()-start:.1f}s)\")\n for e in entities[:5]:\n log(f\" - {e['name']} ({e['type']})\")\n if len(entities) > 5:\n log(f\" ... und {len(entities)-5} weitere\")\n all_entities.extend(entities)\n\n # 8. Entities speichern\n log(\"8. Entities in DB speichern...\")",
"new_string": " # 4. PDF laden\n log(\"4. PDF laden...\")\n start = time.time()\n from loaders.pdf_loader import load_pdf\n file_path = \"\/var\/www\/nextcloud\/data\/root\/files\/Documents\/demo.pdf\"\n text, pages = load_pdf(file_path)\n log(f\" OK: {len(text)} chars, {len(pages)} pages ({time.time()-start:.1f}s)\")\n\n # 5. Document erstellen\n log(\"5. Document in DB erstellen...\")\n start = time.time()\n cursor = db.execute(\n \"INSERT INTO documents (title, source_path, status, created_at) VALUES (%s, %s, 'processing', NOW())\",\n (\"demo.pdf\", file_path)\n )\n db.commit()\n doc_id = cursor.lastrowid\n cursor.close()\n log(f\" OK: doc_id={doc_id} ({time.time()-start:.1f}s)\")\n\n # 6. Chunking\n log(\"6. Text chunken...\")\n start = time.time()\n from chunkers.semantic_chunker import semantic_chunk\n chunks = semantic_chunk(text, max_tokens=500)\n log(f\" OK: {len(chunks)} chunks ({time.time()-start:.1f}s)\")\n\n # 7. Chunks speichern\n log(\"7. Chunks in DB speichern...\")\n start = time.time()\n chunk_ids = []\n for i, chunk in enumerate(chunks):\n cursor = db.execute(\n \"INSERT INTO chunks (document_id, chunk_index, content, char_count, created_at) VALUES (%s, %s, %s, %s, NOW())\",\n (doc_id, i, chunk, len(chunk))\n )\n db.commit()\n chunk_ids.append(cursor.lastrowid)\n cursor.close()\n log(f\" OK: {len(chunk_ids)} chunks gespeichert ({time.time()-start:.1f}s)\")\n\n # 8. YAML Prompt laden\n log(\"8. YAML Prompt aus DB laden...\")\n start = time.time()\n prompt_data = db.get_prompt_by_use_case(\"entity_extraction\")\n if prompt_data:\n log(f\" OK: Prompt geladen ({time.time()-start:.1f}s)\")\n # Zeige generierten Prompt\n sample_prompt = _build_prompt_from_yaml(prompt_data[\"content\"], \"SAMPLE TEXT\")\n log(f\" Prompt-Struktur:\\n{sample_prompt[:300]}...\")\n else:\n log(\" WARNUNG: Kein Prompt gefunden!\")\n\n # 9. Entity Extraction pro Chunk\n log(\"9. Entity Extraction (Ollama)...\")\n all_entities = []\n for i, (chunk_id, chunk_text) in enumerate(zip(chunk_ids, chunks)):\n log(f\" Chunk {i+1}\/{len(chunks)}: {len(chunk_text)} chars...\")\n start = time.time()\n entities = extract_entities_ollama(chunk_text)\n log(f\" -> {len(entities)} entities ({time.time()-start:.1f}s)\")\n for e in entities[:5]:\n log(f\" - {e['name']} ({e['type']})\")\n if len(entities) > 5:\n log(f\" ... und {len(entities)-5} weitere\")\n all_entities.extend(entities)\n\n # 10. Entities speichern\n log(\"10. Entities in DB speichern...\")"
}
}
{
"tool_response": {
"filePath": "\/var\/www\/scripts\/pipeline\/run_demo.py",
"oldString": " # 2. PDF laden\n log(\"2. PDF laden...\")\n start = time.time()\n from loaders.pdf_loader import load_pdf\n file_path = \"\/var\/www\/nextcloud\/data\/root\/files\/Documents\/demo.pdf\"\n text, pages = load_pdf(file_path)\n log(f\" OK: {len(text)} chars, {len(pages)} pages ({time.time()-start:.1f}s)\")\n\n # 3. Document erstellen\n log(\"3. Document in DB erstellen...\")\n start = time.time()\n cursor = db.execute(\n \"INSERT INTO documents (title, source_path, status, created_at) VALUES (%s, %s, 'processing', NOW())\",\n (\"demo.pdf\", file_path)\n )\n db.commit()\n doc_id = cursor.lastrowid\n cursor.close()\n log(f\" OK: doc_id={doc_id} ({time.time()-start:.1f}s)\")\n\n # 4. Chunking\n log(\"4. Text chunken...\")\n start = time.time()\n from chunkers.semantic_chunker import semantic_chunk\n chunks = semantic_chunk(text, max_tokens=500)\n log(f\" OK: {len(chunks)} chunks ({time.time()-start:.1f}s)\")\n\n # 5. Chunks speichern\n log(\"5. Chunks in DB speichern...\")\n start = time.time()\n chunk_ids = []\n for i, chunk in enumerate(chunks):\n cursor = db.execute(\n \"INSERT INTO chunks (document_id, chunk_index, content, char_count, created_at) VALUES (%s, %s, %s, %s, NOW())\",\n (doc_id, i, chunk, len(chunk))\n )\n db.commit()\n chunk_ids.append(cursor.lastrowid)\n cursor.close()\n log(f\" OK: {len(chunk_ids)} chunks gespeichert ({time.time()-start:.1f}s)\")\n\n # 6. YAML Prompt laden\n log(\"6. YAML Prompt aus DB laden...\")\n start = time.time()\n prompt_data = db.get_prompt_by_use_case(\"entity_extraction\")\n if prompt_data:\n log(f\" OK: Prompt geladen ({time.time()-start:.1f}s)\")\n # Zeige generierten Prompt\n sample_prompt = _build_prompt_from_yaml(prompt_data[\"content\"], \"SAMPLE TEXT\")\n log(f\" Prompt-Struktur:\\n{sample_prompt[:300]}...\")\n else:\n log(\" WARNUNG: Kein Prompt gefunden!\")\n\n # 7. Entity Extraction pro Chunk\n log(\"7. Entity Extraction (Ollama)...\")\n all_entities = []\n for i, (chunk_id, chunk_text) in enumerate(zip(chunk_ids, chunks)):\n log(f\" Chunk {i+1}\/{len(chunks)}: {len(chunk_text)} chars...\")\n start = time.time()\n entities = extract_entities_ollama(chunk_text)\n log(f\" -> {len(entities)} entities ({time.time()-start:.1f}s)\")\n for e in entities[:5]:\n log(f\" - {e['name']} ({e['type']})\")\n if len(entities) > 5:\n log(f\" ... und {len(entities)-5} weitere\")\n all_entities.extend(entities)\n\n # 8. Entities speichern\n log(\"8. Entities in DB speichern...\")",
"newString": " # 4. PDF laden\n log(\"4. PDF laden...\")\n start = time.time()\n from loaders.pdf_loader import load_pdf\n file_path = \"\/var\/www\/nextcloud\/data\/root\/files\/Documents\/demo.pdf\"\n text, pages = load_pdf(file_path)\n log(f\" OK: {len(text)} chars, {len(pages)} pages ({time.time()-start:.1f}s)\")\n\n # 5. Document erstellen\n log(\"5. Document in DB erstellen...\")\n start = time.time()\n cursor = db.execute(\n \"INSERT INTO documents (title, source_path, status, created_at) VALUES (%s, %s, 'processing', NOW())\",\n (\"demo.pdf\", file_path)\n )\n db.commit()\n doc_id = cursor.lastrowid\n cursor.close()\n log(f\" OK: doc_id={doc_id} ({time.time()-start:.1f}s)\")\n\n # 6. Chunking\n log(\"6. Text chunken...\")\n start = time.time()\n from chunkers.semantic_chunker import semantic_chunk\n chunks = semantic_chunk(text, max_tokens=500)\n log(f\" OK: {len(chunks)} chunks ({time.time()-start:.1f}s)\")\n\n # 7. Chunks speichern\n log(\"7. Chunks in DB speichern...\")\n start = time.time()\n chunk_ids = []\n for i, chunk in enumerate(chunks):\n cursor = db.execute(\n \"INSERT INTO chunks (document_id, chunk_index, content, char_count, created_at) VALUES (%s, %s, %s, %s, NOW())\",\n (doc_id, i, chunk, len(chunk))\n )\n db.commit()\n chunk_ids.append(cursor.lastrowid)\n cursor.close()\n log(f\" OK: {len(chunk_ids)} chunks gespeichert ({time.time()-start:.1f}s)\")\n\n # 8. YAML Prompt laden\n log(\"8. YAML Prompt aus DB laden...\")\n start = time.time()\n prompt_data = db.get_prompt_by_use_case(\"entity_extraction\")\n if prompt_data:\n log(f\" OK: Prompt geladen ({time.time()-start:.1f}s)\")\n # Zeige generierten Prompt\n sample_prompt = _build_prompt_from_yaml(prompt_data[\"content\"], \"SAMPLE TEXT\")\n log(f\" Prompt-Struktur:\\n{sample_prompt[:300]}...\")\n else:\n log(\" WARNUNG: Kein Prompt gefunden!\")\n\n # 9. Entity Extraction pro Chunk\n log(\"9. Entity Extraction (Ollama)...\")\n all_entities = []\n for i, (chunk_id, chunk_text) in enumerate(zip(chunk_ids, chunks)):\n log(f\" Chunk {i+1}\/{len(chunks)}: {len(chunk_text)} chars...\")\n start = time.time()\n entities = extract_entities_ollama(chunk_text)\n log(f\" -> {len(entities)} entities ({time.time()-start:.1f}s)\")\n for e in entities[:5]:\n log(f\" - {e['name']} ({e['type']})\")\n if len(entities) > 5:\n log(f\" ... und {len(entities)-5} weitere\")\n all_entities.extend(entities)\n\n # 10. Entities speichern\n log(\"10. Entities in DB speichern...\")",
"originalFile": "#!\/usr\/bin\/env python3\n\"\"\"\nVerbose Pipeline Runner for demo.pdf\nShows every step with timing.\n\"\"\"\n\nimport sys\nimport time\n\nimport requests\n\nsys.path.insert(0, \"\/var\/www\/scripts\/pipeline\")\n\nfrom db import db\nfrom analyzers.entity_extractor import extract_entities_ollama, store_entities, _build_prompt_from_yaml\n\n\ndef log(msg):\n print(f\"[{time.strftime('%H:%M:%S')}] {msg}\", flush=True)\n\n\ndef main():\n log(\"=== START ===\")\n\n # 1. DB Connect\n log(\"1. DB verbinden...\")\n start = time.time()\n db.connect()\n log(f\" OK ({time.time()-start:.1f}s)\")\n\n # 2. DB Reset\n log(\"2. DB Reset (Tabellen leeren)...\")\n start = time.time()\n tables_order = [\n \"entity_relations\",\n \"chunk_entities\",\n \"document_entities\",\n \"chunk_semantics\",\n \"chunk_taxonomy\",\n \"document_taxonomy\",\n \"document_pages\",\n \"entities\",\n \"chunks\",\n \"documents\",\n ]\n for table in tables_order:\n try:\n db.execute(f\"DELETE FROM {table}\")\n db.commit()\n log(f\" {table}: OK\")\n except Exception as e:\n log(f\" {table}: skip ({e})\")\n log(f\" DB Reset done ({time.time()-start:.1f}s)\")\n\n # 3. Qdrant Reset\n log(\"3. Qdrant Reset...\")\n start = time.time()\n try:\n # Delete all points\n resp = requests.post(\n \"http:\/\/localhost:6333\/collections\/documents\/points\/delete\",\n json={\"filter\": {\"must\": []}},\n timeout=10\n )\n log(f\" Qdrant: {resp.status_code} ({time.time()-start:.1f}s)\")\n except Exception as e:\n log(f\" Qdrant: {e}\")\n\n # 2. PDF laden\n log(\"2. PDF laden...\")\n start = time.time()\n from loaders.pdf_loader import load_pdf\n file_path = \"\/var\/www\/nextcloud\/data\/root\/files\/Documents\/demo.pdf\"\n text, pages = load_pdf(file_path)\n log(f\" OK: {len(text)} chars, {len(pages)} pages ({time.time()-start:.1f}s)\")\n\n # 3. Document erstellen\n log(\"3. Document in DB erstellen...\")\n start = time.time()\n cursor = db.execute(\n \"INSERT INTO documents (title, source_path, status, created_at) VALUES (%s, %s, 'processing', NOW())\",\n (\"demo.pdf\", file_path)\n )\n db.commit()\n doc_id = cursor.lastrowid\n cursor.close()\n log(f\" OK: doc_id={doc_id} ({time.time()-start:.1f}s)\")\n\n # 4. Chunking\n log(\"4. Text chunken...\")\n start = time.time()\n from chunkers.semantic_chunker import semantic_chunk\n chunks = semantic_chunk(text, max_tokens=500)\n log(f\" OK: {len(chunks)} chunks ({time.time()-start:.1f}s)\")\n\n # 5. Chunks speichern\n log(\"5. Chunks in DB speichern...\")\n start = time.time()\n chunk_ids = []\n for i, chunk in enumerate(chunks):\n cursor = db.execute(\n \"INSERT INTO chunks (document_id, chunk_index, content, char_count, created_at) VALUES (%s, %s, %s, %s, NOW())\",\n (doc_id, i, chunk, len(chunk))\n )\n db.commit()\n chunk_ids.append(cursor.lastrowid)\n cursor.close()\n log(f\" OK: {len(chunk_ids)} chunks gespeichert ({time.time()-start:.1f}s)\")\n\n # 6. YAML Prompt laden\n log(\"6. YAML Prompt aus DB laden...\")\n start = time.time()\n prompt_data = db.get_prompt_by_use_case(\"entity_extraction\")\n if prompt_data:\n log(f\" OK: Prompt geladen ({time.time()-start:.1f}s)\")\n # Zeige generierten Prompt\n sample_prompt = _build_prompt_from_yaml(prompt_data[\"content\"], \"SAMPLE TEXT\")\n log(f\" Prompt-Struktur:\\n{sample_prompt[:300]}...\")\n else:\n log(\" WARNUNG: Kein Prompt gefunden!\")\n\n # 7. Entity Extraction pro Chunk\n log(\"7. Entity Extraction (Ollama)...\")\n all_entities = []\n for i, (chunk_id, chunk_text) in enumerate(zip(chunk_ids, chunks)):\n log(f\" Chunk {i+1}\/{len(chunks)}: {len(chunk_text)} chars...\")\n start = time.time()\n entities = extract_entities_ollama(chunk_text)\n log(f\" -> {len(entities)} entities ({time.time()-start:.1f}s)\")\n for e in entities[:5]:\n log(f\" - {e['name']} ({e['type']})\")\n if len(entities) > 5:\n log(f\" ... und {len(entities)-5} weitere\")\n all_entities.extend(entities)\n\n # 8. Entities speichern\n log(\"8. Entities in DB speichern...\")\n start = time.time()\n stored = store_entities(doc_id, all_entities)\n log(f\" OK: {stored} entities gespeichert ({time.time()-start:.1f}s)\")\n\n # 9. Zusammenfassung\n log(\"=== ERGEBNIS ===\")\n cursor = db.execute(\"SELECT type, COUNT(*) as cnt FROM entities GROUP BY type ORDER BY cnt DESC\")\n for row in cursor.fetchall():\n log(f\" {row['type']}: {row['cnt']}\")\n cursor.close()\n\n db.disconnect()\n log(\"=== DONE ===\")\n\n\nif __name__ == \"__main__\":\n main()\n",
"structuredPatch": [
{
"oldStart": 66,
"oldLines": 16,
"newStart": 66,
"newLines": 16,
"lines": [
" except Exception as e:",
" log(f\" Qdrant: {e}\")",
" ",
"- # 2. PDF laden",
"- log(\"2. PDF laden...\")",
"+ # 4. PDF laden",
"+ log(\"4. PDF laden...\")",
" start = time.time()",
" from loaders.pdf_loader import load_pdf",
" file_path = \"\/var\/www\/nextcloud\/data\/root\/files\/Documents\/demo.pdf\"",
" text, pages = load_pdf(file_path)",
" log(f\" OK: {len(text)} chars, {len(pages)} pages ({time.time()-start:.1f}s)\")",
" ",
"- # 3. Document erstellen",
"- log(\"3. Document in DB erstellen...\")",
"+ # 5. Document erstellen",
"+ log(\"5. Document in DB erstellen...\")",
" start = time.time()",
" cursor = db.execute(",
" \"INSERT INTO documents (title, source_path, status, created_at) VALUES (%s, %s, 'processing', NOW())\","
]
},
{
"oldStart": 86,
"oldLines": 15,
"newStart": 86,
"newLines": 15,
"lines": [
" cursor.close()",
" log(f\" OK: doc_id={doc_id} ({time.time()-start:.1f}s)\")",
" ",
"- # 4. Chunking",
"- log(\"4. Text chunken...\")",
"+ # 6. Chunking",
"+ log(\"6. Text chunken...\")",
" start = time.time()",
" from chunkers.semantic_chunker import semantic_chunk",
" chunks = semantic_chunk(text, max_tokens=500)",
" log(f\" OK: {len(chunks)} chunks ({time.time()-start:.1f}s)\")",
" ",
"- # 5. Chunks speichern",
"- log(\"5. Chunks in DB speichern...\")",
"+ # 7. Chunks speichern",
"+ log(\"7. Chunks in DB speichern...\")",
" start = time.time()",
" chunk_ids = []",
" for i, chunk in enumerate(chunks):"
]
},
{
"oldStart": 107,
"oldLines": 8,
"newStart": 107,
"newLines": 8,
"lines": [
" cursor.close()",
" log(f\" OK: {len(chunk_ids)} chunks gespeichert ({time.time()-start:.1f}s)\")",
" ",
"- # 6. YAML Prompt laden",
"- log(\"6. YAML Prompt aus DB laden...\")",
"+ # 8. YAML Prompt laden",
"+ log(\"8. YAML Prompt aus DB laden...\")",
" start = time.time()",
" prompt_data = db.get_prompt_by_use_case(\"entity_extraction\")",
" if prompt_data:"
]
},
{
"oldStart": 119,
"oldLines": 8,
"newStart": 119,
"newLines": 8,
"lines": [
" else:",
" log(\" WARNUNG: Kein Prompt gefunden!\")",
" ",
"- # 7. Entity Extraction pro Chunk",
"- log(\"7. Entity Extraction (Ollama)...\")",
"+ # 9. Entity Extraction pro Chunk",
"+ log(\"9. Entity Extraction (Ollama)...\")",
" all_entities = []",
" for i, (chunk_id, chunk_text) in enumerate(zip(chunk_ids, chunks)):",
" log(f\" Chunk {i+1}\/{len(chunks)}: {len(chunk_text)} chars...\")"
]
},
{
"oldStart": 133,
"oldLines": 8,
"newStart": 133,
"newLines": 8,
"lines": [
" log(f\" ... und {len(entities)-5} weitere\")",
" all_entities.extend(entities)",
" ",
"- # 8. Entities speichern",
"- log(\"8. Entities in DB speichern...\")",
"+ # 10. Entities speichern",
"+ log(\"10. Entities in DB speichern...\")",
" start = time.time()",
" stored = store_entities(doc_id, all_entities)",
" log(f\" OK: {stored} entities gespeichert ({time.time()-start:.1f}s)\")"
]
}
],
"userModified": false,
"replaceAll": false
}
}