Backup #419

Dateiinhalt

<?php

namespace Controller;

use Framework\Controller;
use Infrastructure\Persistence\PipelineRepository;

class ContentPipelineController extends Controller
{
    private PipelineRepository $repository;

    public function __construct()
    {
        $this->repository = new PipelineRepository();
    }

    /**
     * GET /content-pipeline
     */
    public function index(): void
    {
        $pipelines = $this->repository->findAll();
        $stats = $this->repository->getStatistics();

        $this->view('content-pipeline.index', [
            'title' => 'Content Pipeline',
            'pipelines' => $pipelines,
            'stats' => $stats,
        ]);
    }

    /**
     * GET /content-pipeline/import
     */
    public function import(): void
    {
        $pipeline = $this->repository->findDefault();

        if ($pipeline === null) {
            $pipelines = $this->repository->findAll(1);
            $pipeline = $pipelines[0] ?? null;
        }

        $latestRun = $pipeline !== null
            ? $this->repository->findLatestRun((int) $pipeline['id'])
            : null;

        $this->view('content-pipeline.import', [
            'title' => 'Import Pipeline',
            'pipeline' => $pipeline,
            'latestRun' => $latestRun,
        ]);
    }

    /**
     * GET /content-pipeline/new
     */
    public function pipelineNew(): void
    {
        $this->view('content-pipeline.form', [
            'title' => 'Neue Pipeline',
            'pipeline' => null,
            'stepTypes' => $this->getStepTypes(),
        ]);
    }

    /**
     * GET /content-pipeline/{id}
     */
    public function show(string $id): void
    {
        $pipeline = $this->repository->findById((int) $id);

        if ($pipeline === null) {
            $this->notFound('Pipeline nicht gefunden');
        }

        $runs = $this->repository->findRuns((int) $id, 10);

        $this->view('content-pipeline.show', [
            'title' => 'Pipeline: ' . $pipeline['name'],
            'pipeline' => $pipeline,
            'runs' => $runs,
            'stepTypes' => $this->getStepTypes(),
        ]);
    }

    /**
     * GET /content-pipeline/{id}/edit
     */
    public function edit(string $id): void
    {
        $pipeline = $this->repository->findById((int) $id);

        if ($pipeline === null) {
            $this->notFound('Pipeline nicht gefunden');
        }

        $this->view('content-pipeline.form', [
            'title' => 'Pipeline bearbeiten: ' . $pipeline['name'],
            'pipeline' => $pipeline,
            'stepTypes' => $this->getStepTypes(),
        ]);
    }

    /**
     * POST /content-pipeline
     */
    public function store(): void
    {
        $this->requireCsrf();

        $name = trim($_POST['name'] ?? '');
        $description = trim($_POST['description'] ?? '');
        $sourcePath = trim($_POST['source_path'] ?? '/var/www/nextcloud/data/root/files/Documents');
        $extensions = $this->parseExtensions($_POST['extensions'] ?? '');
        $isDefault = isset($_POST['is_default']) ? 1 : 0;

        if ($name === '') {
            $_SESSION['error'] = 'Name ist erforderlich.';
            header('Location: /content-pipeline/new');
            exit;
        }

        $pipelineId = $this->repository->create([
            'name' => $name,
            'description' => $description,
            'source_path' => $sourcePath,
            'extensions' => $extensions,
            'is_default' => $isDefault,
        ]);

        // Standard-Steps hinzufuegen
        $this->createDefaultSteps($pipelineId);

        $_SESSION['success'] = 'Pipeline erfolgreich erstellt.';
        header('Location: /content-pipeline/' . $pipelineId);
        exit;
    }

    /**
     * POST /content-pipeline/{id}
     */
    public function update(string $id): void
    {
        $this->requireCsrf();

        $pipeline = $this->repository->findById((int) $id);

        if ($pipeline === null) {
            $this->notFound('Pipeline nicht gefunden');
        }

        $name = trim($_POST['name'] ?? '');
        $description = trim($_POST['description'] ?? '');
        $sourcePath = trim($_POST['source_path'] ?? '');
        $extensions = $this->parseExtensions($_POST['extensions'] ?? '');
        $isDefault = isset($_POST['is_default']) ? 1 : 0;

        if ($name === '') {
            $_SESSION['error'] = 'Name ist erforderlich.';
            header('Location: /content-pipeline/' . $id . '/edit');
            exit;
        }

        $this->repository->update((int) $id, [
            'name' => $name,
            'description' => $description,
            'source_path' => $sourcePath,
            'extensions' => $extensions,
            'is_default' => $isDefault,
        ]);

        $_SESSION['success'] = 'Pipeline aktualisiert.';
        header('Location: /content-pipeline/' . $id);
        exit;
    }

    /**
     * POST /content-pipeline/{id}/run
     */
    public function run(string $id): void
    {
        $this->requireCsrf();

        $pipeline = $this->repository->findById((int) $id);

        if ($pipeline === null) {
            $this->notFound('Pipeline nicht gefunden');
        }

        // Neuen Run erstellen
        $runId = $this->repository->createRun((int) $id);

        // Pipeline im Hintergrund starten
        $pipelineScript = '/opt/scripts/pipeline/pipeline.py';
        $venvPython = '/opt/scripts/pipeline/venv/bin/python';
        $logFile = '/tmp/pipeline_run_' . $runId . '.log';

        $cmd = sprintf(
            'nohup %s %s all --pipeline-id=%d --run-id=%d > %s 2>&1 &',
            escapeshellarg($venvPython),
            escapeshellarg($pipelineScript),
            (int) $id,
            $runId,
            escapeshellarg($logFile)
        );

        exec($cmd);

        $_SESSION['success'] = 'Pipeline gestartet (Run #' . $runId . ')';
        header('Location: /content-pipeline/' . $id);
        exit;
    }

    /**
     * GET /content-pipeline/{id}/status
     * AJAX endpoint for run status
     */
    public function status(string $id): void
    {
        $pipeline = $this->repository->findById((int) $id);

        if ($pipeline === null) {
            $this->json(['error' => 'Pipeline nicht gefunden'], 404);
            return;
        }

        $latestRun = $this->repository->findLatestRun((int) $id);

        $this->json([
            'pipeline_id' => (int) $id,
            'run' => $latestRun,
        ]);
    }

    /**
     * POST /content-pipeline/{id}/steps/{stepId}/toggle
     */
    public function toggleStep(string $id, string $stepId): void
    {
        $this->requireCsrf();

        $pipeline = $this->repository->findById((int) $id);

        if ($pipeline === null) {
            $this->notFound('Pipeline nicht gefunden');
        }

        // Find step and toggle
        foreach ($pipeline['steps'] as $step) {
            if ((int) $step['id'] === (int) $stepId) {
                $this->repository->updateStep((int) $stepId, [
                    'enabled' => $step['enabled'] ? 0 : 1,
                ]);
                break;
            }
        }

        header('Location: /content-pipeline/' . $id);
        exit;
    }

    /**
     * POST /content-pipeline/{id}/delete
     */
    public function delete(string $id): void
    {
        $this->requireCsrf();

        $pipeline = $this->repository->findById((int) $id);

        if ($pipeline === null) {
            $this->notFound('Pipeline nicht gefunden');
        }

        $this->repository->delete((int) $id);

        $_SESSION['success'] = 'Pipeline geloescht.';
        header('Location: /content-pipeline');
        exit;
    }

    /**
     * @return array<string, array<string, mixed>>
     */
    private function getStepTypes(): array
    {
        return [
            // Phase 1: Preprocessing
            'detect' => [
                'label' => '1. Detect',
                'icon' => 'search',
                'description' => 'Dateien scannen und Format pruefen',
                'phase' => 'Preprocessing',
            ],
            'validate' => [
                'label' => '2. Validate',
                'icon' => 'check',
                'description' => 'Datei-Validierung und Korruptionspruefung',
                'phase' => 'Preprocessing',
            ],
            'extract' => [
                'label' => '3. Extract',
                'icon' => 'file-text',
                'description' => 'Text extrahieren und bereinigen',
                'phase' => 'Preprocessing',
            ],
            'structure' => [
                'label' => '4. Structure',
                'icon' => 'list',
                'description' => 'Ueberschriften und Struktur erkennen',
                'phase' => 'Preprocessing',
            ],
            'segment' => [
                'label' => '5. Segment',
                'icon' => 'layout',
                'description' => 'Logische Dokumentgliederung',
                'phase' => 'Preprocessing',
            ],
            'chunk' => [
                'label' => '6. Chunk',
                'icon' => 'scissors',
                'description' => 'Textbausteine erstellen (max 800 Token)',
                'phase' => 'Preprocessing',
            ],
            // Phase 2: Embedding & Storage
            'embed' => [
                'label' => '7. Embed',
                'icon' => 'cpu',
                'description' => 'Vektorisierung mit mxbai-embed-large',
                'phase' => 'Embedding',
            ],
            'collection_setup' => [
                'label' => '8. Collection',
                'icon' => 'database',
                'description' => 'Qdrant-Collection einrichten',
                'phase' => 'Embedding',
            ],
            'vector_store' => [
                'label' => '9. Store',
                'icon' => 'upload',
                'description' => 'Vektoren in Qdrant speichern',
                'phase' => 'Embedding',
            ],
            'metadata_store' => [
                'label' => '10. Metadata',
                'icon' => 'tag',
                'description' => 'Metadaten in MariaDB speichern',
                'phase' => 'Embedding',
            ],
            'index_optimize' => [
                'label' => '11. Index',
                'icon' => 'zap',
                'description' => 'HNSW-Index optimieren',
                'phase' => 'Embedding',
            ],
            // Phase 3: Analysis
            'entity_extract' => [
                'label' => '12. Entities',
                'icon' => 'users',
                'description' => 'Entitaeten erkennen (Personen, Konzepte)',
                'phase' => 'Analysis',
            ],
            'relation_extract' => [
                'label' => '13. Relations',
                'icon' => 'git-branch',
                'description' => 'Beziehungen zwischen Entitaeten',
                'phase' => 'Analysis',
            ],
            'taxonomy_build' => [
                'label' => '14. Taxonomy',
                'icon' => 'folder-tree',
                'description' => 'Hierarchische Kategorisierung',
                'phase' => 'Analysis',
            ],
            'semantic_analyze' => [
                'label' => '15. Semantic',
                'icon' => 'brain',
                'description' => 'Bedeutungs-Analyse und Konzepte',
                'phase' => 'Analysis',
            ],
            'summarize' => [
                'label' => '16. Summary',
                'icon' => 'file-minus',
                'description' => 'Dokument-Zusammenfassung',
                'phase' => 'Analysis',
            ],
            'finalize' => [
                'label' => '17. Finalize',
                'icon' => 'check-circle',
                'description' => 'Job abschliessen und Status setzen',
                'phase' => 'Analysis',
            ],
            'question_generate' => [
                'label' => '18. Questions',
                'icon' => 'help-circle',
                'description' => 'Beispielfragen fuer RAG-Chat generieren',
                'phase' => 'Analysis',
            ],
            // Legacy
            'analyze' => [
                'label' => 'Analyze',
                'icon' => 'brain',
                'description' => 'Kombinierte Analyse (Legacy)',
                'phase' => 'Analysis',
            ],
        ];
    }

    /**
     * @param string $input
     * @return array<string>
     */
    private function parseExtensions(string $input): array
    {
        $extensions = [];
        $parts = preg_split('/[\s,;]+/', $input);

        if ($parts === false) {
            return ['.pdf', '.docx', '.pptx', '.md', '.txt'];
        }

        foreach ($parts as $ext) {
            $ext = trim($ext);
            if ($ext === '') {
                continue;
            }
            if ($ext[0] !== '.') {
                $ext = '.' . $ext;
            }
            $extensions[] = strtolower($ext);
        }

        return $extensions !== [] ? $extensions : ['.pdf', '.docx', '.pptx', '.md', '.txt'];
    }

    private function createDefaultSteps(int $pipelineId): void
    {
        $defaultSteps = [
            ['step_type' => 'detect', 'config' => ['hash_algorithm' => 'sha256'], 'sort_order' => 1, 'enabled' => 1],
            ['step_type' => 'extract', 'config' => ['ocr_enabled' => true, 'ocr_language' => 'deu'], 'sort_order' => 2, 'enabled' => 1],
            ['step_type' => 'chunk', 'config' => ['min_size' => 100, 'max_size' => 2000, 'overlap' => 0.1], 'sort_order' => 3, 'enabled' => 1],
            ['step_type' => 'embed', 'config' => ['model' => 'mxbai-embed-large', 'collection' => 'documents', 'dimensions' => 1024], 'sort_order' => 4, 'enabled' => 1],
            ['step_type' => 'analyze', 'config' => ['extract_entities' => true, 'extract_relations' => true, 'classify_taxonomy' => true], 'sort_order' => 5, 'enabled' => 0],
        ];

        foreach ($defaultSteps as $step) {
            $this->repository->addStep($pipelineId, $step);
        }
    }
}

Vollständig herunterladen

Aktionen

Andere Versionen dieser Datei

← Zurück zur Übersicht