爬虫

app/Services/Crawl/Adapters/AiSjtuResearchCenterAdapter.php

@@ -196,7 +196,7 @@ class AiSjtuResearchCenterAdapter implements CrawlerAdapterInterface
             return [];
         }
 
-        $parts = preg_split('/[、，,;；\/]+/u', $direction) ?: [];
+        $parts = preg_split('/(?:、|，|,|；|;|\/)+/u', $direction) ?: [];
 
         return array_values(array_unique(array_filter(array_map(
             fn (string $part) => trim($part),

app/Services/Crawl/Adapters/FacultyListHtmlAdapter.php

@@ -1995,8 +1995,11 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
 
         $existingDirections = $item->extra['research_direction_names'] ?? ($lead['research_direction_names'] ?? null);
         if (! is_array($existingDirections) || $existingDirections === []) {
-            $directionText = $this->extractResearchDirectionTextFromProfileHtml($html);
-            $directionNames = $this->parseResearchDirectionNames($directionText ?? '');
+            $directionNames = $this->extractResearchDirectionNamesFromProfileHtml($html);
+            if ($directionNames === []) {
+                $directionText = $this->extractResearchDirectionTextFromProfileHtml($html);
+                $directionNames = $this->parseResearchDirectionNames($directionText ?? '');
+            }
             if ($directionNames !== []) {
                 $lead['research_direction_names'] = $directionNames;
                 $changed = true;
@@ -2174,7 +2177,8 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
                         }
                     }
                     if ($parts !== []) {
-                        return implode('、', $parts);
+                        // 用换行拼接，避免顿号字符类拆分破坏多字节编码
+                        return implode("\n", $parts);
                     }
                 }
             }
@@ -2193,7 +2197,7 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
                     }
                 }
                 if ($numberedParts !== []) {
-                    return implode('、', $numberedParts);
+                    return implode("\n", $numberedParts);
                 }
                 if ($parts !== []) {
                     return implode("\n", $parts);
@@ -2209,44 +2213,103 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
         return null;
     }
 
+    /**
+     * 优先按列表项整段提取研究方向，避免再按顿号拆分时破坏 UTF-8。
+     *
+     * @return list<string>
+     */
+    protected function extractResearchDirectionNamesFromProfileHtml(string $html): array
+    {
+        if (! preg_match('#研究方向\s*</div>.*?panel-body[^>]*>\s*<ul>(.*?)</ul>#su', $html, $match)) {
+            return [];
+        }
+
+        if (! preg_match_all('#<li[^>]*>(.*?)</li>#su', (string) $match[1], $lis)) {
+            return [];
+        }
+
+        $names = [];
+        foreach ($lis[1] as $li) {
+            $text = CrawlAuthorParser::cleanText($this->htmlToPlain($li));
+            if ($text === null || $text === '') {
+                continue;
+            }
+            $text = preg_replace('/^研究方向[一二三四五六七八九十\d]+[：:]\s*/u', '', $text) ?? $text;
+            $text = trim($text);
+            if ($text === '' || mb_strlen($text) > 80 || preg_match('/^(已形成|主要|长期从事)/u', $text)) {
+                continue;
+            }
+            $names[] = $text;
+        }
+
+        return array_values(array_unique($names));
+    }
+
     /**
      * @return list<string>
      */
     protected function parseResearchDirectionNames(string $direction): array
     {
-        $direction = trim($direction);
+        $direction = CrawlAuthorParser::ensureUtf8(trim($direction));
         if ($direction === '') {
             return [];
         }
 
-        if (preg_match_all('/\d+[\.、．]\s*([^0-9\n]{2,80}?)(?=\s*\d+[\.、．]|[；;]|$)/u', $direction, $numbered)) {
+        // 换行分隔的条目优先整段保留
+        if (str_contains($direction, "\n")) {
+            $lines = preg_split('/\R+/u', $direction) ?: [];
+            $names = [];
+            foreach ($lines as $line) {
+                foreach ($this->parseResearchDirectionNames(trim($line)) as $name) {
+                    $names[] = $name;
+                }
+            }
+
+            return array_values(array_unique(array_filter(
+                $names,
+                fn (string $part) => $part !== '' && mb_check_encoding($part, 'UTF-8') && mb_strlen($part) <= 80,
+            )));
+        }
+
+        if (preg_match_all('/(?:^|[\n；;])\s*\d+(?:\.|、|．)\s*(.+?)(?=(?:[\n；;]\s*\d+(?:\.|、|．))|$)/us', $direction, $numbered)) {
             $parts = array_values(array_unique(array_filter(array_map(
-                fn (string $part) => trim($part, " \t\n\r\0\x0B;；,，。"),
+                function (string $part): string {
+                    $part = CrawlAuthorParser::ensureUtf8(trim($part));
+
+                    return preg_replace('/^[\s;,.；，。]+|[\s;,.；，。]+$/u', '', $part) ?? $part;
+                },
                 $numbered[1],
-            ), fn (string $part) => $part !== '' && mb_strlen($part) <= 80)));
+            ), fn (string $part) => $part !== '' && mb_check_encoding($part, 'UTF-8') && mb_strlen($part) <= 80)));
             if (count($parts) >= 2) {
                 return $parts;
             }
         }
 
-        $parts = preg_split('/[、，,;；\/\|\r\n]+/u', $direction) ?: [];
+        // 禁用多字节字符类，改用交替，避免 PCRE 按字节误切 UTF-8
+        $parts = preg_split('/(?:、|，|,|；|;|\/|\||\r|\n)+/u', $direction) ?: [];
 
         return array_values(array_unique(array_filter(array_map(
             function (string $part): string {
-                $part = trim($part);
-                $part = preg_replace('/^\d+[\.、．]\s*/u', '', $part) ?? $part;
+                $part = CrawlAuthorParser::ensureUtf8(trim($part));
+                $part = preg_replace('/^\d+(?:\.|、|．)\s*/u', '', $part) ?? $part;
+                $part = preg_replace('/^研究方向[一二三四五六七八九十\d]+[：:]\s*/u', '', $part) ?? $part;
+                // 勿用含多字节字符的 trim 掩码（按字节裁剪会破坏 UTF-8）
+                $part = preg_replace('/^[\s;,.；，。]+|[\s;,.；，。]+$/u', '', $part) ?? $part;
 
-                return trim($part, " \t\n\r\0\x0B;；,，。.");
+                return $part;
             },
             $parts,
-        ), fn (string $part) => $part !== '' && mb_strlen($part) <= 80)));
+        ), fn (string $part) => $part !== ''
+            && mb_check_encoding($part, 'UTF-8')
+            && mb_strlen($part) <= 80
+            && ! preg_match('/^(已形成|主要|长期从事)/u', $part))));
     }
 
     protected function normalizePhone(string $phone): string
     {
         $phone = html_entity_decode($phone, ENT_QUOTES | ENT_HTML5, 'UTF-8');
         $phone = trim(preg_replace('/\s+/u', ' ', $phone) ?? '');
-        $phone = trim($phone, " \t\n\r\0\x0B;；,，");
+        $phone = preg_replace('/^[\s;,；，]+|[\s;,；，]+$/u', '', $phone) ?? $phone;
 
         return $phone;
     }

app/Services/Crawl/CrawlAuthorParser.php

@@ -74,11 +74,61 @@ class CrawlAuthorParser
 
     public static function cleanText(?string $text): ?string
     {
-        $text = trim(preg_replace('/\s+/u', ' ', (string) $text) ?? '');
+        $text = self::ensureUtf8((string) $text);
+        if ($text === '') {
+            return null;
+        }
+
+        $text = trim(preg_replace('/\s+/u', ' ', $text) ?? '');
 
         return $text === '' ? null : $text;
     }
 
+    public static function ensureUtf8(string $text): string
+    {
+        if ($text === '') {
+            return '';
+        }
+
+        if (! mb_check_encoding($text, 'UTF-8')) {
+            $converted = @mb_convert_encoding($text, 'UTF-8', 'UTF-8,GBK,GB2312,BIG5,ISO-8859-1');
+            $text = is_string($converted) ? $converted : $text;
+        }
+
+        if (function_exists('mb_scrub')) {
+            $text = mb_scrub($text, 'UTF-8');
+        } else {
+            $text = iconv('UTF-8', 'UTF-8//IGNORE', $text) ?: '';
+        }
+
+        return $text;
+    }
+
+    /**
+     * 递归清洗数组/字符串，确保可安全 json_encode。
+     *
+     * @param  mixed  $value
+     * @return mixed
+     */
+    public static function sanitizeUtf8Deep(mixed $value): mixed
+    {
+        if (is_string($value)) {
+            return self::ensureUtf8($value);
+        }
+
+        if (is_array($value)) {
+            $out = [];
+            foreach ($value as $key => $item) {
+                $safeKey = is_string($key) ? self::ensureUtf8($key) : $key;
+                $out[$safeKey] = self::sanitizeUtf8Deep($item);
+            }
+
+            return $out;
+        }
+
+        return $value;
+    }
+
     public static function universityFromAffiliation(?string $affiliation): ?string
     {
         $affiliation = self::cleanText($affiliation);

app/Services/Crawl/CrawlImportService.php

@@ -335,7 +335,7 @@ class CrawlImportService
 
         $directionNames = $payload['research_direction_names'] ?? $lead['research_direction_names'] ?? [];
         if (is_string($directionNames)) {
-            $directionNames = preg_split('/[、，,;；\/]+/u', $directionNames) ?: [];
+            $directionNames = preg_split('/(?:、|，|,|；|;|\/)+/u', $directionNames) ?: [];
         }
         if (! is_array($directionNames) || $directionNames === []) {
             return [];

app/Services/Crawl/CrawlJobRunnerService.php

@@ -103,7 +103,7 @@ class CrawlJobRunnerService
             $extra['lead_author'] = CrawlAuthorParser::leadAuthor($dto->authors, $dto->authorsParsed);
         }
 
-        return [
+        return CrawlAuthorParser::sanitizeUtf8Deep([
             'authors' => $dto->authors,
             'summary' => $dto->summary,
             'published_at' => $dto->publishedAt,
@@ -113,7 +113,7 @@ class CrawlJobRunnerService
             'extra' => $extra,
             'authors_parsed' => $dto->authorsParsed,
             'lead_author' => $extra['lead_author'] ?? null,
-        ];
+        ]);
     }
 
     /**
@@ -145,12 +145,12 @@ class CrawlJobRunnerService
             [
                 'canonical_url' => $dto->canonicalUrl,
                 'title' => (string) $lead['name'],
-                'payload' => [
+                'payload' => CrawlAuthorParser::sanitizeUtf8Deep([
                     'lead_author' => $lead,
                     'paper_external_id' => $dto->externalId,
                     'paper_title' => $dto->title,
                     'school_name' => $lead['university_name'] ?? $dto->schoolName,
-                ],
+                ]),
                 'status' => $dup ? 'duplicate' : ($paperStatus === 'duplicate' ? 'preview' : 'preview'),
                 'target_type' => 'teacher_lead',
                 'source_name' => $source->name,
@@ -172,7 +172,7 @@ class CrawlJobRunnerService
             [
                 'canonical_url' => $dto->canonicalUrl,
                 'title' => $dto->title,
-                'payload' => [
+                'payload' => CrawlAuthorParser::sanitizeUtf8Deep([
                     'lead_author' => $lead,
                     'school_name' => $dto->schoolName,
                     'summary' => $dto->summary,
@@ -183,7 +183,7 @@ class CrawlJobRunnerService
                     'bio' => $dto->extra['bio'] ?? (is_array($lead) ? ($lead['bio'] ?? null) : null),
                     'research_direction_names' => $dto->extra['research_direction_names']
                         ?? (is_array($lead) ? ($lead['research_direction_names'] ?? null) : null),
-                ],
+                ]),
                 'status' => 'preview',
                 'target_type' => 'teacher',
                 'source_name' => $source->name,

tests/Unit/FacultyListHtmlAdapterTest.php

@@ -653,4 +653,60 @@ HTML;
         $this->assertSame('教授', $items[array_search('杨鲲', $names, true)]->extra['academic_title']);
         $this->assertSame('副教授', $items[array_search('邵栋', $names, true)]->extra['academic_title']);
     }
+
+    public function test_parse_research_direction_names_keeps_valid_utf8_with_chinese_separators(): void
+    {
+        $adapter = new FacultyListHtmlAdapter;
+        $method = new \ReflectionMethod($adapter, 'parseResearchDirectionNames');
+        $method->setAccessible(true);
+
+        $names = $method->invoke(
+            $adapter,
+            '纳米材料与器件、先进功能涂层、轻质高强金属材料',
+        );
+
+        $this->assertSame(
+            ['纳米材料与器件', '先进功能涂层', '轻质高强金属材料'],
+            $names,
+        );
+        foreach ($names as $name) {
+            $this->assertTrue(mb_check_encoding($name, 'UTF-8'));
+        }
+        $this->assertNotFalse(json_encode(['research_direction_names' => $names], JSON_THROW_ON_ERROR));
+    }
+
+    public function test_smse_long_research_li_items_are_kept_as_whole_and_json_safe(): void
+    {
+        $html = <<<'HTML'
+<div class="panel-head"><div class="title">研究方向</div></div>
+<div class="panel-body">
+    <ul>
+        <li>研究方向一：高性能陶瓷基复合材料及其构件制备</li>
+        <li>已形成完整的研究体系并在多个领域取得突破</li>
+        <li>金属材料强韧化与表面改性</li>
+    </ul>
+</div>
+HTML;
+
+        $adapter = new FacultyListHtmlAdapter;
+        $method = new \ReflectionMethod($adapter, 'applyProfileMetadataToItem');
+        $method->setAccessible(true);
+        $item = $method->invoke(
+            $adapter,
+            new \App\Services\Crawl\CrawlItemDto(
+                externalId: 'faculty:smse-utf8',
+                title: '赵斌元',
+                canonicalUrl: 'https://smse.sjtu.edu.cn/people/detail_new/1',
+                extra: ['lead_author' => ['name' => '赵斌元']],
+            ),
+            $html,
+        );
+
+        $names = $item->extra['research_direction_names'];
+        $this->assertSame(
+            ['高性能陶瓷基复合材料及其构件制备', '金属材料强韧化与表面改性'],
+            $names,
+        );
+        $this->assertNotFalse(json_encode(['research_direction_names' => $names], JSON_THROW_ON_ERROR));
+    }
 }