diff --git a/app/Services/Crawl/Adapters/AiSjtuResearchCenterAdapter.php b/app/Services/Crawl/Adapters/AiSjtuResearchCenterAdapter.php index 39727f9..82e5a04 100644 --- a/app/Services/Crawl/Adapters/AiSjtuResearchCenterAdapter.php +++ b/app/Services/Crawl/Adapters/AiSjtuResearchCenterAdapter.php @@ -196,7 +196,7 @@ class AiSjtuResearchCenterAdapter implements CrawlerAdapterInterface return []; } - $parts = preg_split('/[、，,;；\/]+/u', $direction) ?: []; + $parts = preg_split('/(?:、|，|,|；|;|\/)+/u', $direction) ?: []; return array_values(array_unique(array_filter(array_map( fn (string $part) => trim($part), diff --git a/app/Services/Crawl/Adapters/FacultyListHtmlAdapter.php b/app/Services/Crawl/Adapters/FacultyListHtmlAdapter.php index ac4f1bd..9bab8bf 100644 --- a/app/Services/Crawl/Adapters/FacultyListHtmlAdapter.php +++ b/app/Services/Crawl/Adapters/FacultyListHtmlAdapter.php @@ -1995,8 +1995,11 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface $existingDirections = $item->extra['research_direction_names'] ?? ($lead['research_direction_names'] ?? null); if (! is_array($existingDirections) || $existingDirections === []) { - $directionText = $this->extractResearchDirectionTextFromProfileHtml($html); - $directionNames = $this->parseResearchDirectionNames($directionText ?? ''); + $directionNames = $this->extractResearchDirectionNamesFromProfileHtml($html); + if ($directionNames === []) { + $directionText = $this->extractResearchDirectionTextFromProfileHtml($html); + $directionNames = $this->parseResearchDirectionNames($directionText ?? ''); + } if ($directionNames !== []) { $lead['research_direction_names'] = $directionNames; $changed = true; @@ -2174,7 +2177,8 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface } } if ($parts !== []) { - return implode('、', $parts); + // 用换行拼接，避免顿号字符类拆分破坏多字节编码 + return implode("\n", $parts); } } } @@ -2193,7 +2197,7 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface } } if ($numberedParts !== []) { - return implode('、', $numberedParts); + return implode("\n", $numberedParts); } if ($parts !== []) { return implode("\n", $parts); @@ -2209,44 +2213,103 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface return null; } + /** + * 优先按列表项整段提取研究方向，避免再按顿号拆分时破坏 UTF-8。 + * + * @return list + */ + protected function extractResearchDirectionNamesFromProfileHtml(string $html): array + { + if (! preg_match('#研究方向\s*.*?panel-body[^>]*>\s*#su', $html, $match)) { + return []; + } + + if (! preg_match_all('#]*>(.*?)#su', (string) $match[1], $lis)) { + return []; + } + + $names = []; + foreach ($lis[1] as $li) { + $text = CrawlAuthorParser::cleanText($this->htmlToPlain($li)); + if ($text === null || $text === '') { + continue; + } + $text = preg_replace('/^研究方向[一二三四五六七八九十\d]+[：:]\s*/u', '', $text) ?? $text; + $text = trim($text); + if ($text === '' || mb_strlen($text) > 80 || preg_match('/^(已形成|主要|长期从事)/u', $text)) { + continue; + } + $names[] = $text; + } + + return array_values(array_unique($names)); + } + /** * @return list */ protected function parseResearchDirectionNames(string $direction): array { - $direction = trim($direction); + $direction = CrawlAuthorParser::ensureUtf8(trim($direction)); if ($direction === '') { return []; } - if (preg_match_all('/\d+[\.、．]\s*([^0-9\n]{2,80}?)(?=\s*\d+[\.、．]|[；;]|$)/u', $direction, $numbered)) { + // 换行分隔的条目优先整段保留 + if (str_contains($direction, "\n")) { + $lines = preg_split('/\R+/u', $direction) ?: []; + $names = []; + foreach ($lines as $line) { + foreach ($this->parseResearchDirectionNames(trim($line)) as $name) { + $names[] = $name; + } + } + + return array_values(array_unique(array_filter( + $names, + fn (string $part) => $part !== '' && mb_check_encoding($part, 'UTF-8') && mb_strlen($part) <= 80, + ))); + } + + if (preg_match_all('/(?:^|[\n；;])\s*\d+(?:\.|、|．)\s*(.+?)(?=(?:[\n；;]\s*\d+(?:\.|、|．))|$)/us', $direction, $numbered)) { $parts = array_values(array_unique(array_filter(array_map( - fn (string $part) => trim($part, " \t\n\r\0\x0B;；,，。"), + function (string $part): string { + $part = CrawlAuthorParser::ensureUtf8(trim($part)); + + return preg_replace('/^[\s;,.；，。]+|[\s;,.；，。]+$/u', '', $part) ?? $part; + }, $numbered[1], - ), fn (string $part) => $part !== '' && mb_strlen($part) <= 80))); + ), fn (string $part) => $part !== '' && mb_check_encoding($part, 'UTF-8') && mb_strlen($part) <= 80))); if (count($parts) >= 2) { return $parts; } } - $parts = preg_split('/[、，,;；\/\|\r\n]+/u', $direction) ?: []; + // 禁用多字节字符类，改用交替，避免 PCRE 按字节误切 UTF-8 + $parts = preg_split('/(?:、|，|,|；|;|\/|\||\r|\n)+/u', $direction) ?: []; return array_values(array_unique(array_filter(array_map( function (string $part): string { - $part = trim($part); - $part = preg_replace('/^\d+[\.、．]\s*/u', '', $part) ?? $part; + $part = CrawlAuthorParser::ensureUtf8(trim($part)); + $part = preg_replace('/^\d+(?:\.|、|．)\s*/u', '', $part) ?? $part; + $part = preg_replace('/^研究方向[一二三四五六七八九十\d]+[：:]\s*/u', '', $part) ?? $part; + // 勿用含多字节字符的 trim 掩码（按字节裁剪会破坏 UTF-8） + $part = preg_replace('/^[\s;,.；，。]+|[\s;,.；，。]+$/u', '', $part) ?? $part; - return trim($part, " \t\n\r\0\x0B;；,，。."); + return $part; }, $parts, - ), fn (string $part) => $part !== '' && mb_strlen($part) <= 80))); + ), fn (string $part) => $part !== '' + && mb_check_encoding($part, 'UTF-8') + && mb_strlen($part) <= 80 + && ! preg_match('/^(已形成|主要|长期从事)/u', $part)))); } protected function normalizePhone(string $phone): string { $phone = html_entity_decode($phone, ENT_QUOTES | ENT_HTML5, 'UTF-8'); $phone = trim(preg_replace('/\s+/u', ' ', $phone) ?? ''); - $phone = trim($phone, " \t\n\r\0\x0B;；,，"); + $phone = preg_replace('/^[\s;,；，]+|[\s;,；，]+$/u', '', $phone) ?? $phone; return $phone; } diff --git a/app/Services/Crawl/CrawlAuthorParser.php b/app/Services/Crawl/CrawlAuthorParser.php index dbae74d..8b72a8a 100644 --- a/app/Services/Crawl/CrawlAuthorParser.php +++ b/app/Services/Crawl/CrawlAuthorParser.php @@ -74,11 +74,61 @@ class CrawlAuthorParser public static function cleanText(?string $text): ?string { - $text = trim(preg_replace('/\s+/u', ' ', (string) $text) ?? ''); + $text = self::ensureUtf8((string) $text); + if ($text === '') { + return null; + } + + $text = trim(preg_replace('/\s+/u', ' ', $text) ?? ''); return $text === '' ? null : $text; } + public static function ensureUtf8(string $text): string + { + if ($text === '') { + return ''; + } + + if (! mb_check_encoding($text, 'UTF-8')) { + $converted = @mb_convert_encoding($text, 'UTF-8', 'UTF-8,GBK,GB2312,BIG5,ISO-8859-1'); + $text = is_string($converted) ? $converted : $text; + } + + if (function_exists('mb_scrub')) { + $text = mb_scrub($text, 'UTF-8'); + } else { + $text = iconv('UTF-8', 'UTF-8//IGNORE', $text) ?: ''; + } + + return $text; + } + + /** + * 递归清洗数组/字符串，确保可安全 json_encode。 + * + * @param mixed $value + * @return mixed + */ + public static function sanitizeUtf8Deep(mixed $value): mixed + { + if (is_string($value)) { + return self::ensureUtf8($value); + } + + if (is_array($value)) { + $out = []; + foreach ($value as $key => $item) { + $safeKey = is_string($key) ? self::ensureUtf8($key) : $key; + $out[$safeKey] = self::sanitizeUtf8Deep($item); + } + + return $out; + } + + return $value; + } + public static function universityFromAffiliation(?string $affiliation): ?string { $affiliation = self::cleanText($affiliation); diff --git a/app/Services/Crawl/CrawlImportService.php b/app/Services/Crawl/CrawlImportService.php index 194b7a3..8477d55 100644 --- a/app/Services/Crawl/CrawlImportService.php +++ b/app/Services/Crawl/CrawlImportService.php @@ -335,7 +335,7 @@ class CrawlImportService $directionNames = $payload['research_direction_names'] ?? $lead['research_direction_names'] ?? []; if (is_string($directionNames)) { - $directionNames = preg_split('/[、，,;；\/]+/u', $directionNames) ?: []; + $directionNames = preg_split('/(?:、|，|,|；|;|\/)+/u', $directionNames) ?: []; } if (! is_array($directionNames) || $directionNames === []) { return []; diff --git a/app/Services/Crawl/CrawlJobRunnerService.php b/app/Services/Crawl/CrawlJobRunnerService.php index 036c40e..1290f49 100644 --- a/app/Services/Crawl/CrawlJobRunnerService.php +++ b/app/Services/Crawl/CrawlJobRunnerService.php @@ -103,7 +103,7 @@ class CrawlJobRunnerService $extra['lead_author'] = CrawlAuthorParser::leadAuthor($dto->authors, $dto->authorsParsed); } - return [ + return CrawlAuthorParser::sanitizeUtf8Deep([ 'authors' => $dto->authors, 'summary' => $dto->summary, 'published_at' => $dto->publishedAt, @@ -113,7 +113,7 @@ class CrawlJobRunnerService 'extra' => $extra, 'authors_parsed' => $dto->authorsParsed, 'lead_author' => $extra['lead_author'] ?? null, - ]; + ]); } /** @@ -145,12 +145,12 @@ class CrawlJobRunnerService [ 'canonical_url' => $dto->canonicalUrl, 'title' => (string) $lead['name'], - 'payload' => [ + 'payload' => CrawlAuthorParser::sanitizeUtf8Deep([ 'lead_author' => $lead, 'paper_external_id' => $dto->externalId, 'paper_title' => $dto->title, 'school_name' => $lead['university_name'] ?? $dto->schoolName, - ], + ]), 'status' => $dup ? 'duplicate' : ($paperStatus === 'duplicate' ? 'preview' : 'preview'), 'target_type' => 'teacher_lead', 'source_name' => $source->name, @@ -172,7 +172,7 @@ class CrawlJobRunnerService [ 'canonical_url' => $dto->canonicalUrl, 'title' => $dto->title, - 'payload' => [ + 'payload' => CrawlAuthorParser::sanitizeUtf8Deep([ 'lead_author' => $lead, 'school_name' => $dto->schoolName, 'summary' => $dto->summary, @@ -183,7 +183,7 @@ class CrawlJobRunnerService 'bio' => $dto->extra['bio'] ?? (is_array($lead) ? ($lead['bio'] ?? null) : null), 'research_direction_names' => $dto->extra['research_direction_names'] ?? (is_array($lead) ? ($lead['research_direction_names'] ?? null) : null), - ], + ]), 'status' => 'preview', 'target_type' => 'teacher', 'source_name' => $source->name, diff --git a/tests/Unit/FacultyListHtmlAdapterTest.php b/tests/Unit/FacultyListHtmlAdapterTest.php index 735d273..8905c53 100644 --- a/tests/Unit/FacultyListHtmlAdapterTest.php +++ b/tests/Unit/FacultyListHtmlAdapterTest.php @@ -653,4 +653,60 @@ HTML; $this->assertSame('教授', $items[array_search('杨鲲', $names, true)]->extra['academic_title']); $this->assertSame('副教授', $items[array_search('邵栋', $names, true)]->extra['academic_title']); } + + public function test_parse_research_direction_names_keeps_valid_utf8_with_chinese_separators(): void + { + $adapter = new FacultyListHtmlAdapter; + $method = new \ReflectionMethod($adapter, 'parseResearchDirectionNames'); + $method->setAccessible(true); + + $names = $method->invoke( + $adapter, + '纳米材料与器件、先进功能涂层、轻质高强金属材料', + ); + + $this->assertSame( + ['纳米材料与器件', '先进功能涂层', '轻质高强金属材料'], + $names, + ); + foreach ($names as $name) { + $this->assertTrue(mb_check_encoding($name, 'UTF-8')); + } + $this->assertNotFalse(json_encode(['research_direction_names' => $names], JSON_THROW_ON_ERROR)); + } + + public function test_smse_long_research_li_items_are_kept_as_whole_and_json_safe(): void + { + $html = <<<'HTML' +
研究方向
+
+
    +
  • 研究方向一：高性能陶瓷基复合材料及其构件制备
    • +
  • 已形成完整的研究体系并在多个领域取得突破
    • +
  • 金属材料强韧化与表面改性
    • +
+
+HTML; + + $adapter = new FacultyListHtmlAdapter; + $method = new \ReflectionMethod($adapter, 'applyProfileMetadataToItem'); + $method->setAccessible(true); + $item = $method->invoke( + $adapter, + new \App\Services\Crawl\CrawlItemDto( + externalId: 'faculty:smse-utf8', + title: '赵斌元', + canonicalUrl: 'https://smse.sjtu.edu.cn/people/detail_new/1', + extra: ['lead_author' => ['name' => '赵斌元']], + ), + $html, + ); + + $names = $item->extra['research_direction_names']; + $this->assertSame( + ['高性能陶瓷基复合材料及其构件制备', '金属材料强韧化与表面改性'], + $names, + ); + $this->assertNotFalse(json_encode(['research_direction_names' => $names], JSON_THROW_ON_ERROR)); + } }