From 4fc0c4aba4a605d8148c9b2c6e6271fdd25ce029 Mon Sep 17 00:00:00 2001 From: lion <120344285@qq.com> Date: Wed, 29 Jul 2026 14:48:33 +0800 Subject: [PATCH] =?UTF-8?q?=E7=88=AC=E8=99=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Crawl/Adapters/FacultyListHtmlAdapter.php | 113 ++++++++++++++---- app/Services/Crawl/CrawlAuthorParser.php | 2 +- config/crawl.php | 4 +- tests/Unit/FacultyListHtmlAdapterTest.php | 38 ++++-- 4 files changed, 122 insertions(+), 35 deletions(-) diff --git a/app/Services/Crawl/Adapters/FacultyListHtmlAdapter.php b/app/Services/Crawl/Adapters/FacultyListHtmlAdapter.php index dac7e20..6565abf 100644 --- a/app/Services/Crawl/Adapters/FacultyListHtmlAdapter.php +++ b/app/Services/Crawl/Adapters/FacultyListHtmlAdapter.php @@ -318,7 +318,12 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface return 0; } - $configured = (int) ($params['profile_enrich_max'] ?? config('crawl.faculty.profile_enrich_max', 200)); + $configured = (int) ($params['profile_enrich_max'] ?? config('crawl.faculty.profile_enrich_max', 500)); + // 与本次抓取上限对齐，避免名单很长时详情补全被截断 + $maxResults = (int) ($params['max_results'] ?? 0); + if ($maxResults > 0) { + $configured = max($configured, min(500, $maxResults)); + } return max(0, min($itemCount, min(500, $configured))); } @@ -421,27 +426,21 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface { $scoped = $this->profileContentHtml($html); - $labeledPatterns = [ - '/电子邮箱[：:]\s*<\/(?:strong|span|b)>\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u', - '/电子邮箱[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u', - '/电子信箱[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u', - '/电子邮件[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u', - '/E-?mail[：:]\s*(?:<[^>]+>\s*)*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/iu', - '/details-tag">\s*电子邮件\s*<\/span>\s*\s*(?:]*>)?\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u', - '/电子邮箱[：:]\s*<\/span>\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u', - '/邮箱[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u', - '/mailto:([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/i', - ]; - - foreach ($labeledPatterns as $pattern) { - if (preg_match($pattern, $scoped, $match)) { - $email = CrawlAuthorParser::normalizeEmail($match[1]); - if ($email && ! $this->isNoiseEmail($email)) { + // 计算机学院等：优先从联系卡片 .dt 取「邮箱」 + if (preg_match_all('#
]*>(.*?)
#su', $scoped, $cards)) { + foreach ($cards[1] as $cardHtml) { + $email = $this->extractEmailFromLabeledChunk((string) $cardHtml); + if ($email !== null) { return $email; } } } + $email = $this->extractEmailFromLabeledChunk($scoped); + if ($email !== null) { + return $email; + } + $candidates = []; if (preg_match_all( '#([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})#', @@ -449,9 +448,9 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface $emailMatches, )) { foreach ($emailMatches[1] as $raw) { - $email = CrawlAuthorParser::normalizeEmail($raw); - if ($email && ! $this->isNoiseEmail($email)) { - $candidates[] = $email; + $normalized = CrawlAuthorParser::normalizeEmail($raw); + if ($normalized && ! $this->isNoiseEmail($normalized)) { + $candidates[] = $normalized; } } } @@ -462,15 +461,51 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface $candidates = array_values(array_unique($candidates)); - foreach ($candidates as $email) { - if (str_ends_with($email, '.edu.cn') || str_ends_with($email, '.edu')) { - return $email; + foreach ($candidates as $candidate) { + if (str_ends_with($candidate, '.edu.cn') || str_ends_with($candidate, '.edu')) { + return $candidate; } } return $candidates[0]; } + protected function extractEmailFromLabeledChunk(string $html): ?string + { + $labeledPatterns = [ + '/电子邮箱[：:]\s*<\/(?:strong|span|b)>\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u', + '/电子邮箱[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u', + '/电子信箱[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u', + '/电子邮件[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u', + '/E-?mail[：:]\s*(?:<[^>]+>\s*)*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/iu', + '/details-tag">\s*电子邮件\s*<\/span>\s*\s*(?:]*>)?\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u', + '/电子邮箱[：:]\s*<\/span>\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u', + '/邮箱[：:]\s*(?:]*href=["\']mailto:([^"\']+)["\'][^>]*>\s*)?([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})?/u', + '/mailto:([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/i', + ]; + + foreach ($labeledPatterns as $pattern) { + if (! preg_match($pattern, $html, $match)) { + continue; + } + + $raw = ''; + foreach (array_slice($match, 1) as $group) { + if (is_string($group) && trim($group) !== '') { + $raw = trim($group); + break; + } + } + + $email = CrawlAuthorParser::normalizeEmail($raw); + if ($email && ! $this->isNoiseEmail($email)) { + return $email; + } + } + + return null; + } + protected function isNoiseEmail(string $email): bool { $local = strtolower((string) strstr($email, '@', true)); @@ -2067,6 +2102,14 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface protected function extractAcademicTitleFromProfileHtml(string $html): ?string { + // 计算机学院等：
长聘教轨副教授
+ if (preg_match('#
]*>\s*([^<]+?)\s*
#su', $html, $match)) { + $title = CrawlAuthorParser::cleanText($match[1]); + if (CrawlAuthorParser::looksLikeAcademicTitle((string) $title)) { + return $title; + } + } + // 电院等：

姓名

教授
if (preg_match('#
]*>\s*

[^<]+

\s*\s*([^<]+?)\s*#su', $html, $match)) { $title = CrawlAuthorParser::cleanText($match[1]); @@ -2137,6 +2180,9 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface protected function extractBioFromProfileHtml(string $html): ?string { $patterns = [ + // 计算机学院：js-dt 下「个人简介」整块 + '#
]*>.*?
\s*(?:

)?\s*个人简介\s*(?:

)?\s*
\s*
(.*?)
#su', + '#
]*>\s*
\s*(?:

)?\s*个人简介\s*(?:

)?\s*
\s*
(.*?)
#su', '#个人简介\s*
\s*
]*>(.*?)
#su', '#class="h3">\s*个人简介\s*
\s*
(.*?)
#su', '#class="name">

个人简介

\s*
(.*?)
#su', @@ -2151,7 +2197,7 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface if (preg_match($pattern, $html, $match)) { $bio = $this->htmlToPlain($match[1]); $bio = CrawlAuthorParser::cleanText($bio); - if ($bio !== null && mb_strlen($bio) >= 20) { + if ($bio !== null && mb_strlen($bio) >= 20 && ! $this->isContactLikeBio($bio)) { return Str::limit($bio, 2000, ''); } } @@ -2165,7 +2211,7 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface continue; } $plain = CrawlAuthorParser::cleanText($plain); - if ($plain !== null && mb_strlen($plain) >= 40) { + if ($plain !== null && mb_strlen($plain) >= 40 && ! $this->isContactLikeBio($plain)) { return Str::limit($plain, 2000, ''); } } @@ -2174,6 +2220,23 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface return null; } + protected function isContactLikeBio(string $bio): bool + { + // 避免把联系卡片（邮箱/地址/研究所）误当成简介 + if (preg_match('/^(?:邮箱|地址|所在研究所|个人主页|办公电话|联系电话)/u', $bio)) { + return true; + } + + $contactHits = 0; + foreach (['邮箱', '地址', '所在研究所', '个人主页'] as $label) { + if (str_contains($bio, $label)) { + $contactHits++; + } + } + + return $contactHits >= 2 && mb_strlen($bio) < 120; + } + protected function extractResearchDirectionTextFromProfileHtml(string $html): ?string { $patterns = [ diff --git a/app/Services/Crawl/CrawlAuthorParser.php b/app/Services/Crawl/CrawlAuthorParser.php index 0e43a0d..dca6fb5 100644 --- a/app/Services/Crawl/CrawlAuthorParser.php +++ b/app/Services/Crawl/CrawlAuthorParser.php @@ -119,7 +119,7 @@ class CrawlAuthorParser } return (bool) preg_match( - '/教授|副教授|讲师|助教|研究员|副研究员|助理研究员|工程师|院士|博导|导师|专家|学者|长聘|准聘|特聘|兼职|访问|青年|副高|正高|中级|初级/u', + '/教授|副教授|讲师|助教|研究员|副研究员|助理研究员|实验师|工程师|院士|博导|导师|专家|学者|长聘|准聘|特聘|教轨|兼职|访问|青年|副高|正高|中级|初级/u', $title, ); } diff --git a/config/crawl.php b/config/crawl.php index 2458ff9..21a54f0 100644 --- a/config/crawl.php +++ b/config/crawl.php @@ -31,8 +31,8 @@ return [ 'faculty' => [ /** 是否请求教师主页补全邮箱/电话/简介/研究方向 */ 'profile_email_enrich_enabled' => (bool) env('FACULTY_PROFILE_EMAIL_ENRICH', true), - /** 单次任务最多补全主页数（缺邮箱/电话/简介/研究方向时访问详情页） */ - 'profile_enrich_max' => (int) env('FACULTY_PROFILE_ENRICH_MAX', 200), + /** 单次任务最多补全主页数（缺邮箱/电话/简介/研究方向/职称时访问详情页） */ + 'profile_enrich_max' => (int) env('FACULTY_PROFILE_ENRICH_MAX', 500), 'profile_http_timeout_seconds' => (int) env('FACULTY_PROFILE_HTTP_TIMEOUT', 10), /** 并发请求教师主页数 */ 'profile_enrich_pool_size' => (int) env('FACULTY_PROFILE_ENRICH_POOL', 8), diff --git a/tests/Unit/FacultyListHtmlAdapterTest.php b/tests/Unit/FacultyListHtmlAdapterTest.php index 8a36d6e..7614ceb 100644 --- a/tests/Unit/FacultyListHtmlAdapterTest.php +++ b/tests/Unit/FacultyListHtmlAdapterTest.php @@ -313,11 +313,26 @@ HTML; $method->setAccessible(true); $csHtml = <<<'HTML' -

邮箱：byzang@sjtu.edu.cn

+
+
+
张晓凡
+
长聘教轨副教授
+
+

邮箱：xiaofan.zhang@sjtu.edu.cn

+

地址：剑川路930号A栋322B

+

所在研究所：清源研究院

+

个人主页：https://zhangxiaofan101.github.io

+
+
+

个人简介

-

上海交通大学特聘教授，主要研究领域为操作系统、分布式系统与系统安全。

+

上海交通大学计算机学院清源研究院长聘教轨副教授，博士生导师。研究领域：医学图像、文本分析，多模态决策。

+
+
+

教育背景

+

北京航空航天大学学士，美国北卡罗莱纳大学夏洛特分校博士。

HTML; @@ -325,13 +340,21 @@ HTML; $adapter, new \App\Services\Crawl\CrawlItemDto( externalId: 'faculty:cs', - title: '臧斌宇', - canonicalUrl: 'https://www.cs.sjtu.edu.cn/jiaoshiml/zangbinyu.html', - extra: ['lead_author' => ['name' => '臧斌宇']], + title: '张晓凡', + canonicalUrl: 'https://www.cs.sjtu.edu.cn/jiaoshiml/zhangxiaofan.html', + extra: ['lead_author' => ['name' => '张晓凡']], ), $csHtml, ); - $this->assertStringContainsString('操作系统', (string) $csItem->extra['bio']); + $this->assertSame('长聘教轨副教授', $csItem->extra['academic_title']); + $this->assertSame('清源研究院', $csItem->extra['college_name']); + $this->assertStringContainsString('医学图像', (string) $csItem->extra['bio']); + $this->assertStringNotContainsString('教育背景', (string) $csItem->extra['bio']); + $this->assertStringNotContainsString('北京航空航天大学学士', (string) $csItem->extra['bio']); + + $emailMethod = new \ReflectionMethod($adapter, 'extractEmailFromProfileHtml'); + $emailMethod->setAccessible(true); + $this->assertSame('xiaofan.zhang@sjtu.edu.cn', $emailMethod->invoke($adapter, $csHtml)); $frontierHtml = <<<'HTML'
@@ -419,8 +442,9 @@ HTML; $method = new \ReflectionMethod($adapter, 'resolveProfileEnrichMax'); $method->setAccessible(true); - $this->assertSame(200, $method->invoke($adapter, [], 500)); + $this->assertSame(500, $method->invoke($adapter, [], 500)); $this->assertSame(10, $method->invoke($adapter, ['profile_enrich_max' => 10], 500)); + $this->assertSame(300, $method->invoke($adapter, ['max_results' => 300], 300)); $this->assertSame(0, $method->invoke($adapter, ['skip_profile_enrich' => true], 500)); }