wangyifang
/
slake-school-service
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

爬虫

Browse Source
master
lion 2 days ago
parent 58b4b55054
commit 4fc0c4aba4
4 changed files with 122 additions and 35 deletions
Show Stats Download Patch File Download Diff File

113
app/Services/Crawl/Adapters/FacultyListHtmlAdapter.php
Unescape View File

@ -318,7 +318,12 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
return 0;
}
$configured = (int) ($params['profile_enrich_max'] ?? config('crawl.faculty.profile_enrich_max', 200));
$configured = (int) ($params['profile_enrich_max'] ?? config('crawl.faculty.profile_enrich_max', 500));
// 与本次抓取上限对齐，避免名单很长时详情补全被截断
$maxResults = (int) ($params['max_results'] ?? 0);
if ($maxResults > 0) {
$configured = max($configured, min(500, $maxResults));
}
return max(0, min($itemCount, min(500, $configured)));
}
@ -421,27 +426,21 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
{
$scoped = $this->profileContentHtml($html);
$labeledPatterns = [
'/电子邮箱[：:]\s*<\/(?:strong|span|b)>\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u',
'/电子邮箱[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u',
'/电子信箱[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u',
'/电子邮件[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u',
'/E-?mail[：:]\s*(?:<[^>]+>\s*)*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/iu',
'/details-tag">\s*电子邮件\s*<\/span>\s*<span class="details-con">\s*(?:<a[^>]*>)?\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u',
'/电子邮箱[：:]\s*<\/span>\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u',
'/邮箱[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u',
'/mailto:([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/i',
];
foreach ($labeledPatterns as $pattern) {
if (preg_match($pattern, $scoped, $match)) {
$email = CrawlAuthorParser::normalizeEmail($match[1]);
if ($email && ! $this->isNoiseEmail($email)) {
// 计算机学院等：优先从联系卡片 .dt 取「邮箱」
if (preg_match_all('#<div class="dt"[^>]*>(.*?)</div>#su', $scoped, $cards)) {
foreach ($cards[1] as $cardHtml) {
$email = $this->extractEmailFromLabeledChunk((string) $cardHtml);
if ($email !== null) {
return $email;
}
}
}
$email = $this->extractEmailFromLabeledChunk($scoped);
if ($email !== null) {
return $email;
}
$candidates = [];
if (preg_match_all(
'#([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})#',
@ -449,9 +448,9 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
$emailMatches,
)) {
foreach ($emailMatches[1] as $raw) {
$email = CrawlAuthorParser::normalizeEmail($raw);
if ($email && ! $this->isNoiseEmail($email)) {
$candidates[] = $email;
$normalized = CrawlAuthorParser::normalizeEmail($raw);
if ($normalized && ! $this->isNoiseEmail($normalized)) {
$candidates[] = $normalized;
}
}
}
@ -462,15 +461,51 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
$candidates = array_values(array_unique($candidates));
foreach ($candidates as $email) {
if (str_ends_with($email, '.edu.cn') || str_ends_with($email, '.edu')) {
return $email;
foreach ($candidates as $candidate) {
if (str_ends_with($candidate, '.edu.cn') || str_ends_with($candidate, '.edu')) {
return $candidate;
}
}
return $candidates[0];
}
protected function extractEmailFromLabeledChunk(string $html): ?string
{
$labeledPatterns = [
'/电子邮箱[：:]\s*<\/(?:strong|span|b)>\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u',
'/电子邮箱[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u',
'/电子信箱[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u',
'/电子邮件[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u',
'/E-?mail[：:]\s*(?:<[^>]+>\s*)*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/iu',
'/details-tag">\s*电子邮件\s*<\/span>\s*<span class="details-con">\s*(?:<a[^>]*>)?\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u',
'/电子邮箱[：:]\s*<\/span>\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u',
'/邮箱[：:]\s*(?:<a[^>]*href=["\']mailto:([^"\']+)["\'][^>]*>\s*)?([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})?/u',
'/mailto:([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/i',
];
foreach ($labeledPatterns as $pattern) {
if (! preg_match($pattern, $html, $match)) {
continue;
}
$raw = '';
foreach (array_slice($match, 1) as $group) {
if (is_string($group) && trim($group) !== '') {
$raw = trim($group);
break;
}
}
$email = CrawlAuthorParser::normalizeEmail($raw);
if ($email && ! $this->isNoiseEmail($email)) {
return $email;
}
}
return null;
}
protected function isNoiseEmail(string $email): bool
{
$local = strtolower((string) strstr($email, '@', true));
@ -2067,6 +2102,14 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
protected function extractAcademicTitleFromProfileHtml(string $html): ?string
{
// 计算机学院等：<div class="zw">长聘教轨副教授</div>
if (preg_match('#<div class="zw"[^>]*>\s*([^<]+?)\s*</div>#su', $html, $match)) {
$title = CrawlAuthorParser::cleanText($match[1]);
if (CrawlAuthorParser::looksLikeAcademicTitle((string) $title)) {
return $title;
}
}
// 电院等：<div class="tit"><p>姓名</p><span>教授</span></div>
if (preg_match('#<div class="tit"[^>]*>\s*<p>[^<]+</p>\s*<span>\s*([^<]+?)\s*</span>#su', $html, $match)) {
$title = CrawlAuthorParser::cleanText($match[1]);
@ -2137,6 +2180,9 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
protected function extractBioFromProfileHtml(string $html): ?string
{
$patterns = [
// 计算机学院：js-dt 下「个人简介」整块
'#<div class="js-dt"[^>]*>.*?<div class="name">\s*(?:<p>)?\s*个人简介\s*(?:</p>)?\s*</div>\s*<div class="txt">(.*?)</div>#su',
'#<div class="item[^"]*"[^>]*>\s*<div class="name">\s*(?:<p>)?\s*个人简介\s*(?:</p>)?\s*</div>\s*<div class="txt">(.*?)</div>#su',
'#个人简介</(?:p|div|span)>\s*</div>\s*<div class="(?:txt|p|detail)"[^>]*>(.*?)</div>#su',
'#class="h3">\s*个人简介\s*</div>\s*<div class="p">(.*?)</div>#su',
'#class="name"><p>个人简介</p></div>\s*<div class="txt">(.*?)</div>#su',
@ -2151,7 +2197,7 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
if (preg_match($pattern, $html, $match)) {
$bio = $this->htmlToPlain($match[1]);
$bio = CrawlAuthorParser::cleanText($bio);
if ($bio !== null && mb_strlen($bio) >= 20) {
if ($bio !== null && mb_strlen($bio) >= 20 && ! $this->isContactLikeBio($bio)) {
return Str::limit($bio, 2000, '');
}
}
@ -2165,7 +2211,7 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
continue;
}
$plain = CrawlAuthorParser::cleanText($plain);
if ($plain !== null && mb_strlen($plain) >= 40) {
if ($plain !== null && mb_strlen($plain) >= 40 && ! $this->isContactLikeBio($plain)) {
return Str::limit($plain, 2000, '');
}
}
@ -2174,6 +2220,23 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
return null;
}
protected function isContactLikeBio(string $bio): bool
{
// 避免把联系卡片（邮箱/地址/研究所）误当成简介
if (preg_match('/^(?:邮箱|地址|所在研究所|个人主页|办公电话|联系电话)/u', $bio)) {
return true;
}
$contactHits = 0;
foreach (['邮箱', '地址', '所在研究所', '个人主页'] as $label) {
if (str_contains($bio, $label)) {
$contactHits++;
}
}
return $contactHits >= 2 && mb_strlen($bio) < 120;
}
protected function extractResearchDirectionTextFromProfileHtml(string $html): ?string
{
$patterns = [

2
app/Services/Crawl/CrawlAuthorParser.php
Unescape View File

@ -119,7 +119,7 @@ class CrawlAuthorParser
}
return (bool) preg_match(
'/教授|副教授|讲师|助教|研究员|副研究员|助理研究员|工程师|院士|博导|导师|专家|学者|长聘|准聘|特聘|兼职|访问|青年|副高|正高|中级|初级/u',
'/教授|副教授|讲师|助教|研究员|副研究员|助理研究员|实验师|工程师|院士|博导|导师|专家|学者|长聘|准聘|特聘|教轨|兼职|访问|青年|副高|正高|中级|初级/u',
$title,
);
}

4
config/crawl.php
Unescape View File

@ -31,8 +31,8 @@ return [
'faculty' => [
/** 是否请求教师主页补全邮箱/电话/简介/研究方向 */
'profile_email_enrich_enabled' => (bool) env('FACULTY_PROFILE_EMAIL_ENRICH', true),
/** 单次任务最多补全主页数（缺邮箱/电话/简介/研究方向时访问详情页） */
'profile_enrich_max' => (int) env('FACULTY_PROFILE_ENRICH_MAX', 200),
/** 单次任务最多补全主页数（缺邮箱/电话/简介/研究方向/职称时访问详情页） */
'profile_enrich_max' => (int) env('FACULTY_PROFILE_ENRICH_MAX', 500),
'profile_http_timeout_seconds' => (int) env('FACULTY_PROFILE_HTTP_TIMEOUT', 10),
/** 并发请求教师主页数 */
'profile_enrich_pool_size' => (int) env('FACULTY_PROFILE_ENRICH_POOL', 8),

38
tests/Unit/FacultyListHtmlAdapterTest.php
Unescape View File

@ -313,11 +313,26 @@ HTML;
$method->setAccessible(true);
$csHtml = <<<'HTML'
<div class="dt"><p>邮箱：byzang@sjtu.edu.cn</p></div>
<div class="js-info">
<div class="txt">
<div class="name">张晓凡</div>
<div class="zw">长聘教轨副教授</div>
<div class="dt">
<p>邮箱：xiaofan.zhang@sjtu.edu.cn</p>
<p>地址：剑川路930号A栋322B</p>
<p>所在研究所：清源研究院</p>
<p>个人主页：<a href="https://zhangxiaofan101.github.io" target="_blank">https://zhangxiaofan101.github.io</a></p>
</div>
</div>
</div>
<div class="js-dt">
<div class="item item2">
<div class="name"><p>个人简介</p></div>
<div class="txt"><p>上海交通大学特聘教授，主要研究领域为操作系统、分布式系统与系统安全。</p></div>
<div class="txt"><p><span>上海交通大学计算机学院清源研究院长聘教轨副教授，博士生导师。</span><span>研究领域：医学图像、文本分析，多模态决策。</span></p></div>
</div>
<div class="item">
<div class="name"><p>教育背景</p></div>
<div class="txt"><p>北京航空航天大学学士，美国北卡罗莱纳大学夏洛特分校博士。</p></div>
</div>
</div>
HTML;
@ -325,13 +340,21 @@ HTML;
$adapter,
new \App\Services\Crawl\CrawlItemDto(
externalId: 'faculty:cs',
title: '臧斌宇',
canonicalUrl: 'https://www.cs.sjtu.edu.cn/jiaoshiml/zangbinyu.html',
extra: ['lead_author' => ['name' => '臧斌宇']],
title: '张晓凡',
canonicalUrl: 'https://www.cs.sjtu.edu.cn/jiaoshiml/zhangxiaofan.html',
extra: ['lead_author' => ['name' => '张晓凡']],
),
$csHtml,
);
$this->assertStringContainsString('操作系统', (string) $csItem->extra['bio']);
$this->assertSame('长聘教轨副教授', $csItem->extra['academic_title']);
$this->assertSame('清源研究院', $csItem->extra['college_name']);
$this->assertStringContainsString('医学图像', (string) $csItem->extra['bio']);
$this->assertStringNotContainsString('教育背景', (string) $csItem->extra['bio']);
$this->assertStringNotContainsString('北京航空航天大学学士', (string) $csItem->extra['bio']);
$emailMethod = new \ReflectionMethod($adapter, 'extractEmailFromProfileHtml');
$emailMethod->setAccessible(true);
$this->assertSame('xiaofan.zhang@sjtu.edu.cn', $emailMethod->invoke($adapter, $csHtml));
$frontierHtml = <<<'HTML'
<div class="person-contact">
@ -419,8 +442,9 @@ HTML;
$method = new \ReflectionMethod($adapter, 'resolveProfileEnrichMax');
$method->setAccessible(true);
$this->assertSame(200, $method->invoke($adapter, [], 500));
$this->assertSame(500, $method->invoke($adapter, [], 500));
$this->assertSame(10, $method->invoke($adapter, ['profile_enrich_max' => 10], 500));
$this->assertSame(300, $method->invoke($adapter, ['max_results' => 300], 300));
$this->assertSame(0, $method->invoke($adapter, ['skip_profile_enrich' => true], 500));
}

Write Preview
Loading…
Cancel
Save