|null $itemIds * @param array{university_id?:int,department?:string,city?:string,research_direction_ids?:int[]} $teacherDefaults * @return array{ * imported:int, * imported_primary:int, * skipped:int, * failed:int, * papers_imported:int, * teacher_leads_imported:int, * teachers_imported:int, * teachers_updated:int, * news_imported:int * } */ public function import( CrawlJob $job, ?array $itemIds = null, bool $selectAll = false, array $teacherDefaults = [], array $newsDefaults = [], ): array { $newsDefaults = $this->enrichNewsDefaults($job, $newsDefaults); $query = CrawlJobItem::query() ->where('crawl_job_id', $job->id) ->whereIn('status', ['preview']); if (! $selectAll && $itemIds !== null && $itemIds !== []) { $query->whereIn('id', $itemIds); } $items = $query->orderBy('id')->get(); $imported = 0; $skipped = 0; $failed = 0; $papersImported = 0; $teacherLeadsImported = 0; $teachersImported = 0; $teachersUpdated = 0; $newsImported = 0; DB::transaction(function () use ( $items, $job, $teacherDefaults, $newsDefaults, &$imported, &$skipped, &$failed, &$papersImported, &$teacherLeadsImported, &$teachersImported, &$teachersUpdated, &$newsImported, ) { foreach ($items as $item) { try { $result = match ($item->target_type) { 'paper' => ['id' => $this->importPaper($job, $item), 'updated' => false], 'teacher_lead', 'teacher' => $this->importTeacher($job, $item, $teacherDefaults), default => ['id' => $this->importNews($job, $item, $newsDefaults), 'updated' => false], }; $id = is_array($result) ? ($result['id'] ?? null) : $result; $updated = is_array($result) ? (bool) ($result['updated'] ?? false) : false; if ($id) { $item->update(['status' => 'imported', 'target_id' => $id]); $imported++; match ($item->target_type) { 'paper' => $papersImported++, 'teacher_lead' => $teacherLeadsImported++, 'teacher' => $updated ? $teachersUpdated++ : $teachersImported++, default => $newsImported++, }; } else { if ($item->status === 'preview') { $item->update(['status' => 'skipped']); } $skipped++; } } catch (\Throwable $e) { $failed++; Log::warning('crawl_import_item_failed', [ 'crawl_job_id' => $job->id, 'item_id' => $item->id, 'target_type' => $item->target_type, 'message' => $e->getMessage(), ]); } } $job->update([ 'items_imported' => $this->countPrimaryImported($job), ]); }); return [ 'imported' => $imported, 'imported_primary' => $this->countPrimaryImported($job), 'skipped' => $skipped, 'failed' => $failed, 'papers_imported' => $papersImported, 'teacher_leads_imported' => $teacherLeadsImported, 'teachers_imported' => $teachersImported, 'teachers_updated' => $teachersUpdated, 'news_imported' => $newsImported, ]; } protected function countPrimaryImported(CrawlJob $job): int { $targetType = match ($job->target_type) { 'paper' => 'paper', 'teacher' => 'teacher', default => 'news', }; return CrawlJobItem::query() ->where('crawl_job_id', $job->id) ->where('target_type', $targetType) ->where('status', 'imported') ->count(); } protected function importPaper(CrawlJob $job, CrawlJobItem $item): ?int { $payload = $item->payload ?? []; $externalId = $item->external_id; $title = trim((string) $item->title); if ($title === '') { return null; } $existing = Paper::query() ->where('external_id', $externalId) ->where('source', 'crawl') ->first(); if ($existing) { $item->update(['status' => 'duplicate']); return null; } $paper = Paper::query()->create([ 'title' => $title, 'authors' => $payload['authors'] ?? null, 'school_name' => $payload['school_name'] ?? null, 'published_at' => $this->normalizePaperPublishedAt($payload['published_at'] ?? null), 'url' => $item->canonical_url, 'summary' => $payload['summary'] ?? null, 'source' => 'crawl', 'external_id' => $externalId, 'source_site' => $item->source_name, 'crawl_job_id' => $job->id, ]); return $paper->id; } /** * @param array{university_id?:int,department?:string,city?:string,research_direction_ids?:int[]} $defaults * @return array{id:?int,updated:bool} */ protected function importTeacher(CrawlJob $job, CrawlJobItem $item, array $defaults = []): array { $payload = $item->payload ?? []; $lead = $payload['lead_author'] ?? null; if (! is_array($lead)) { $lead = [ 'name' => $item->title, 'email' => null, 'affiliation' => null, 'university_name' => $payload['school_name'] ?? null, ]; } $name = trim((string) ($lead['name'] ?? '')); if ($name === '') { return ['id' => null, 'updated' => false]; } $email = CrawlAuthorParser::normalizeEmail($lead['email'] ?? null); $phone = trim((string) ($lead['phone'] ?? $payload['phone'] ?? '')); $academicTitle = trim((string) ($lead['academic_title'] ?? $payload['academic_title'] ?? '')); $collegeName = trim((string) ($lead['college'] ?? $lead['affiliation'] ?? $payload['college_name'] ?? '')); $defaultDepartment = trim((string) ($defaults['department'] ?? '')); if ($defaultDepartment !== '') { $collegeName = $defaultDepartment; } $bio = trim((string) ($lead['bio'] ?? $payload['bio'] ?? '')); $profileUrl = trim((string) ($payload['profile_url'] ?? $item->canonical_url ?? '')); $universityId = isset($defaults['university_id']) ? (int) $defaults['university_id'] : null; $city = isset($defaults['city']) ? trim((string) $defaults['city']) : null; if ($universityId && ! $city) { $city = University::query()->whereKey($universityId)->value('city'); } $leadUniversityName = trim((string) ($lead['university_name'] ?? '')); if (! $universityId && $leadUniversityName !== '') { $universityId = $this->resolveUniversityId($leadUniversityName); if ($universityId) { $city = $city ?: University::query()->whereKey($universityId)->value('city'); } } $directionIds = $this->resolveTeacherDirectionIds($payload, $lead, $defaults); $existing = $this->findExistingTeacher( $name, $email, $profileUrl, $universityId, $leadUniversityName, $collegeName, ); if ($existing) { $updated = $this->fillEmptyTeacherFields($existing, [ 'email' => $email, 'phone' => $phone !== '' ? $phone : null, 'title' => $academicTitle !== '' ? $academicTitle : null, 'bio' => $bio !== '' ? $bio : null, 'university_id' => $universityId, 'department' => $collegeName !== '' ? $collegeName : null, 'city' => $city, 'profile_url' => $profileUrl !== '' ? $profileUrl : null, 'research_direction_ids' => $directionIds, ]); if ($updated) { return ['id' => $existing->id, 'updated' => true]; } $item->update(['status' => 'duplicate']); return ['id' => null, 'updated' => false]; } $sourceId = $this->resolveTeacherSourceId($item->target_type); if (! $sourceId) { throw new \RuntimeException('老师库字典未配置'); } $remarkParts = [ match ($item->target_type) { 'teacher_lead' => '论文库入库', 'teacher' => '高校抓取入库', default => '爬虫入库', }, ]; if ($profileUrl !== '') { $remarkParts[] = '主页:'.$profileUrl; } $universityText = null; if (! $universityId) { if ($leadUniversityName !== '' && $collegeName !== '') { $universityText = $leadUniversityName.' · '.$collegeName; } elseif ($leadUniversityName !== '') { $universityText = $leadUniversityName; } elseif ($collegeName !== '') { $universityText = $collegeName; } } $libraryStatus = $item->target_type === 'teacher_lead' ? TeacherLibraryStatus::Pending : TeacherLibraryStatus::Active; $teacher = Teacher::query()->create([ 'name' => $name, 'university_id' => $universityId, 'university_text' => $universityText, 'department' => $collegeName !== '' ? $collegeName : null, 'bio' => $bio !== '' ? $bio : null, 'city' => $city ?: '待补充', 'title' => $academicTitle !== '' ? $academicTitle : '待补充', 'email' => $email, 'phone' => $phone !== '' ? $phone : null, 'source_dict_item_id' => $sourceId, 'status_dict_item_id' => null, 'library_status' => $libraryStatus, 'remark' => implode(';', $remarkParts), ]); if ($directionIds !== []) { $teacher->researchDirections()->sync($directionIds); } $paperExternalId = $payload['paper_external_id'] ?? null; if ($paperExternalId) { $paper = Paper::query() ->where('external_id', $paperExternalId) ->where('source', 'crawl') ->first(); if ($paper) { $teacher->papers()->syncWithoutDetaching([$paper->id]); } } return ['id' => $teacher->id, 'updated' => false]; } /** * @param array $payload * @param array $lead * @param array{university_id?:int,department?:string,city?:string,research_direction_ids?:int[]} $defaults * @return list */ protected function resolveTeacherDirectionIds(array $payload, array $lead, array $defaults): array { if (! empty($defaults['research_direction_ids']) && is_array($defaults['research_direction_ids'])) { return array_values(array_unique(array_map('intval', $defaults['research_direction_ids']))); } $directionNames = $payload['research_direction_names'] ?? $lead['research_direction_names'] ?? []; if (is_string($directionNames)) { $directionNames = preg_split('/(?:、|,|,|;|;|\/)+/u', $directionNames) ?: []; } if (! is_array($directionNames) || $directionNames === []) { return []; } return app(ResearchDirectionResolver::class)->resolveIds([], $directionNames); } /** * 仅补空字段:已有值不覆盖。 * * @param array{ * email?:?string, * phone?:?string, * title?:?string, * bio?:?string, * university_id?:?int, * department?:?string, * city?:?string, * profile_url?:?string, * research_direction_ids?:list * } $incoming */ public function fillEmptyTeacherFields(Teacher $teacher, array $incoming): bool { $updates = []; $changed = false; $email = CrawlAuthorParser::normalizeEmail($incoming['email'] ?? null); if ($this->isBlankTeacherValue($teacher->email) && $email) { $emailTaken = Teacher::query() ->where('email', $email) ->where('id', '!=', $teacher->id) ->exists(); if (! $emailTaken) { $updates['email'] = $email; } } $phone = trim((string) ($incoming['phone'] ?? '')); if ($this->isBlankTeacherValue($teacher->phone) && $phone !== '') { $updates['phone'] = $phone; } $title = trim((string) ($incoming['title'] ?? '')); if ($title !== '' && CrawlAuthorParser::looksLikeAcademicTitle($title) && ($this->isBlankTeacherValue($teacher->title) || ! CrawlAuthorParser::looksLikeAcademicTitle((string) $teacher->title))) { $updates['title'] = $title; } $bio = trim((string) ($incoming['bio'] ?? '')); if ($this->isBlankTeacherValue($teacher->bio) && $bio !== '') { $updates['bio'] = $bio; } $department = trim((string) ($incoming['department'] ?? '')); if ($this->isBlankTeacherValue($teacher->department) && $department !== '') { $updates['department'] = $department; } $city = trim((string) ($incoming['city'] ?? '')); if ($this->isBlankTeacherValue($teacher->city) && $city !== '') { $updates['city'] = $city; } $universityId = isset($incoming['university_id']) ? (int) $incoming['university_id'] : 0; if (! $teacher->university_id && $universityId > 0) { $updates['university_id'] = $universityId; } $profileUrl = trim((string) ($incoming['profile_url'] ?? '')); if ($profileUrl !== '') { $remark = (string) ($teacher->remark ?? ''); if ($remark === '' || ! str_contains($remark, $profileUrl)) { $updates['remark'] = trim($remark.($remark !== '' ? ';' : '').'主页:'.$profileUrl, ';'); } } if ($updates !== []) { $teacher->fill($updates); $teacher->save(); $changed = true; } $directionIds = array_values(array_unique(array_filter(array_map( 'intval', $incoming['research_direction_ids'] ?? [], )))); if ($directionIds !== [] && $teacher->researchDirections()->count() === 0) { $teacher->researchDirections()->sync($directionIds); $changed = true; } return $changed; } protected function isBlankTeacherValue(mixed $value): bool { $text = trim((string) ($value ?? '')); return $text === '' || $text === '待补充'; } protected function findExistingTeacher( string $name, ?string $email, ?string $profileUrl, ?int $universityId, string $leadUniversityName, ?string $collegeName, ): ?Teacher { if ($email) { $byEmail = Teacher::query()->where('email', $email)->first(); if ($byEmail) { return $byEmail; } } if ($profileUrl !== null && $profileUrl !== '') { $byProfile = Teacher::query()->where('remark', 'like', '%'.$profileUrl.'%')->first(); if ($byProfile) { return $byProfile; } } $dupQuery = Teacher::query()->where('name', $name); if ($collegeName !== null && $collegeName !== '') { $dupQuery->where('department', $collegeName); } if ($universityId) { $dupQuery->where('university_id', $universityId); } elseif ($leadUniversityName !== '') { $dupQuery->where(function ($q) use ($leadUniversityName) { $q->where('university_text', $leadUniversityName) ->orWhere('university_text', 'like', $leadUniversityName.'%'); }); } return $dupQuery->first(); } /** * @param array{source?:string, category_dict_item_id?:int} $newsDefaults */ protected function importNews(CrawlJob $job, CrawlJobItem $item, array $newsDefaults = []): ?int { $url = $item->canonical_url; if ($url && News::query()->where('source_url', $url)->exists()) { $item->update(['status' => 'duplicate']); return null; } $title = trim((string) $item->title); if ($title === '') { throw new \RuntimeException('标题为空,无法入库'); } $payload = $item->payload ?? []; $extra = $payload['extra'] ?? []; $categoryId = 0; if (! empty($newsDefaults['category_dict_item_id'])) { $categoryId = (int) $newsDefaults['category_dict_item_id']; } elseif (isset($extra['category_dict_item_id'])) { $categoryId = (int) $extra['category_dict_item_id']; } if ($categoryId <= 0) { $categoryId = (int) (app(NewsCategoryMatcher::class)->resolveCategoryId( $item->title, $payload['summary'] ?? null, $extra['keywords'] ?? CrawlKeywordParser::parse((string) ($job->keyword ?? '')), ) ?? 0); } if ($categoryId <= 0) { $categoryId = null; } $content = app(NewsHtmlImageLocalizer::class)->localize( $payload['content_html'] ?? '', $url ) ?? ''; if ($content === '') { $content = null; } $importSource = trim((string) ($extra['import_source'] ?? $newsDefaults['source'] ?? '')); if ($importSource === '') { $importSource = app(CrawlAddressSourceResolver::class)->resolveForNews( $job->request_url ?? $job->platform_url, $url, ) ?? ''; } if ($importSource === '') { $fallback = trim((string) ($item->source_name ?? '')); $importSource = in_array($fallback, app(CrawlAddressSourceResolver::class)->genericAdapterSourceNames(), true) ? '爬虫采集' : ($fallback ?: '爬虫采集'); } $news = News::query()->create([ 'title' => $title, 'category_dict_item_id' => $categoryId, 'source' => $importSource, 'source_url' => $url, 'source_site' => $importSource, 'crawl_job_id' => $job->id, 'summary' => $payload['summary'] ?? null, 'content_html' => $content, 'status' => 1, 'published_at' => $this->resolvePublishedAt($payload['published_at'] ?? null), ]); return $news->id; } /** * @param array{source?:string, category_dict_item_id?:int} $newsDefaults * @return array{source?:string, category_dict_item_id?:int} */ protected function enrichNewsDefaults(CrawlJob $job, array $newsDefaults): array { if (trim((string) ($newsDefaults['source'] ?? '')) !== '') { return $newsDefaults; } $addressSource = app(CrawlAddressSourceResolver::class)->resolveByRequestUrl( $job->request_url ?? $job->platform_url, 'industry_news', ); if ($addressSource) { $newsDefaults['source'] = $addressSource; } return $newsDefaults; } protected function resolvePublishedAt(mixed $value): Carbon { if ($value === null || $value === '') { return now(); } try { return Carbon::parse($value); } catch (\Throwable) { $normalized = HtmlCrawlSupport::normalizeDate((string) $value); return $normalized ? Carbon::parse($normalized.' 00:00:00') : now(); } } protected function normalizePaperPublishedAt(mixed $value): ?string { if ($value === null || $value === '') { return null; } try { return Carbon::parse($value)->toDateString(); } catch (\Throwable) { return HtmlCrawlSupport::normalizeDate((string) $value); } } protected function resolveUniversityId(string $name): ?int { $name = trim($name); if ($name === '') { return null; } $id = University::query() ->where('name', $name) ->where('status', 1) ->value('id'); return $id ? (int) $id : null; } /** * 论文任务第一作者 → 论文库;师资列表任务 → 高校抓取;其余 → 手动录入。 */ protected function resolveTeacherSourceId(string $targetType): ?int { $value = match ($targetType) { 'teacher_lead' => 'paper', 'teacher' => 'faculty_crawl', default => 'manual', }; $typeId = DictType::query()->where('code', 'teacher_source')->where('status', 1)->value('id'); if (! $typeId) { return null; } return DictItem::query() ->where('dict_type_id', $typeId) ->where('value', $value) ->where('status', 1) ->value('id'); } protected function defaultTeacherStatusId(): ?int { $typeId = DictType::query()->where('code', 'teacher_status')->where('status', 1)->value('id'); if (! $typeId) { return null; } return DictItem::query() ->where('dict_type_id', $typeId) ->where('value', 'active') ->where('status', 1) ->value('id'); } }