[fix] engine: Semantic Scholar (Science) // rework & fix

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser 2022-03-04 22:00:59 +01:00 committed by Noémi Ványi
parent c56f2f1d6b
commit f231d79a5d
2 changed files with 44 additions and 21 deletions

View File

@ -1,15 +1,23 @@
# SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-License-Identifier: AGPL-3.0-or-later
""" # lint: pylint
Semantic Scholar (Science) """Semantic Scholar (Science)
""" """
from json import dumps, loads from json import dumps, loads
from datetime import datetime
from searx import logger
logger = logger.getChild('semantic scholar') about = {
"website": 'https://www.semanticscholar.org/',
"wikidata_id": 'Q22908627',
"official_api_documentation": 'https://api.semanticscholar.org/',
"use_official_api": True,
"require_api_key": False,
"results": 'JSON',
}
paging = True
search_url = 'https://www.semanticscholar.org/api/1/search' search_url = 'https://www.semanticscholar.org/api/1/search'
paper_url = 'https://www.semanticscholar.org/paper'
def request(query, params): def request(query, params):
@ -35,15 +43,37 @@ def request(query, params):
def response(resp): def response(resp):
res = loads(resp.text) res = loads(resp.text)
results = [] results = []
for result in res['results']:
if 'primaryPaperLink' not in result or 'url' not in result['primaryPaperLink']:
logger.debug('ignore result because of missing link: %s', result)
continue
results.append({ for result in res['results']:
'url': result['primaryPaperLink']['url'], item = {}
'title': result['title']['text'], metadata = []
'content': result['paperAbstractTruncated']
}) url = result.get('primaryPaperLink', {}).get('url')
if not url and result.get('links'):
url = result.get('links')[0]
if not url:
alternatePaperLinks = result.get('alternatePaperLinks')
if alternatePaperLinks:
url = alternatePaperLinks[0].get('url')
if not url:
url = paper_url + '/%s' % result['id']
item['url'] = url
item['title'] = result['title']['text']
item['content'] = result['paperAbstract']['text']
metadata = result.get('fieldsOfStudy') or []
venue = result.get('venue', {}).get('text')
if venue:
metadata.append(venue)
if metadata:
item['metadata'] = ', '.join(metadata)
pubDate = result.get('pubDate')
if pubDate:
item['publishedDate'] = datetime.strptime(pubDate, "%Y-%m-%d")
results.append(item)
return results return results

View File

@ -1133,13 +1133,6 @@ engines:
disabled : True disabled : True
shortcut : se shortcut : se
categories : science categories : science
about:
website: https://www.semanticscholar.org/
wikidata_id: Q22908627
official_api_documentation: https://api.semanticscholar.org/
use_official_api: false
require_api_key: false
results: JSON
# Spotify needs API credentials # Spotify needs API credentials
# - name : spotify # - name : spotify