From c1d456b1366e339b09bd3744b45bf80da1e7d808 Mon Sep 17 00:00:00 2001 From: Kirill Isakov Date: Sun, 27 Mar 2016 03:50:44 +0600 Subject: [PATCH] Add F-Droid search engine --- searx/engines/fdroid.py | 53 +++++++++++++++++++++++++++++++ searx/settings.yml | 5 +++ tests/unit/engines/test_fdroid.py | 49 ++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+) create mode 100644 searx/engines/fdroid.py create mode 100644 tests/unit/engines/test_fdroid.py diff --git a/searx/engines/fdroid.py b/searx/engines/fdroid.py new file mode 100644 index 00000000..0b16773e --- /dev/null +++ b/searx/engines/fdroid.py @@ -0,0 +1,53 @@ +""" + F-Droid (a repository of FOSS applications for Android) + + @website https://f-droid.org/ + @provide-api no + @using-api no + @results HTML + @stable no (HTML can change) + @parse url, title, content +""" + +from cgi import escape +from urllib import urlencode +from searx.engines.xpath import extract_text +from lxml import html + +# engine dependent config +categories = ['files'] +paging = True + +# search-url +base_url = 'https://f-droid.org/' +search_url = base_url + 'repository/browse/?{query}' + + +# do search-request +def request(query, params): + query = urlencode({'fdfilter': query, + 'fdpage': params['pageno']}) + params['url'] = search_url.format(query=query) + return params + + +# get response from search-request +def response(resp): + results = [] + + dom = html.fromstring(resp.text) + + for app in dom.xpath('//div[@id="appheader"]'): + url = app.xpath('./ancestor::a/@href')[0] + title = app.xpath('./p/span/text()')[0] + img_src = app.xpath('.//img/@src')[0] + + content = extract_text(app.xpath('./p')[0]) + content = escape(content.replace(title, '', 1).strip()) + + results.append({'url': url, + 'title': title, + 'content': content, + 'img_src': img_src}) + + return results diff --git a/searx/settings.yml b/searx/settings.yml index 7ecb477b..51e31320 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -105,6 +105,11 @@ engines: shortcut : 1x disabled : True + - name : fdroid + engine : fdroid + shortcut : fd + disabled : True + - name : flickr categories : images shortcut : fl diff --git a/tests/unit/engines/test_fdroid.py b/tests/unit/engines/test_fdroid.py new file mode 100644 index 00000000..d75f4f0b --- /dev/null +++ b/tests/unit/engines/test_fdroid.py @@ -0,0 +1,49 @@ +import mock +from collections import defaultdict +from searx.engines import fdroid +from searx.testing import SearxTestCase + + +class TestFdroidEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dic = defaultdict(dict) + dic['pageno'] = 1 + params = fdroid.request(query, dic) + self.assertTrue('url' in params) + self.assertTrue(query in params['url']) + self.assertTrue('f-droid.org' in params['url']) + + def test_response(self): + resp = mock.Mock(text='') + self.assertEqual(fdroid.response(resp), []) + + html = """ + +
+
+ +
+
+

Details...

+
+

+ Sample title +
+ Sample content +

+
+ + """ + + resp = mock.Mock(text=html) + results = fdroid.response(resp) + + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['url'], 'https://google.com/qwerty') + self.assertEqual(results[0]['title'], 'Sample title') + self.assertEqual(results[0]['content'], 'Sample content') + self.assertEqual(results[0]['img_src'], 'http://example.com/image.png')