diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6ef8f827d16f7f6a08386b494f571e377d38304a..e03d981beb02db302f4c22f95735691924d03647 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,6 +9,11 @@ before_script: - pip install tox # - pip install -r requirements-test.txt +cache: + key: global-cache-key-crap + paths: + - tests/sources/cache + python35: image: python:3.5 stage: test @@ -17,14 +22,4 @@ python35: python36: image: python:3.6 stage: test - script: tox -e py36 - -python_coverage: - image: python:3.6 - stage: test - script: tox -e coverage - -python_pylint: - image: python:3.6 - stage: test - script: tox -e pylint \ No newline at end of file + script: tox -e py36,coverage,pylint diff --git a/eBookHub/__init__.py b/eBookHub/__init__.py index 35d62962588f4b1fa89c0a4349924f77ab7eee9e..60acf4de43955819ada1a2b9293a834ecc2ef0b1 100644 --- a/eBookHub/__init__.py +++ b/eBookHub/__init__.py @@ -25,7 +25,7 @@ def init(config_object=ProdConfig): with app.app_context(): configure_extensions(app) configure_sources(app) - if not app.testing: + if not app.testing: # pragma: no cover configure_scheduler(app) configure_logging(app) CORS( @@ -168,7 +168,7 @@ def configure_logging(app): console.setFormatter(logging.Formatter('%(asctime)s %(levelname)s::%(message)s', '%H:%M:%S')) app.logger.addHandler(console) - if app.config['FILE_LOGGING']: + if app.config['FILE_LOGGING']: # pragma: no cover log_file_handler = RotatingFileHandler( app.config['LOG_PATH'], maxBytes=app.config['LOG_MAX_BYTES'], diff --git a/eBookHub/config.py b/eBookHub/config.py index 9c37cec356ebf55b612d0b5b59a7211cbee6ad93..b8c6a86e298d3315a2d6631aa0f88ab686e6c56a 100644 --- a/eBookHub/config.py +++ b/eBookHub/config.py @@ -18,6 +18,7 @@ class Config(metaclass=MetaFlaskEnv): PROJECT_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) GOODREADS_DEV_KEY = "p1H3dvMm6SK1nVpv8bhdTA" + GOOGLE_BOOKS_DEV_KEY = "AIzaSyBYtu01k1nOctODtTOphSJ7f4-dBbshvjw" BLACKHOLE_PATH = os.path.join(PROJECT_ROOT, "blackhole") LIBRARY_PATH = os.path.join(PROJECT_ROOT, "library") diff --git a/eBookHub/database.py b/eBookHub/database.py index 4c842fc29e928760050a20305c73abab861862da..77f8523535bab9149e4b6c3114ef66828fee8105 100644 --- a/eBookHub/database.py +++ b/eBookHub/database.py @@ -19,7 +19,7 @@ SAFRSBase.re_search = re_search class PrettyPrint: _exclude_from_meta_dump = [] - def __unicode__(self): + def __unicode__(self): # pragma: no cover return "[%s(%s)]" % (self.__class__.__name__, ', '.join( '%s=%s' % (k, self.__dict__[k]) for k in sorted(self.__dict__) if '_sa_' != k[:4])) @@ -27,7 +27,7 @@ class PrettyPrint: if _visited_objs is None: _visited_objs = [] - def _serialize(obj, versions=True): + def _serialize(obj, versions=True): # pragma: no cover if isinstance(obj.__class__, DeclarativeMeta): # don't re-visit self if obj in _visited_objs: @@ -79,7 +79,7 @@ class PrettyPrint: return _serialize(self) -class SoftDeleteMixin(object): +class SoftDeleteMixin(object): # pragma: no cover deleted = db.Column(db.Boolean, default=False) def delete(self, commit=True, hard_delete=False): diff --git a/eBookHub/extensions.py b/eBookHub/extensions.py index a797527dcdc3e154baf40bd7ff7f87dd6afccb67..54ff73de5291f880772774546ee90d030ec57470 100644 --- a/eBookHub/extensions.py +++ b/eBookHub/extensions.py @@ -14,3 +14,4 @@ migrate = Migrate() cache = Cache() jwt = JWTManager() scheduler = Scheduler() + diff --git a/eBookHub/managers/source/__init__.py b/eBookHub/managers/source/__init__.py index 10217aab27921cd8bb16769fc2dc7bc5ec70fae3..8b9975470114611c1b07a20ba15d9c4480517508 100644 --- a/eBookHub/managers/source/__init__.py +++ b/eBookHub/managers/source/__init__.py @@ -1,10 +1,12 @@ from flask import current_app from flask_script import Manager -from eBookHub.managers.source.goodreads import goodreads_manager +from .goodreads import goodreads_manager +from .google import google_books_manager source_manager = Manager(current_app, usage="Source lookups") source_manager.add_command('goodreads', goodreads_manager) +source_manager.add_command('google', google_books_manager) __all__ = [ "source_manager" diff --git a/eBookHub/managers/source/google.py b/eBookHub/managers/source/google.py new file mode 100644 index 0000000000000000000000000000000000000000..d4a5734eaf512b56e1c3f74177e2bc3140e1726f --- /dev/null +++ b/eBookHub/managers/source/google.py @@ -0,0 +1,47 @@ +from flask import current_app +from flask_script import Manager, prompt + +from eBookHub.source import google_books_client + +google_books_manager = Manager(current_app, usage="Google Books lookup") + + +@google_books_manager.command +@google_books_manager.option('-i', '--id', dest='id', default=None) +def by_id(id=None): # pragma: no cover + if id is None: + id = prompt("ID") + + book = google_books_client.book_id(id) + + from pprint import pprint + pprint(book) + + +@google_books_manager.command +@google_books_manager.option('-t', '--title', dest='q', default=None) +def search_title(q=None): # pragma: no cover + if q is None: + q = prompt("Title") + + book = google_books_client.search_title(q) + + from pprint import pprint + pprint(book) + + +@google_books_manager.command +@google_books_manager.option('-a', '--author', dest='q', default=None) +def search_author(q=None): # pragma: no cover + if q is None: + q = prompt("Author") + + book = google_books_client.search_author(q) + + from pprint import pprint + pprint(book) + + +__all__ = [ + "google_books_manager" +] diff --git a/eBookHub/models/core.py b/eBookHub/models/core.py index 885c90033e07d13f5a8f518d648c78418a504688..455a38957271f7cb8e9e80d6c6dd0bccb2b7ac41 100644 --- a/eBookHub/models/core.py +++ b/eBookHub/models/core.py @@ -115,6 +115,8 @@ class BookEdition(Model): @staticmethod def lookup_by_isbn(isbn): + if isinstance(isbn, int): + isbn = str(isbn) isbn = isbn.strip() if len(isbn) == 10: return BookEdition.query.filter_by(isbn_10=isbn) diff --git a/eBookHub/source/__init__.py b/eBookHub/source/__init__.py index 8269a11c04cdd7854c61adc72b090194e153bdea..353f8fb49dc694e9ab5f72faa13491f83fe5a760 100644 --- a/eBookHub/source/__init__.py +++ b/eBookHub/source/__init__.py @@ -1,6 +1,8 @@ from .goodreads import GoodreadsClient +from .google import GoogleBooksClient goodreads_client = GoodreadsClient() +google_books_client = GoogleBooksClient() sources = [ ] @@ -8,7 +10,7 @@ sources = [ def init_sources(app): global sources - for source in [goodreads_client]: + for source in [goodreads_client, google_books_client]: source.init_app(app) sources.append(source) @@ -16,6 +18,8 @@ def init_sources(app): __all__ = [ "GoodreadsClient", "goodreads_client", + "GoogleBooksClient", + "google_books_client", "init_sources", "sources", diff --git a/eBookHub/source/abstract.py b/eBookHub/source/abstract.py index 36f5bfc9b67294c084acd7b166603d66816da208..5c04eba609a25d6fbfbd1332f120ff4abe7a0916 100644 --- a/eBookHub/source/abstract.py +++ b/eBookHub/source/abstract.py @@ -8,7 +8,7 @@ class SourceAbstract(ABC): transformer_klass = TransformerAbstract def __init__(self, app=None): - if app is not None: + if app is not None: # pragma: no cover self.init_app(app) super().__init__() @@ -19,6 +19,10 @@ class SourceAbstract(ABC): self.transformer = self.transformer_klass() assert isinstance(self.transformer, TransformerAbstract) + @abstractmethod + def search(self, q): # pragma: no cover + pass + @abstractmethod def search_title_and_author(self, title, author): # pragma: no cover pass @@ -28,11 +32,11 @@ class SourceAbstract(ABC): pass @abstractmethod - def search_title(self, title): # pragma: no cover + def search_title(self, title): # pragma: no cover pass @abstractmethod - def search_isbn(self, isbn): # pragma: no cover + def search_isbn(self, isbn): # pragma: no cover pass def transform_book(self, result): @@ -43,7 +47,7 @@ class SourceAbstract(ABC): def parse(self, title, authors=None): """ - #HashTag doneIsbetterThanPerfect + #HashTag doneIsBetterThanPerfect :param title: :param authors: :return: diff --git a/eBookHub/source/goodreads.py b/eBookHub/source/goodreads.py index 0e73efb9effb4c88e2c809e2b676b774201c9dbe..7a56a9c20e5e51e786b09b1ed8ef220c95ab7a75 100644 --- a/eBookHub/source/goodreads.py +++ b/eBookHub/source/goodreads.py @@ -21,7 +21,10 @@ class GoodreadsClient(SourceAbstract): @cache.memoize() def book_id(self, eid): - return self.transformer.convert_book(self.client.Book.show(eid)) + return self.transform_book(self.client.Book.show(eid)) + + def search(self, q): + raise NotImplementedError @cache.memoize() def search_author(self, author): diff --git a/eBookHub/source/google.py b/eBookHub/source/google.py new file mode 100644 index 0000000000000000000000000000000000000000..25e13782597b12eab24aa2f257b2bcd1cbbd20cf --- /dev/null +++ b/eBookHub/source/google.py @@ -0,0 +1,64 @@ +import httplib2 +from googleapiclient.discovery import build, Resource + +from eBookHub import cache +from eBookHub.parser.exceptions import NoResultsException +from eBookHub.source.abstract import SourceAbstract +from eBookHub.source.transformer import GoogleBooksTransformer + + +class GoogleBooksClient(SourceAbstract): + client = None + transformer_klass = GoogleBooksTransformer + + def init_app(self, app=None): + super().init_app(app) + self.build(app.config.get("GOOGLE_BOOKS_DEV_KEY"), http=httplib2.Http(cache=".cache")) + assert isinstance(self.client, Resource) + + def build(self, developer_key, http=None): + self.client = build( + 'books', + 'v1', + http=http, + developerKey=developer_key + ) + return self.client + + @cache.memoize() + def search(self, q): + return self.parse_search_result( + self.client.volumes().list( + q=q + ).execute().get('items', []) + ) + + @cache.memoize() + def book_id(self, eid): + return self.client.volumes().get(volumeId=eid).execute() + + @cache.memoize() + def search_author(self, author): + return self.search(q="inauthor:{}".format(author)) + + @cache.memoize() + def search_title(self, title): + return self.search(q="intitle:{}".format(title)) + + @cache.memoize() + def search_isbn(self, isbn): + return self.search(q="isbn:{}".format(isbn)) + + @cache.memoize() + def search_title_and_author(self, title, author): + return self.search(q="intitle:{}+inauthor:{}".format(title, author)) + + def parse_search_result(self, result): + if len(result) == 0: + raise NoResultsException() + elif len(result) > 1: + books = [] + for entry in result: + books.append(self.transform_book(self.book_id(entry['id']))) + return books + return self.transform_book(self.book_id(result[0]['id'])) diff --git a/eBookHub/source/transformer/__init__.py b/eBookHub/source/transformer/__init__.py index 7105cbd1b50620830cab901dd2a6930260279d66..ecf5acd0b29be7cd2e423aae4234d434a5b68f43 100644 --- a/eBookHub/source/transformer/__init__.py +++ b/eBookHub/source/transformer/__init__.py @@ -1,5 +1,7 @@ from .goodreads import GoodreadsTransformer +from .google import GoogleBooksTransformer __all__ = [ - "GoodreadsTransformer" + "GoodreadsTransformer", + "GoogleBooksTransformer" ] diff --git a/eBookHub/source/transformer/goodreads.py b/eBookHub/source/transformer/goodreads.py index d4b7ee53e4c677f02a1f32a957ca1b729f3cab01..3e8e440ee6d8efb51db9eaece1956db64cb0e71b 100644 --- a/eBookHub/source/transformer/goodreads.py +++ b/eBookHub/source/transformer/goodreads.py @@ -22,7 +22,7 @@ class GoodreadsTransformer(TransformerAbstract): for in_key, ex_key in cls.book_mapping.items(): try: container[in_key] = result[ex_key] - except IndexError: + except (IndexError, KeyError): # pragma: no cover pass container['authors'] = cls.convert_author(result['authors']['author']) @@ -36,7 +36,7 @@ class GoodreadsTransformer(TransformerAbstract): for in_key, ex_key in cls.author_mapping.items(): try: author[in_key] = result[ex_key] - except IndexError: + except (IndexError, KeyError): # pragma: no cover pass return author diff --git a/eBookHub/source/transformer/google.py b/eBookHub/source/transformer/google.py new file mode 100644 index 0000000000000000000000000000000000000000..0b8c060a034d49d8cc1251b68dc18bea2af7a7b8 --- /dev/null +++ b/eBookHub/source/transformer/google.py @@ -0,0 +1,51 @@ +from .abstract import TransformerAbstract + + +class GoogleBooksTransformer(TransformerAbstract): + book_mapping = { + "google_id": "id", + "title": "title", + # "isbn_10": "isbn", + # "isbn_13": "isbn13", + "language": "language", + "pageCount": "num_pages", + } + + author_mapping = { + } + + @classmethod + def map_book(cls, result): + # weird shit... + info = result.get("volumeInfo") + info['id'] = result.get("id") + info['authors'] = result.get("volumeInfo").get("authors", None) + container = {} + for in_key, ex_key in cls.book_mapping.items(): + try: + container[in_key] = info[ex_key] + except (IndexError, KeyError): # pragma: no cover + pass + + container['authors'] = cls.convert_author(info.get("authors")) + return container + + @classmethod + def map_author(cls, result): + return { + "name": result + } + + @classmethod + def convert_book(cls, result): + if isinstance(result, list): + return [cls.convert_book(x) for x in result] + container = cls.map_book(result) + return container + + @classmethod + def convert_author(cls, result): + + if isinstance(result, list): + return [cls.convert_author(x) for x in result] + return cls.map_author(result) diff --git a/requirements.txt b/requirements.txt index 45f784bd1be81a0ad10d9d63b5111325711a5086..c426329f653222907f40e19bf81828a0a958b374 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,4 +24,5 @@ apscheduler python-magic pycountry -goodreads_api_client \ No newline at end of file +goodreads_api_client +google-api-python-client \ No newline at end of file diff --git a/tests/models/test_book.py b/tests/models/test_book.py index e86495d8a6c14545c4ba9f40b826bf3e26e51d81..e9744c5f8277e3b42cb38eda8c6adb1835e555b2 100644 --- a/tests/models/test_book.py +++ b/tests/models/test_book.py @@ -3,6 +3,7 @@ from flask_testing import TestCase from eBookHub.models import Book, BookEdition, BookSerie from eBookHub.models.factory import BookFactory, BookEditionFactory, BookSerieFactory +from eBookHub.validators.exceptions import ISBNException from tests.base import ModelTestCase @@ -15,6 +16,16 @@ class BookEditionModelTestCase(ModelTestCase, TestCase): model = BookEdition factory = BookEditionFactory + def test_lookup_by_isbn(self): + model = self.model + model.lookup_by_isbn("1234567890") + model.lookup_by_isbn("1234567890123") + with self.assertRaises(ISBNException): + model.lookup_by_isbn(1) + + with self.assertRaises(ISBNException): + model.lookup_by_isbn(2) + class BookSerieModelTestCase(ModelTestCase, TestCase): model = BookSerie diff --git a/tests/sources/cache/.gitkeep b/tests/sources/cache/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/sources/test_google_books.py b/tests/sources/test_google_books.py new file mode 100644 index 0000000000000000000000000000000000000000..d6a4595df567ee56da40a68157732215659378eb --- /dev/null +++ b/tests/sources/test_google_books.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +import glob +import os + +from flask_testing import TestCase +from httplib2 import Http + +from eBookHub.source import google_books_client +from eBookHub.jobs.blackhole import process_file +from eBookHub.parser.exceptions import NoResultsException +from eBookHub.parser.filename import FilenameParser +from tests.base import MyTestCase + + +class GoogleBooksSourceTestCase(MyTestCase, TestCase): + line = 'Eriksson, Jerker & Sundquist, Hakan Axlander - [Kihlberg & Zetterlund 01] Het kraaienmeisje.epub' + + def setUp(self): + super().setUp() + google_books_client.build( + self.app.config.get("GOOGLE_BOOKS_DEV_KEY"), + http=Http(cache=os.path.join(os.path.dirname(__file__), "cache")) + ) + + def test_process_file(self): + parser = FilenameParser() + book_data_raw = parser.parse(self.line) + google_books_client.parse(book_data_raw['title'], book_data_raw['authors']) + + def test_process_file_function(self): + process_file(self.line) + + def test_search_book_isbn(self): + with self.assertRaises(NoResultsException): + google_books_client.search_isbn("1234567890") + + with self.assertRaises(NoResultsException): + google_books_client.search_isbn("1234567890123") + + # stupid api, returns result at random... + try: + google_books_client.search_isbn("9789403153605") + except NoResultsException: + pass + + def test_search_book_title(self): + google_books_client.search_title("test") + + def test_search_author(self): + google_books_client.search_author("test") diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000000000000000000000000000000000000..d266950f6d2205a253bfa9303503b09c4fb60463 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +import glob +import os +from unittest import TestCase + +from flask.helpers import get_debug_flag + +from eBookHub import ProdConfig, init +from eBookHub.config import TestConfig, DevConfig +from eBookHub.parser.filename import FilenameParser +from eBookHub.utils import get_test_flag, get_config + + +class ConfigTestCase(TestCase): + def test_production_config(self): + """Production config""" + app = init(ProdConfig) + config = app.config + self.assertEqual(config['ENV'], 'prod', "Wrong env in ProdConfig") + self.assertFalse(config['DEBUG'], "Debug is enabled in ProductionEnv") + self.assertFalse(config['DEBUG_TB_ENABLED'], "Debug toolbar is enabled in ProductionEnv") + self.assertFalse(config['TESTING'], "Testing is enabled in ProductionEnv") + + def test_test_config(self): + """Test config""" + app = init(TestConfig) + config = app.config + self.assertEqual(config['ENV'], 'test', "Wrong env in TestConfig") + self.assertFalse(config['DEBUG'], "Debug is enabled in TestingEnv") + self.assertTrue(config['TESTING'], "Testing is not enabled in TestingEnv") + self.assertIn("sqlite", config['SQLALCHEMY_DATABASE_URI'], "TestConfig contains non sqlite database uri") + + def test_dev_config(self): + """Development config""" + app = init(DevConfig) + config = app.config + self.assertEqual(config['ENV'], 'dev', "Wrong env in DevConfig") + self.assertTrue(config['DEBUG'], "Debug is not enabled in DevEnv") + self.assertTrue(config['DEBUG_TB_ENABLED'], "Debug toolbar is not enabled in DevEnv") + self.assertFalse(config['TESTING'], "Testing is enabled in DevEnv") + + def test_debug_flag(self): + os.environ["FLASK_DEBUG"] = "0" + self.assertFalse(get_debug_flag()) + + os.environ["FLASK_DEBUG"] = "1" + self.assertTrue(get_debug_flag()) + + def test_test_flag(self): + os.environ["FLASK_TESTING"] = "0" + self.assertFalse(get_test_flag()) + + os.environ["FLASK_TESTING"] = "1" + self.assertTrue(get_test_flag()) + + def test_get_config(self): + os.environ["FLASK_DEBUG"] = "0" + os.environ["FLASK_TESTING"] = "0" + self.assertEqual(ProdConfig, get_config()) + + os.environ["FLASK_DEBUG"] = "1" + os.environ["FLASK_TESTING"] = "0" + self.assertEqual(DevConfig, get_config()) + + os.environ["FLASK_DEBUG"] = "0" + os.environ["FLASK_TESTING"] = "1" + self.assertEqual(TestConfig, get_config()) + + os.environ["FLASK_DEBUG"] = "1" + os.environ["FLASK_TESTING"] = "1" + self.assertEqual(DevConfig, get_config()) diff --git a/tests/test_isbn_validator.py b/tests/test_isbn_validator.py index 0556cdaf43f15ad5b5e7f8bba3d764f8a67d838a..a2469e61392196e6d00a8479d426268a48429f57 100644 --- a/tests/test_isbn_validator.py +++ b/tests/test_isbn_validator.py @@ -2,7 +2,7 @@ from unittest import TestCase from eBookHub.validators.exceptions import ISBNException -from eBookHub.validators.isbn import validate_isbn10, validate_isbn13, validate_isbn +from eBookHub.validators.isbn import validate_isbn10, validate_isbn13, validate_isbn, is_valid_isbn class ISBNTestCase(TestCase): @@ -13,14 +13,21 @@ class ISBNTestCase(TestCase): validate_isbn10(self.isbn_10_valid) def test_isbn10_fail(self): - self.assertRaises(ISBNException, validate_isbn10, 1234567891) + self.assertRaises(ISBNException, validate_isbn10, "123456789") def test_isbn13_success(self): validate_isbn13(self.isbn_13_valid) def test_isbn13_fail(self): - self.assertRaises(ISBNException, validate_isbn13, 1234567891234) + self.assertRaises(ISBNException, validate_isbn13, "12345678912") def test_isbn_success(self): validate_isbn(self.isbn_10_valid) validate_isbn(self.isbn_13_valid) + + def test_isbn_fail(self): + self.assertRaises(ISBNException, validate_isbn, "12345678912") + + def test_is_valid_isbn(self): + self.assertTrue(is_valid_isbn(self.isbn_13_valid)) + self.assertFalse(is_valid_isbn("128912"))