Commit e9e510db authored by Patrick van der Leer's avatar Patrick van der Leer
Browse files

fucking stupid 'Object is already attached to session' error in unittest

parent 2207540d
......@@ -95,7 +95,6 @@ def configure_logging(app):
loglvl = logging.DEBUG
app.logger.setLevel(loglvl)
config = app.config
console = logging.StreamHandler()
console.setLevel(loglvl)
......@@ -104,9 +103,9 @@ def configure_logging(app):
if app.config['FILE_LOGGING']:
log_file_handler = RotatingFileHandler(
config['LOG_PATH'],
maxBytes=config['LOG_MAX_BYTES'],
backupCount=config['LOG_BACKUP_COUNT']
app.config['LOG_PATH'],
maxBytes=app.config['LOG_MAX_BYTES'],
backupCount=app.config['LOG_BACKUP_COUNT']
)
log_file_handler.setLevel(loglvl)
log_file_handler.setFormatter(
......
......@@ -68,16 +68,6 @@ class ProdConfig(Config):
FILE_LOGGING = False
DEBUG_TB_ENABLED = False # Disable Debug toolbar
DEBUG_TB_INTERCEPT_REDIRECTS = False
SENTRY = True
SENTRY_CONFIG = {
'dsn': 'https://680a9fb3fb13484c9669f2e762ae0707:d693d9eb08ec4bffb0610c7cc06fe5d2@sentry.io/304987',
'release': __version__,
"environment": "STAGE",
"tags": {
"track": "track",
"tier": "tier",
}
}
class DevConfig(Config):
......
from .core import AuthorFactory, BookFactory, BookEditionFactory, BookSerieFactory, GenreFactory, PublisherFactory
__all__ = (
AuthorFactory,
BookFactory,
BookEditionFactory,
BookSerieFactory,
GenreFactory,
PublisherFactory
)
......@@ -3,7 +3,12 @@ import random
import factory
from eBookHub import db
from eBookHub.models.core import Author, Book
from eBookHub.models import Author, Book, BookEdition, BookSerie, Genre, Publisher
from .faker.providers import GenreProvider, LanguageProvider
factory.Faker.add_provider(GenreProvider)
factory.Faker.add_provider(LanguageProvider)
class AuthorFactory(factory.alchemy.SQLAlchemyModelFactory):
......@@ -15,6 +20,48 @@ class AuthorFactory(factory.alchemy.SQLAlchemyModelFactory):
name = factory.Faker('name')
class PublisherFactory(factory.alchemy.SQLAlchemyModelFactory):
class Meta:
model = Publisher
sqlalchemy_session = db.session
id = factory.Sequence(lambda n: n)
name = factory.Faker('name')
class GenreFactory(factory.alchemy.SQLAlchemyModelFactory):
class Meta:
model = Genre
sqlalchemy_session = db.session
id = factory.Sequence(lambda n: n)
name = factory.Faker('genre')
fiction = factory.Faker("pybool")
class BookEditionFactory(factory.alchemy.SQLAlchemyModelFactory):
class Meta:
model = BookEdition
sqlalchemy_session = db.session
id = factory.Sequence(lambda n: n)
title = factory.Faker('name')
@factory.post_generation
def authors(self, create, extracted, **kwargs):
return AuthorFactory.create_batch(random.randint(1, 4))
class BookSerieFactory(factory.alchemy.SQLAlchemyModelFactory):
class Meta:
model = BookSerie
sqlalchemy_session = db.session
id = factory.Sequence(lambda n: n)
name = factory.Faker('name')
language = factory.Faker('iso639_2')
class BookFactory(factory.alchemy.SQLAlchemyModelFactory):
class Meta:
model = Book
......@@ -26,3 +73,13 @@ class BookFactory(factory.alchemy.SQLAlchemyModelFactory):
@factory.post_generation
def authors(self, create, extracted, **kwargs):
return AuthorFactory.create_batch(random.randint(1, 4))
__all__ = (
AuthorFactory,
BookFactory,
BookEditionFactory,
BookSerieFactory,
GenreFactory,
PublisherFactory
)
from .genre import GenreProvider
from .language import LanguageProvider
__all__ = (
GenreProvider,
LanguageProvider,
)
from faker.providers import BaseProvider
class GenreProvider(BaseProvider):
_genre_data = [
"Action and adventure",
"Alternate history",
"Anthology",
"Chick lit",
"Children's literature",
"Comic book",
"Coming-of-age",
"Crime",
"Drama",
"Fairytale",
"Fantasy",
"Graphic novel",
"Historical fiction",
"Horror",
"Mystery",
"Paranormal romance",
"Picture book",
"Poetry",
"Political thriller",
"Romance",
"Satire",
"Science fiction",
"Short story",
"Suspense",
"Thriller",
"Young adult",
"Art",
"Autobiography",
"Biography",
"Book review",
"Cookbook",
"Diary",
"Dictionary",
"Encyclopedia",
"Guide",
"Health",
"History",
"Journal",
"Math",
"Memoir",
"Prayer",
"Religion, spirituality, and new age",
"Textbook",
"Review",
"Science",
"Self help",
"Travel",
"True crime",
]
def genre(self):
return self.random_element(self._genre_data)
from faker.providers import BaseProvider
from eBookHub.utils import languages
class LanguageProvider(BaseProvider):
def iso639_2(self):
return self.random_element(languages)
# coding=utf-8
import os
from BookWurm.exceptions import NoDataError
from BookWurm.metadata.parser.exceptions import InvalidImportData, MetadataMissing
from BookWurm.metadata.parser.parser import Parser
from BookWurm.models import Book
def handle_import_ebooks(path, calibre=False):
def walk(path):
for root, dirs, files in os.walk(path):
for name in files:
yield name
for directory in dirs:
yield [os.path.join(directory, x) for x in walk(os.path.join(root, directory))]
parser = Parser()
if calibre:
for root, authors, _ in os.walk(path):
for author in authors:
for author_path, titles, _ in os.walk(os.path.join(root, author)):
for title in titles:
for title_path, _, files in os.walk(os.path.join(author_path, title)):
try:
if 'metadata.opf' not in files:
raise MetadataMissing("Missing in %s" % title_path)
data = parser.parse_metadata_file(os.path.join(title_path, 'metadata.opf'))
for file in files:
if not file.lower().endswith(('.pdf', '.epub', '.mobi')):
continue
try:
book, created = Book.objects.import_create_book(os.path.join(title_path, file), data)
if created:
print(str(book))
except (InvalidImportData, NoDataError) as e:
print(str(e))
except MetadataMissing as e:
print(str(e))
else:
for file in walk(path):
# ToDo Clean up
# ToDo the check for is file shouldn't be necessary...
if isinstance(file, list):
for _file in file:
if not _file.lower().endswith(('.pdf', '.epub', '.mobi')):
continue
if os.path.isfile(os.path.join(path, _file)):
try:
print(file)
Book.objects.import_create_book(os.path.join(path, _file), parser.parse(_file))
except (InvalidImportData, NoDataError) as e:
print("No clue what '%s' is" % _file)
else:
if not file.lower().endswith(('.pdf', '.epub', '.mobi')):
continue
if os.path.isfile(os.path.join(path, file)):
try:
print(file)
Book.objects.import_create_book(os.path.join(path, file), parser.parse(file))
except (InvalidImportData, NoDataError) as e:
print("No clue what '%s' is" % file)
# coding=utf-8
from django.core.management.base import BaseCommand
from BookWurm.management.handle_import import handle_import_ebooks
class Command(BaseCommand):
help = "Creates an application directory structure for the specified application name."
args = "directory"
missing_args_message = "No directory supplied."
def add_arguments(self, parser):
parser.add_argument(
'--directory', '-d', action='store', dest='directory',
help='The path to the books')
parser.add_argument(
'--calibre', action='store_true', dest='calibre',
help='Treat it as a Calibre library')
def handle(self, *app_labels, **options):
handle_import_ebooks(path=options.get('directory'), calibre=options.get('calibre'))
import bs4
from bs4 import BeautifulSoup
class MetadataParser(object):
mapping = {
'dc:title': 'title',
'dc:creator': 'author',
'dc:publisher': 'publisher',
'dc:language': 'language',
}
meta_mapping = {
'calibre:rating': 'rating',
'calibre:title_sort': 'sortable_title',
'calibre:series': 'serie_name',
'calibre:series_index': 'serie_index',
}
def parse(self, content):
if not content:
# todo raise exception
return
soup = BeautifulSoup(content, "lxml")
data = {}
metadata = soup.package.metadata
for tag in metadata.children:
if isinstance(tag, bs4.element.Tag):
if tag.name in self.mapping.keys():
data[self.mapping[tag.name]] = tag.text
elif tag.name == 'dc:identifier':
for k, v in tag.attrs.items():
# todo clean this up
if k == 'opf:scheme':
if v == 'ISBN':
isbn = tag.text
if len(isbn) == 10:
data['isbn_10'] = isbn
elif len(isbn) == 13:
data['isbn_13'] = isbn
elif v == 'GOOGLE':
data['google'] = tag.text
elif v == 'AMAZON':
data['amazon'] = tag.text
elif tag.name == 'meta':
if tag.attrs['name'] in self.meta_mapping.keys():
data[self.meta_mapping[tag.attrs['name']]] = tag.attrs['content']
elif tag.name == 'dc:subject':
if 'tags' not in data.keys():
data['tags'] = []
data['tags'].append(tag.text)
if 'author' in data.keys():
data['authors'] = [a.strip() for a in data['author'].split('&')]
del data['author']
return data
def parse_file(self, filename):
with open(filename, "r") as fh:
return self.parse("".join([l.strip() for l in fh]))
class InternalCompareEngine(object):
our_data = {}
ext_data = []
def __init__(self, our_data):
if isinstance(our_data, object):
our_data = dict(our_data)
self.our_data = our_data
def add_ext_data(self, data):
self.ext_data.append(data)
def set_ext_data(self, data):
if not isinstance(data, list):
data = list(data)
self.ext_data = data
def compare(self):
data = self.our_data
# for now, just get the first one
for book in self.ext_data:
if 'isbn_10' in data.keys() and book.isbn_10 is not None:
if data['isbn_10'] != book.isbn_10:
# isbn mismatch
continue
if 'isbn_13' in data.keys() and book.isbn_13 is not None:
if data['isbn_13'] != book.isbn_13:
# isbn mismatch
continue
data['isbn_10'] = book.isbn_10
data['isbn_13'] = book.isbn_13
data['published_year'] = book.published_year
data['publisher'] = book.publisher
return data
from django.core.exceptions import SuspiciousOperation
class ParserException(Exception):
""" abstract parser exception """
pass
class FilenameParserException(ParserException):
"""Invalid name"""
pass
class InvalidImportData(ParserException):
"""Invalid import data"""
pass
class MetadataException(ParserException):
"""Invalid metadata"""
pass
class MetadataMissing(MetadataException):
"""Missing metadata"""
pass
class BaseFactory(object):
def convert(self, data):
raise NotImplementedError
from ..utils.format import title_sort
class Author(object):
name = None
sortable_name = None
def set_name(self, name):
self.name = name
self.sortable_name = name
class Book(object):
title = None
sortable_title = None
authors = []
language = None
serie = None
tags = []
edition = None
isbn_10 = None
isbn_13 = None
ean_13 = None
sbn = None
amazon = None
google = None
goodreads = None
published_year = None
published_month = None
page_count = None
cover = None
rating = None
publisher = None
def set_title(self, title):
self.title = title
self.sortable_title = title_sort(title)
def lookup_author(self, name):
for author in self.authors:
if author.name == name or author.sortable_name == name:
return author
return False
def set_authors(self, authors):
if isinstance(authors, list):
for author in authors:
self.add_author(author)
else:
self.add_author(authors)
def add_author(self, author):
if self.lookup_author(author):
return
_author = Author()
_author.set_name(author)
self.authors.append(_author)
from .model import Book
from .base_factory import BaseFactory
from ..source.openlibrary import Document
class OpenLibraryFactory(BaseFactory):
def convert(self, data):
if not isinstance(data, Document):
raise Exception("Invalid argument supplied")
book = Book()
book.set_title(data.title)
book.set_authors(data.author)
book.publisher = data.publisher
book.published_year = data.first_publish_year
book.goodreads = data.id_goodreads
for isbn in data.isbn:
if isbn.__len__() == 13:
book.isbn_13 = isbn
elif isbn.__len__() == 10:
book.isbn_10 = isbn
book.tags = data.subject
return book
# -*- coding: utf-8 -*-
import os
import re
from BookWurm.metadata.parser.exceptions import FilenameParserException
class FilenameParser(object):
regexes = []
_regexes = [
r'''(?P<a>[\w\d\s,'"&]+?)\s*-\s*(?P<t>[\w\d\s,'"]+)''',
r'''(?P<a>[\w\d\s,\-'"&\.]+?)\s*[-\/]\s*(?:\[\s*(?P<s>(((?P<sa>[[A-Za-z\s,\-'"&]+?)\s*(?P<sb>[\d]+)))|[A-Za-z0-9\s,\-'"]+?)\s*\])?\s*[-\/]?\s*(?P<t>.+)''',
r'''(?P<h>[\w\d\s,\-'"&\.]+)/(?P<a>[\w\d\s,\-'"&\.]+?)\s*-\s*(?:\[\s?(?P<s>(((?P<sa>[[A-Za-z\s,\-'"&]+?)\s*(?P<sb>[\d]+)))|[\w\d\s,\-'"]+?)\s*\])?\s*(?P<t>.+)''',
r'''(?P<a>[\w\d\s,\-'"&\.]+)/(?:\[?\s?(?P<s>(((?P<sa>[[A-Za-z\s,'"&]+?)\s*(?P<sb>[\d]+)))|[\w\d\s,'"&]+?)\s*\]?)?/(?P<t>.+)''',
r'''(?P<a>[\w\d\s,\-'"&\.]+?)\s*-\s*(?P<s>(((?P<sa>[[A-Za-z\s,\-'"&]+?)\s*(?P<sb>[\d]+)))|[\w\d\s,\-'"]+?)?\s*-\s*(?P<t>.+)''',
r'''(?P<a>[\w\d\s,\-'"&\.]+?)\/(?P<s>(((?P<sa>[[A-Za-z\s,\-'"&]+?)\s*(?P<sb>[\d]+)))|[\w\d\s,\-'"]+?)?\/(?P<t>.+)''',
]
regex_map = {
't': 'title',
'a': 'author',
'h': 'head',
's': 'serie_full',
'sa': 'serie_name',
'sb': 'serie_index',
}
def __init__(self):
for regex in self._regexes:
self.regexes.append(re.compile(regex))
def parse(self, line):
def resultmap(data):
mapped = {}
for k, v in self.regex_map.items():
if k in data and data[k]:
mapped[v] = data[k].strip()
if 'head' in mapped:
if mapped['author'] == mapped['head']:
del mapped['head']
mapped['authors'] = [x.strip() for x in mapped['author'].split('&')]
del mapped['author']
return mapped
if line.lower().endswith(('.pdf', '.epub', '.mobi')):
line = os.path.splitext(line)[0]
results = []
for regex in self.regexes:
m = regex.match(line)
if m:
results.append(resultmap(m.groupdict()))
# ToDo
if len(results) == 0:
raise FilenameParserException("Can't make sense of '%s'" % line)
l = 0
result = {}
for d in results:
if len(d) > l:
result = d
return result
# -*- coding: utf-8 -*-
import simplejson
from BookWurm.metadata.parser.compare import InternalCompareEngine
from BookWurm.metadata.parser.exceptions import FilenameParserException
from BookWurm.metadata.parser.factory.openlibrary import OpenLibraryFactory
from .filename import FilenameParser
from .calibre.metadata import MetadataParser
from .source.openlibrary import BookSearch
sources = [
(BookSearch(), OpenLibraryFactory(), )
]
class Parser(object):
fn_parser = None
md_parser = None
def __init__(self, filenameparser=FilenameParser, metadataparser=MetadataParser):
self.fn_parser = filenameparser()
self.md_parser = metadataparser()