#!/usr/local/bin/python2.7
# vim:fileencoding=utf-8
# License: GPLv3 Copyright: 2015, Kovid Goyal <kovid at kovidgoyal.net>

from __future__ import (unicode_literals, division, absolute_import,
                        print_function)
import os
from copy import copy
from collections import namedtuple
from datetime import datetime, time
from functools import partial
from threading import Lock
from urllib import quote

from calibre import prepare_string_for_xml
from calibre.constants import config_dir
from calibre.db.categories import Tag
from calibre.ebooks.metadata.sources.identify import urls_from_identifiers
from calibre.utils.date import isoformat, UNDEFINED_DATE, local_tz
from calibre.utils.config import tweaks
from calibre.utils.formatter import EvalFormatter
from calibre.utils.file_type_icons import EXT_MAP
from calibre.utils.icu import collation_order
from calibre.utils.localization import calibre_langcode_to_name
from calibre.library.comments import comments_to_html, markdown
from calibre.library.field_metadata import category_icon_map

IGNORED_FIELDS = frozenset('cover ondevice path marked au_map size'.split())


def encode_datetime(dateval):
    if dateval is None:
        return "None"
    if not isinstance(dateval, datetime):
        dateval = datetime.combine(dateval, time())
    if hasattr(dateval, 'tzinfo') and dateval.tzinfo is None:
        dateval = dateval.replace(tzinfo=local_tz)
    if dateval <= UNDEFINED_DATE:
        return None
    return isoformat(dateval)

empty_val = ((), '', {})


def add_field(field, db, book_id, ans, field_metadata):
    datatype = field_metadata.get('datatype')
    if datatype is not None:
        val = db._field_for(field, book_id)
        if val is not None and val not in empty_val:
            if datatype == 'datetime':
                val = encode_datetime(val)
                if val is None:
                    return
            elif datatype == 'comments' or field == 'comments':
                ctype = field_metadata.get('display', {}).get('interpret_as', 'html')
                if ctype == 'markdown':
                    val = markdown(val)
                elif ctype == 'long-text':
                    val = '<pre style="white-space:pre-wrap">%s</pre>' % prepare_string_for_xml(val)
                elif ctype == 'short-text':
                    val = '<span">%s</span>' % prepare_string_for_xml(val)
                else:
                    val = comments_to_html(val)
            elif datatype == 'composite' and field_metadata['display'].get('contains_html'):
                val = comments_to_html(val)
            ans[field] = val


def book_as_json(db, book_id):
    db = db.new_api
    with db.safe_read_lock:
        ans = {'formats':db._formats(book_id)}
        if not ans['formats'] and not db.has_id(book_id):
            return None
        fm = db.field_metadata
        for field in fm.all_field_keys():
            if field not in IGNORED_FIELDS:
                add_field(field, db, book_id, ans, fm[field])
        ids = ans.get('identifiers')
        if ids:
            ans['urls_from_identifiers'] = urls_from_identifiers(ids)
        langs = ans.get('languages')
        if langs:
            ans['lang_names'] = {l:calibre_langcode_to_name(l) for l in langs}
    return ans

_include_fields = frozenset(Tag.__slots__) - frozenset({
    'state', 'is_editable', 'is_searchable', 'original_name', 'use_sort_as_name', 'is_hierarchical'
})


def category_as_json(items, category, display_name, count, tooltip=None, parent=None,
        is_editable=True, is_gst=False, is_hierarchical=False, is_searchable=True,
        is_user_category=False, is_first_letter=False):
    ans = {'category': category, 'name': display_name, 'is_category':True, 'count':count}
    if tooltip:
        ans['tooltip'] = tooltip
    if parent:
        ans['parent'] = parent
    if is_editable:
        ans['is_editable'] = True
    if is_gst:
        ans['is_gst'] = True
    if is_hierarchical:
        ans['is_hierarchical'] = is_hierarchical
    if is_searchable:
        ans['is_searchable'] = True
    if is_user_category:
        ans['is_user_category'] = True
    if is_first_letter:
        ans['is_first_letter'] = True
    item_id = 'c' + str(len(items))
    items[item_id] = ans
    return item_id


def category_item_as_json(x, clear_rating=False):
    ans = {}
    for k in _include_fields:
        val = getattr(x, k)
        if val is not None:
            if k == 'original_categories':
                val = tuple(val)
            ans[k] = val.copy() if isinstance(val, set) else val
    s = ans.get('sort')
    if x.use_sort_as_name:
        ans['name'] = s
    if x.original_name != ans['name']:
        ans['original_name'] = x.original_name
    if x.use_sort_as_name or not s or s == ans['name']:
        ans.pop('sort', None)
    if clear_rating:
        del ans['avg_rating']
    return ans

CategoriesSettings = namedtuple(
    'CategoriesSettings', 'dont_collapse collapse_model collapse_at sort_by'
    ' template using_hierarchy grouped_search_terms hidden_categories hide_empty_categories')


class GroupedSearchTerms(object):

    __slots__ = ('keys', 'vals', 'hash')

    def __init__(self, src):
        self.keys = frozenset(src)
        self.hash = hash(self.keys)
        # We dont need to store values since this is used as part of a key for
        # a cache and if the values have changed the cache will be invalidated
        # for other reasons anyway (last_modified() will have changed on the
        # db)

    def __contains__(self, val):
        return val in self.keys

    def __hash__(self):
        return self.hash

    def __eq__(self, other):
        try:
            return self.keys == other.keys
        except AttributeError:
            return False

_icon_map = None
_icon_map_lock = Lock()


def icon_map():
    global _icon_map
    with _icon_map_lock:
        if _icon_map is None:
            from calibre.gui2 import gprefs
            _icon_map = category_icon_map.copy()
            custom_icons = gprefs.get('tags_browser_category_icons', {})
            for k, v in custom_icons.iteritems():
                if os.access(os.path.join(config_dir, 'tb_icons', v), os.R_OK):
                    _icon_map[k] = '_' + quote(v)
            _icon_map['file_type_icons'] = {
                k:'mimetypes/%s.png' % v for k, v in EXT_MAP.iteritems()
            }
        return _icon_map


def categories_settings(query, db):
    dont_collapse = frozenset(query.get('dont_collapse', '').split(','))
    partition_method = query.get('partition_method', 'first letter')
    if partition_method not in {'first letter', 'disable', 'partition'}:
        partition_method = 'first letter'
    try:
        collapse_at = max(0, int(float(query.get('collapse_at', 25))))
    except Exception:
        collapse_at = 25
    sort_by = query.get('sort_tags_by', 'name')
    if sort_by not in {'name', 'popularity', 'rating'}:
        sort_by = 'name'
    collapse_model = partition_method if collapse_at else 'disable'
    template = None
    if collapse_model != 'disable':
        if sort_by != 'name':
            collapse_model = 'partition'
        template = tweaks['categories_collapsed_%s_template' % sort_by]
    using_hierarchy = frozenset(db.pref('categories_using_hierarchy', []))
    hidden_categories = frozenset(db.pref('tag_browser_hidden_categories', set()))
    return CategoriesSettings(
        dont_collapse, collapse_model, collapse_at, sort_by, template,
        using_hierarchy, GroupedSearchTerms(db.pref('grouped_search_terms', {})),
        hidden_categories, query.get('hide_empty_categories') == 'yes')


def create_toplevel_tree(category_data, items, field_metadata, opts):
    # Create the basic tree, containing all top level categories , user
    # categories and grouped search terms
    last_category_node, category_node_map, root = None, {}, {'id':None, 'children':[]}
    node_id_map = {}
    category_nodes, recount_nodes = [], []
    order = tweaks['tag_browser_category_order']
    defvalue = order.get('*', 100)
    categories = [category for category in field_metadata if category in category_data]
    scats = sorted(categories, key=lambda x: order.get(x, defvalue))

    for category in scats:
        is_user_category = category.startswith('@')
        is_gst, tooltip = (is_user_category and category[1:] in opts.grouped_search_terms), ''
        cdata = category_data[category]
        if is_gst:
            tooltip = _('The grouped search term name is "{0}"').format(category)
        elif category != 'news':
            cust_desc = ''
            fm = field_metadata[category]
            if fm['is_custom']:
                cust_desc = fm['display'].get('description', '')
                if cust_desc:
                    cust_desc = '\n' + _('Description:') + ' ' + cust_desc
            tooltip = _('The lookup/search name is "{0}"{1}').format(category, cust_desc)

        if is_user_category:
            path_parts = category.split('.')
            path = ''
            last_category_node = None
            current_root = root
            for i, p in enumerate(path_parts):
                path += p
                if path not in category_node_map:
                    last_category_node = category_as_json(
                        items, path, (p[1:] if i == 0 else p), len(cdata),
                        parent=last_category_node, tooltip=tooltip,
                        is_gst=is_gst, is_editable=((not is_gst) and (i == (len(path_parts)-1))),
                        is_hierarchical=False if is_gst else 5, is_user_category=True
                    )
                    node_id_map[last_category_node] = category_node_map[path] = node = {'id':last_category_node, 'children':[]}
                    category_nodes.append(last_category_node)
                    recount_nodes.append(node)
                    current_root['children'].append(node)
                    current_root = node
                else:
                    current_root = category_node_map[path]
                    last_category_node = current_root['id']
                path += '.'
        else:
            last_category_node = category_as_json(
                items, category, field_metadata[category]['name'], len(cdata),
                tooltip=tooltip
            )
            category_node_map[category] = node_id_map[last_category_node] = node = {'id':last_category_node, 'children':[]}
            root['children'].append(node)
            category_nodes.append(last_category_node)
            recount_nodes.append(node)

    return root, node_id_map, category_nodes, recount_nodes


def build_first_letter_list(category_items):
    # Build a list of 'equal' first letters by noticing changes
    # in ICU's 'ordinal' for the first letter. In this case, the
    # first letter can actually be more than one letter long.
    cl_list = [None] * len(category_items)
    last_ordnum = 0
    last_c = ' '
    for idx, tag in enumerate(category_items):
        if not tag.sort:
            c = ' '
        else:
            c = icu_upper(tag.sort)
        ordnum, ordlen = collation_order(c)
        if last_ordnum != ordnum:
            last_c = c[0:ordlen]
            last_ordnum = ordnum
        cl_list[idx] = last_c
    return cl_list

categories_with_ratings = {'authors', 'series', 'publisher', 'tags'}


def get_name_components(name):
    components = filter(None, [t.strip() for t in name.split('.')])
    if not components or '.'.join(components) != name:
        components = [name]
    return components


def collapse_partition(collapse_nodes, items, category_node, idx, tag, opts, top_level_component,
    cat_len, category_is_hierarchical, category_items, eval_formatter, is_gst,
    last_idx, node_parent):
    # Only partition at the top level. This means that we must not do a break
    # until the outermost component changes.
    if idx >= last_idx + opts.collapse_at and not tag.original_name.startswith(top_level_component+'.'):
        last = idx + opts.collapse_at - 1 if cat_len > idx + opts.collapse_at else cat_len - 1
        if category_is_hierarchical:
            ct = copy(category_items[last])
            components = get_name_components(ct.original_name)
            ct.sort = ct.name = components[0]
            # Do the first node after the last node so that the components
            # array contains the right values to be used later
            ct2 = copy(tag)
            components = get_name_components(ct2.original_name)
            ct2.sort = ct2.name = components[0]
            format_data = {'last': ct, 'first':ct2}
        else:
            format_data = {'first': tag, 'last': category_items[last]}

        name = eval_formatter.safe_format(opts.template, format_data, '##TAG_VIEW##', None)
        if not name.startswith('##TAG_VIEW##'):
            # Formatter succeeded
            node_id = category_as_json(
                items, items[category_node['id']].category, name, 0,
                parent=category_node['id'], is_editable=False, is_gst=is_gst,
                is_hierarchical=category_is_hierarchical, is_searchable=False)
            node_parent = {'id':node_id, 'children':[]}
            collapse_nodes.append(node_parent)
            category_node['children'].append(node_parent)
        last_idx = idx  # remember where we last partitioned
    return last_idx, node_parent


def collapse_first_letter(collapse_nodes, items, category_node, cl_list, idx, is_gst, category_is_hierarchical, collapse_letter, node_parent):
    cl = cl_list[idx]
    if cl != collapse_letter:
        collapse_letter = cl
        node_id = category_as_json(
            items, items[category_node['id']]['category'], collapse_letter, 0,
            parent=category_node['id'], is_editable=False, is_gst=is_gst,
            is_hierarchical=category_is_hierarchical, is_first_letter=True)
        node_parent = {'id':node_id, 'children':[]}
        category_node['children'].append(node_parent)
        collapse_nodes.append(node_parent)
    return collapse_letter, node_parent


def process_category_node(
        category_node, items, category_data, eval_formatter, field_metadata,
        opts, tag_map, hierarchical_tags, node_to_tag_map, collapse_nodes,
        intermediate_nodes, hierarchical_items):
    category = items[category_node['id']]['category']
    category_items = category_data[category]
    cat_len = len(category_items)
    if cat_len <= 0:
        return
    collapse_letter = None
    is_gst = items[category_node['id']].get('is_gst', False)
    collapse_model = 'disable' if category in opts.dont_collapse else opts.collapse_model
    fm = field_metadata[category]
    category_child_map = {}
    is_user_category = fm['kind'] == 'user' and not is_gst
    top_level_component = 'z' + category_items[0].original_name
    last_idx = -opts.collapse_at
    category_is_hierarchical = (
        category in opts.using_hierarchy and opts.sort_by == 'name' and
        category not in {'authors', 'publisher', 'news', 'formats', 'rating'}
    )
    clear_rating = category not in categories_with_ratings and not fm['is_custom'] and not fm['kind'] == 'user'
    collapsible = collapse_model != 'disable' and cat_len > opts.collapse_at
    partitioned = collapse_model == 'partition'
    cl_list = build_first_letter_list(category_items) if collapsible and collapse_model == 'first letter' else ()
    node_parent = category_node

    def create_tag_node(tag, parent):
        # User categories contain references to items in other categories, so
        # reflect that in the node structure as well.
        node_data = tag_map.get(id(tag), None)
        if node_data is None:
            node_id = 'n%d' % len(tag_map)
            node_data = items[node_id] = category_item_as_json(tag, clear_rating=clear_rating)
            tag_map[id(tag)] = (node_id, node_data)
            node_to_tag_map[node_id] = tag
        else:
            node_id, node_data = node_data
        node = {'id':node_id, 'children':[]}
        parent['children'].append(node)
        return node, node_data

    for idx, tag in enumerate(category_items):

        if collapsible:
            if partitioned:
                last_idx, node_parent = collapse_partition(
                collapse_nodes, items, category_node, idx, tag, opts, top_level_component,
                cat_len, category_is_hierarchical, category_items,
                eval_formatter, is_gst, last_idx, node_parent)
            else:  # by 'first letter'
                collapse_letter, node_parent = collapse_first_letter(
                    collapse_nodes, items, category_node, cl_list, idx, is_gst, category_is_hierarchical, collapse_letter, node_parent)
        else:
            node_parent = category_node

        tag_is_hierarchical = id(tag) in hierarchical_tags
        components = get_name_components(tag.original_name) if category_is_hierarchical or tag_is_hierarchical else (tag.original_name,)

        if not tag_is_hierarchical and (
                is_user_category or not category_is_hierarchical or len(components) == 1 or
                (fm['is_custom'] and fm['display'].get('is_names', False))
        ):  # A non-hierarchical leaf item in a non-hierarchical category
            node, item = create_tag_node(tag, node_parent)
            category_child_map[item['name'], item['category']] = node
            intermediate_nodes[tag.category, tag.original_name] = node
        else:
            orig_node_parent = node_parent
            for i, component in enumerate(components):
                if i == 0:
                    child_map = category_child_map
                else:
                    child_map = {}
                    for sibling in node_parent['children']:
                        item = items[sibling['id']]
                        if not item.get('is_category', False):
                            child_map[item['name'], item['category']] = sibling
                cm_key = component, tag.category
                if cm_key in child_map:
                    node_parent = child_map[cm_key]
                    node_id = node_parent['id']
                    item = items[node_id]
                    item['is_hierarchical'] = 3 if tag.category == 'search' else 5
                    if tag.id_set is not None:
                        item['id_set'] |= tag.id_set
                    hierarchical_items.add(node_id)
                    hierarchical_tags.add(id(node_to_tag_map[node_parent['id']]))
                else:
                    if i < len(components) - 1:  # Non-leaf node
                        original_name = '.'.join(components[:i+1])
                        inode = intermediate_nodes.get((tag.category, original_name), None)
                        if inode is None:
                            t = copy(tag)
                            t.original_name, t.count = original_name, 0
                            t.is_editable, t.is_searchable = False, category == 'search'
                            node_parent, item = create_tag_node(t, node_parent)
                            hierarchical_tags.add(id(t))
                            intermediate_nodes[tag.category, original_name] = node_parent
                        else:
                            item = items[inode['id']]
                            ch = node_parent['children']
                            node_parent = {'id':inode['id'], 'children':[]}
                            ch.append(node_parent)
                    else:
                        node_parent, item = create_tag_node(tag, node_parent)
                        if not is_user_category:
                            item['original_name'] = tag.name
                        intermediate_nodes[tag.category, tag.original_name] = node_parent
                    item['name'] = component
                    item['is_hierarchical'] = 3 if tag.category == 'search' else 5
                    hierarchical_tags.add(id(tag))
                    child_map[cm_key] = node_parent
                items[node_parent['id']]['id_set'] |= tag.id_set
            node_parent = orig_node_parent


def iternode_descendants(node):
    for child in node['children']:
        yield child
        for x in iternode_descendants(child):
            yield x


def fillout_tree(root, items, node_id_map, category_nodes, category_data, field_metadata, opts, book_rating_map):
    eval_formatter = EvalFormatter()
    tag_map, hierarchical_tags, node_to_tag_map = {}, set(), {}
    first, later, collapse_nodes, intermediate_nodes, hierarchical_items = [], [], [], {}, set()
    # User categories have to be processed after normal categories as they can
    # reference hierarchical nodes that were created only during processing of
    # normal categories
    for category_node_id in category_nodes:
        cnode = items[category_node_id]
        coll = later if cnode.get('is_user_category', False) else first
        coll.append(node_id_map[category_node_id])

    for coll in (first, later):
        for cnode in coll:
            process_category_node(
                cnode, items, category_data, eval_formatter, field_metadata,
                opts, tag_map, hierarchical_tags, node_to_tag_map,
                collapse_nodes, intermediate_nodes, hierarchical_items)

    # Do not store id_set in the tag items as it is a lot of data, with not
    # much use. Instead only update the ratings and counts based on id_set
    for item_id in hierarchical_items:
        item = items[item_id]
        total = count = 0
        for book_id in item['id_set']:
            rating = book_rating_map.get(book_id, 0)
            if rating:
                total += rating/2.0
                count += 1
        item['avg_rating'] = float(total)/count if count else 0

    for item_id, item in tag_map.itervalues():
        id_len = len(item.pop('id_set', ()))
        if id_len:
            item['count'] = id_len

    for node in collapse_nodes:
        item = items[node['id']]
        item['count'] = sum(1 for _ in iternode_descendants(node))


def render_categories(opts, db, category_data):
    items = {}
    with db.safe_read_lock:
        root, node_id_map, category_nodes, recount_nodes = create_toplevel_tree(category_data, items, db.field_metadata, opts)
        fillout_tree(root, items, node_id_map, category_nodes, category_data, db.field_metadata, opts, db.fields['rating'].book_value_map)
    for node in recount_nodes:
        item = items[node['id']]
        item['count'] = sum(1 for x in iternode_descendants(node) if not items[x['id']].get('is_category', False))
    if opts.hidden_categories:
        # We have to remove hidden categories after all processing is done as
        # items from a hidden category could be in a user category
        root['children'] = filter((lambda child:items[child['id']]['category'] not in opts.hidden_categories), root['children'])
    if opts.hide_empty_categories:
        root['children'] = filter((lambda child:items[child['id']]['count'] > 0), root['children'])
    return {'root':root, 'item_map': items}


def categories_as_json(ctx, rd, db):
    opts = categories_settings(rd.query, db)
    return ctx.get_tag_browser(rd, db, opts, partial(render_categories, opts))

# Test tag browser {{{


def dump_categories_tree(data):
    root, items = data['root'], data['item_map']
    ans, indent = [], '  '

    def dump_node(node, level=0):
        item = items[node['id']]
        rating = item.get('avg_rating', None) or 0
        if rating:
            rating = ',rating=%.1f' % rating
        try:
            ans.append(indent*level + item['name'] + ' [count=%s%s]' % (item['count'], rating or ''))
        except KeyError:
            print(item)
            raise
        for child in node['children']:
            dump_node(child, level+1)
        if level == 0:
            ans.append('')
    [dump_node(c) for c in root['children']]
    return '\n'.join(ans)


def dump_tags_model(m):
    from PyQt5.Qt import QModelIndex, Qt
    ans, indent = [], '  '

    def dump_node(index, level=-1):
        if level > -1:
            ans.append(indent*level + index.data(Qt.UserRole).dump_data())
        for i in xrange(m.rowCount(index)):
            dump_node(m.index(i, 0, index), level + 1)
        if level == 0:
            ans.append('')
    dump_node(QModelIndex())
    return '\n'.join(ans)


def test_tag_browser(library_path=None):
    ' Compare output of server and GUI tag browsers '
    from calibre.library import db
    olddb = db(library_path)
    db = olddb.new_api
    opts = categories_settings({}, db)
    # opts = opts._replace(hidden_categories={'publisher'})
    opts = opts._replace(hide_empty_categories=True)
    category_data = db.get_categories(sort=opts.sort_by, first_letter_sort=opts.collapse_model == 'first letter')
    data = render_categories(opts, db, category_data)
    srv_data = dump_categories_tree(data)
    from calibre.gui2 import Application, gprefs
    from calibre.gui2.tag_browser.model import TagsModel
    prefs = {
        'tags_browser_category_icons':gprefs['tags_browser_category_icons'],
        'tags_browser_collapse_at':opts.collapse_at,
        'tags_browser_partition_method': opts.collapse_model,
        'tag_browser_dont_collapse': opts.dont_collapse,
        'tag_browser_hide_empty_categories': opts.hide_empty_categories,
    }
    app = Application([])
    m = TagsModel(None, prefs)
    m.set_database(olddb, opts.hidden_categories)
    m_data = dump_tags_model(m)
    if m_data == srv_data:
        print('No differences found in the two Tag Browser implementations')
        raise SystemExit(0)
    from calibre.gui2.tweak_book.diff.main import Diff
    d = Diff(show_as_window=True)
    d.string_diff(m_data, srv_data, left_name='GUI', right_name='server')
    d.exec_()
    del app
# }}}
