import re
import collections
import datetime
import operator
import urllib
import hashlib
import random
import json

import webapp2
import logging
from google.appengine.ext import ndb
from google.appengine.api import memcache
from google.appengine.runtime import apiproxy_errors
from google.appengine.api import taskqueue
from google.appengine.api import urlfetch

import thumbs
from common import Shortener, Jinja


class DBError(Exception):
    def __init__(self, value):
        self.value = value

    def __str__(self):
        if isinstance(self.value, (str, unicode)):
            return self.value
        else:
            return str(self.value)


class CacheEntry(ndb.Model):
    blob = ndb.PickleProperty(indexed=False)


class Views:
    _views_cache_key = 'views'

    def __init__(self, increment_views=False, clear=False):
        self.increment_views = increment_views
        if not clear:
            self._views_dict = Views._get_from_db()
        else:
            self._views_dict = {}

    @staticmethod
    def _get_from_db():
        entry_key = ndb.Key(CacheEntry, Views._views_cache_key)
        entry = entry_key.get()
        if entry:
            views = entry.blob
        else:
            views = {}

        return views

    def get_views(self, name):
        if not self.increment_views:
            if name in self._views_dict:
                return self._views_dict[name]
            else:
                return 0
        else:
            if name in self._views_dict:
                self._views_dict[name] += 1
            else:
                self._views_dict[name] = 1
            return self._views_dict[name]

    def set_views(self, name, views):
        # int() will raise a ValueError if there's anything wrong
        self._views_dict[name] = int(views)

    def has_views(self, name):
        '''Returns True if video has views set or False if not.'''
        return name in self._views_dict

    def increment_video_views(self, name, amount=1, auto_save=True):
        '''Increment the views of a single video, but only if it already exists
        in the views dict. Returns incremented views or None if videos doesn't
        exist.'''
        if name in self._views_dict:
            self._views_dict[name] += amount

            if auto_save:
                self.save()

            return self._views_dict[name]
        return None

    def rename_video(self, name, new_name):
        if not name in self._views_dict:
            return

        self._views_dict[new_name] = self._views_dict[name]
        del self._views_dict[name]

    def delete_video(self, name):
        if not name in self._views_dict:
            return
        del self._views_dict[name]

    def save(self):
        '''Save view count to DB.'''
        # store in DB
        entry_key = ndb.Key(CacheEntry, Views._views_cache_key)
        entry = CacheEntry(blob=self._views_dict, key=entry_key)
        entry.put()


class Sources:
    _sources_cache_key = 'sources_idx'

    # number of digits to use in shortlink
    _shortlink_len = 8
    _shortlink_regex = re.compile('^[0-9a-f]{8}$', re.I)

    def __init__(self, parent=None):
        self._sources = None
        self._shortlinks = {}

        # store pointer to parent for accessing entries when rebuilding index
        self._pseudo_db = parent

    @property
    def _index(self):
        if self._sources is None:
            self._get_sources_index_from_cache()
        return self._sources

    def get_shortlink(self, source_or_entry):
        '''Generate shortlink for source, if not already generated.'''
        # accept both an entry or the source itself
        if isinstance(source_or_entry, dict):
            entry = source_or_entry
            source = entry['source_text']
        else:
            source = source_or_entry

        if not source:
            raise ValueError('Cannot generate shortlink: missing source')

        if source in self._shortlinks:
            return self._shortlinks[source]

        shortlink = hashlib.sha1(source.encode('utf-8', 'ignore')).hexdigest()[:Sources._shortlink_len]
        self._shortlinks[source] = shortlink
        return shortlink

    def _get_videos_for_shortlink(self, shortlink):
        '''Return list of video names under a shortlink, or an empty list if not
        found.'''
        if shortlink in self._index:
            # if it's an alias to another shortlink, recurse
            if isinstance(self._index[shortlink], str):
                return self._get_videos_for_shortlink(self._index[shortlink])
            else:
                return self._index[shortlink]
        return []

    def get_videos(self, source):
        '''Returns list of video names belonging to a certain source or an empty
        list if not found.'''
        if not source:
            return []

        # if source is a shortlink, use it
        if Sources._shortlink_regex.match(source):
            return self._get_videos_for_shortlink(source)

        # if not, generate shortlink and try that
        shortlink = self.get_shortlink(source)
        return self._get_videos_for_shortlink(shortlink)

    def _get_sources_index_from_cache(self):
        '''Retrieve sources index from memcache.'''
        self._sources = memcache.get(Sources._sources_cache_key)

        # if not found, try DB
        if self._sources is None:
            self._get_sources_index_from_db()
        else:
            logging.debug('got sources index from cache')

    def _get_sources_index_from_db(self):
        '''Retrieve sources index from the DB.'''
        entry_key = ndb.Key(CacheEntry, Sources._sources_cache_key)
        entry = entry_key.get()

        if entry:
            self._sources = entry.blob

            # cache it
            memcache.set(Sources._sources_cache_key, self._sources)

            logging.debug('got sources index from DB')
        elif entry is None and self._pseudo_db:
            # build from scratch
            self.build_sources_index(entries=self._pseudo_db.videos)

    def build_sources_index(self, entries=None):
        self._sources = {}

        logging.debug('rebuilding sources index')

        # if DB is empty, just leave the index empty as well
        if entries:
            if isinstance(entries, dict):
                entries = PseudoDB.dict2list(entries)

            for entry in entries:
                shortlink = self.get_shortlink(entry)

                if not shortlink in self._sources:
                    self._sources[shortlink] = []

                self._sources[shortlink].append(entry['name'])

                # build secondary shortlink if necessary
                if 'source_url' in entry and entry['source_url']:
                    secondary_shortlink = self.get_shortlink(entry['source_url'])
                    if not secondary_shortlink in self._sources:
                        # redirect to primary shortlink
                        self._sources[secondary_shortlink] = shortlink

        # store in DB
        entry_key = ndb.Key(CacheEntry, Sources._sources_cache_key)
        entry = CacheEntry(blob=self._sources, key=entry_key)
        entry.put()

        # cache it
        memcache.set(Sources._sources_cache_key, self._sources)

    def index_entry(self, entry):
        shortlink = self.get_shortlink(entry)
        if not shortlink in self._index:
            self._index[shortlink] = []
        if not entry['name'] in self._index[shortlink]:
            self._index[shortlink].append(entry['name'])

        # build secondary shortlink if necessary
        if 'source_url' in entry and entry['source_url']:
            secondary_shortlink = self.get_shortlink(entry['source_url'])
            if not secondary_shortlink in self._index:
                # redirect to primary shortlink
                self._index[secondary_shortlink] = shortlink

    def unindex_entry(self, entry):
        shortlink = self.get_shortlink(entry)
        if shortlink in self._index and entry['name'] in self._index[shortlink]:
            self._index[shortlink].remove(entry['name'])
            if not self._index[shortlink]:
                del self._index[shortlink]

                # remove secondary shortlink if available
                # note that this won't be done if the url exists but this entry
                # is missing it, because it would be too expensive to do it here
                if 'source_url' in entry and entry['source_url']:
                    secondary_shortlink = self.get_shortlink(entry['source_url'])
                    if secondary_shortlink in self._index:
                        del self._index[secondary_shortlink]

    def get_all_shortlinks(self):
        '''Returns list of all primary shortlinks.

        Will also delete secondary shortlinks linking to deleted primary
        shortlinks (and self.save()), if any are found.
        '''
        if not self._index:
            return []

        shortlinks = []
        to_delete = []

        for shortlink in self._index:
            if isinstance(self._index[shortlink], list):
                shortlinks.append(shortlink)
            elif not self._index[shortlink] in self._index:
                logging.debug('broken shortlink found, deleting: {}'.format(shortlink))
                to_delete.append(shortlink)

        if to_delete:
            for shortlink in to_delete:
                del self._index[shortlink]
            self.save()

        return shortlinks

    def save(self, insert_in_db=True):
        '''Save sources to DB and memcache.'''
        # make sure it has been built first
        if self._sources is None:
            raise ValueError('Cannot save sources index before it has been retrieved or built')

        # store in cache
        memcache.set(Sources._sources_cache_key, self._sources)

        # store in DB
        entry_key = ndb.Key(CacheEntry, Sources._sources_cache_key)
        entry = CacheEntry(blob=self._sources, key=entry_key)
        # when not inserting in DB, return entry so the caller can do it
        if insert_in_db:
            entry.put()
        else:
            return entry


class PseudoDB:
    _cache_key_base = 'blog_entries'
    _tags_cache_key = 'tags_idx'
    _pages_cache_key = 'blog_entries_pages'

    _valid_entry_keys = ['posted_at', 'source_text', 'source_url', 'urls',
            'name', 'tags', 'has_audio', 'watch_url', 'views']

    # default parameters accepted by pages
    _accepted_vars = ['order', 'since', 'until', 'offset', 'limit', 'type',
            'names', 'tags', 'source', 'hosts', 'keys']

    _thumb_url_base = 'https://kastden.org/_loopvid_thumbs_ln/'
    _video_basename_regex = re.compile('^[^/]{2}/[^,&]+')

    _root_link = '/'

    _list_separator = ','
    _names_list_separators = [';', '+', ' ']
    # the preferred separator
    names_list_separator = '+'

    # number of videos to store in a single DB entry
    _page_size = 1000

    def __init__(self, increment_views=False):
        self._get_entries_from_cache()

        self._views = Views(increment_views=increment_views)

        self.sources = Sources(parent=self)

    def query(self, return_type='list', order='descending', limit=None,
            offset=None, since=None, until=None, tags=None, tags_op='or',
            names=None, source=None, hosts=None, keys=None, expand=False,
            sort_by='posted_at', _filtered=True):

        entries = PseudoDB.list2ordered_dict(self.videos)
        has_more = False

        if _filtered:
            # filtering that uses direct access first
            if tags:
                entries = self.filter_by_tags(entries=entries, tags=tags,
                                              tags_op=tags_op)

            if names:
                entries = self.filter_by_names(entries=entries, names=names)

            if hosts:
                entries = self.filter_by_hosts(entries=entries, hosts=hosts)

            # source filtering will be O(1) if sources are cached, O(n) otherwise
            if source:
                entries = self.filter_by_source(entries=entries, source=source)

            # filtering that does exhaustive searching last
            entries = self.filter_entries(entries=entries, since=since, until=until)

        # return correct type
        if return_type == 'list':
            entries = PseudoDB.dict2list(entries)

        # sort
        ascending = False
        randomize = False
        if order == 'ascending':
            ascending = True
        elif order == 'random':
            randomize = True

        # add views to entries when sorting by views, even if we're not
        # expanding
        added_views = False
        if sort_by == 'views':
            self.add_views(entries)
            added_views = True

        entries = PseudoDB.sort_entries(entries, ascending=ascending,
                randomize=randomize, limit=limit, key=sort_by)

        # limit
        # note that if randomizing, the entries will have already been limited at
        # sort_entries(), and also offset wouldn't make sense
        if (limit or offset) and (not randomize):
            entries, has_more = PseudoDB.limit_entries(entries, limit=limit, offset=offset)

        # expand entries
        if expand:
            entries = self.expand_entry_dicts(entries)
            # add views if we haven't already
            if not added_views:
                self.add_views(entries)

        # drop unwanted keys
        if keys:
            entries = self.filter_keys(entries=entries, keys=keys)

        # all done
        return entries, has_more

    def filter_by_tags(self, entries=None, tags=None, tags_op='or'):
        # split not tags so we can apply them at the end
        included_tags = []
        excluded_tags = []
        for tag in tags:
            if tag.startswith('-'):
                excluded_tags.append(tag[1:])
            else:
                included_tags.append(tag)

        # gather all entries matching the included tags
        names = None
        for tag in included_tags:
            tag_set = set(self.tags_idx[tag]) if tag in self.tags_idx else set()

            if names is None:
                names = tag_set
            elif tags_op == 'and':
                names = names.intersection(tag_set)
            else: # or
                names = names.union(tag_set)

        # if only excluded tags were specified, start from all the currently
        # selected entries and then exclude the tags
        if excluded_tags and names is None:
            names = set(entries.keys())

        # exclude tags
        for tag in excluded_tags:
            tag_set = set(self.tags_idx[tag]) if tag in self.tags_idx else set()
            names = names.difference(tag_set)

        filtered_dict = {}
        # for each entry matching the tags
        for name in names:
            # if entry is selected, keep it
            if name in entries:
                filtered_dict[name] = entries[name]
        return filtered_dict

    def filter_by_names(self, entries=None, names=None):
        filtered_dict = {}
        for name in names:
            if name in entries:
                filtered_dict[name] = entries[name]
        return filtered_dict

    def filter_by_hosts(self, entries=None, hosts=None):
        filtered_dict = {}
        for name in entries:
            host = name[:2]
            if host in hosts:
                filtered_dict[name] = entries[name]
        return filtered_dict

    def filter_by_source(self, entries=None, source=None):
        if not source:
            return {}

        return self.filter_by_names(entries=entries, names=self.sources.get_videos(source))

    def filter_entries(self, entries=None, since=None, until=None):
        # speed up empty searches
        if not (since or until):
            return entries

        filtered_dict = {}
        for name in entries:
            entry = entries[name]

            if since and entry['posted_at'] < since:
                continue
            if until and entry['posted_at'] > until:
                continue

            filtered_dict[name] = entry
        return filtered_dict

    @staticmethod
    def limit_entries(entries=None, limit=0, offset=0):
        if entries is None:
            entries = {}

        is_dict = False
        if isinstance(entries, dict):
            is_dict = True
            entries = PseudoDB.dict2list(entries)

        # skip first entries
        if offset > 0:
            entries = entries[offset:]

        # limit number of entries
        has_more = False
        if limit < len(entries):
            entries = entries[:limit]
            has_more = True

        if is_dict:
            entries = PseudoDB.list2ordered_dict(entries)

        return entries, has_more

    def filter_keys(self, entries=None, keys=None):
        if entries is None:
            entries = {}
        if keys is None:
            keys = []

        # never drop name since it's what identifies the entry
        if not 'name' in keys:
            keys.append('name')

        if isinstance(entries, dict):
            for name in entries:
                new_entry = {}
                entry = entries[name]
                for key in keys:
                    if key in entry:
                        new_entry[key] = entry[key]
                # replace entry with new one
                entries[name] = new_entry
        else: # list
            for i in range(len(entries)):
                new_entry = {}
                entry = entries[i]
                for key in keys:
                    if key in entry:
                        new_entry[key] = entry[key]
                # replace entry with new one
                entries[i] = new_entry

        return entries

    @staticmethod
    def sort_entries(entries, ascending=False, randomize=False,
            force_list=False, limit=None, key='posted_at'):
        # allow both list or dict
        is_dict = False
        if isinstance(entries, dict):
            is_dict = True
            entries = PseudoDB.dict2list(entries)

        if not randomize:
            entries = sorted(entries, key=operator.itemgetter(key))
        else:
            random.seed()
            # if a limit is set, then we only have to choose that many entries
            if limit and limit < len(entries):
                entries = random.sample(entries, limit)
            else:
                random.shuffle(entries)

        if (not randomize) and (not ascending):
            # reverse so newer entries come first
            entries.reverse()

        # turn back to dict if necessary
        if is_dict and (not force_list):
            entries = PseudoDB.list2ordered_dict(entries)

        return entries

    def expand_entry_dicts(self, entry_dicts):
        entry_number = 0
        # accept both a list or a dict containing entry_dicts
        if isinstance(entry_dicts, dict):
            for key in entry_dicts:
                self.expand_one_entry_dict(entry_dicts[key], entry_number=entry_number)
                entry_number += 1
        else:
            for entry in entry_dicts:
                self.expand_one_entry_dict(entry, entry_number=entry_number)
                entry_number += 1

        return entry_dicts

    @staticmethod
    def get_urls(shortened_url_list):
        return Shortener.expand_urls(shortened_url_list)

    @staticmethod
    def get_watch_url(shortened_url_list):
        return PseudoDB._root_link + shortened_url_list

    def expand_one_entry_dict(self, entry, entry_number=0):
        # expand urls
        entry['urls'] =  PseudoDB.get_urls(entry['name'])
        entry['watch_url'] = PseudoDB.get_watch_url(entry['name'])

        # thumbnail url
        entry['img'] = PseudoDB.get_thumb_url(entry['name'])

        # tags
        entry['has_audio'] = False
        if 'tags' in entry and isinstance(entry['tags'], list):
            entry['tags'].sort()

            if 'audio' in entry['tags']:
                entry['has_audio'] = True

        # source shortlink
        entry['source_shortlink'] = self.sources.get_shortlink(entry)

        # REMOVE
        # post number
        entry['post_number'] = entry_number

        return entry

    @staticmethod
    def dict2list(_dict):
        # tolerate getting a list instead of a dict
        if isinstance(_dict, list):
            return _dict

        _list = []
        for key in _dict:
            _list.append(_dict[key])
        return _list

    @staticmethod
    def list2ordered_dict(entries_list):
        '''Converts a list of entries into an ordered dict where the entries' names are the keys.'''
        # tolerate getting a dict instead of a list
        if isinstance(entries_list, list):
            entries_list = PseudoDB.dict2list(entries_list)

        ordered_dict = collections.OrderedDict()
        for entry in entries_list:
            ordered_dict[entry['name']] = entry

        return ordered_dict

    @staticmethod
    def get_thumb_url(name):
        '''Returns thumbnail url from video's name.'''
        # format is xy/video -> xy_video.jpg
        # on shortened lists, only the first video is used, e.g.
        # xy/video1,video2 -> xy_video1.jpg
        # xy/video3&xz/video4 -> xy_video3.jpg
        match = PseudoDB._video_basename_regex.match(name)
        if not match:
            return ''
        basename = match.group(0)
        return PseudoDB._thumb_url_base + basename.replace('/', '_') + '.jpg'

    def _get_entries_from_cache(self):
        '''Retrieve entries and indexes stored in memcache.'''
        self.videos = []
        pages = memcache.get(PseudoDB._pages_cache_key)

        if pages is None:
            logging.debug('entries not in cache; rebuilding')
            self._get_entries_from_db()
            return

        self.pages = pages
        logging.debug('reading {} pages from cache'.format(pages))

        for i in range(pages):
            page = memcache.get(PseudoDB._cache_key_base+str(i))

            if page is None:
                # missing page in cache, recache
                logging.debug('missing entries page {} in cache; rebuilding'.format(i))
                self._get_entries_from_db()
                return

            # append videos from page
            self.videos += page

            logging.debug('read {} entries from page {} in cache'.format(len(page), i))

        # get tags
        self.tags_idx = memcache.get(PseudoDB._tags_cache_key)

        if self.tags_idx is None:
            # missing tags in cache, recache
            logging.debug('missing tags index in cache; rebuilding')
            self._get_entries_from_db()
            return
        else:
            logging.debug('read {} tags from cache'.format(len(self.tags_idx)))

    def _get_entries_from_db(self):
        '''Retrieve entries and indexes stored in the DB.'''
        self.videos = []
        self.pages = i = 0
        while True:
            entry_key = ndb.Key(CacheEntry, PseudoDB._cache_key_base+str(i))
            entry = entry_key.get()

            if not entry:
                break

            page = entry.blob

            # recache
            memcache.set(PseudoDB._cache_key_base+str(i), page)

            # append videos from page
            self.videos += page

            logging.debug('read {} entries from page {} in db'.format(len(page), i))

            # number of pages read, so we know if we have to delete any when
            # storing
            self.pages = i+1

            # if page is not full, this must be the last one
            if len(page) < PseudoDB._page_size:
                break
            i += 1

        logging.debug('read {} pages from db'.format(self.pages))

        # cache number of pages
        memcache.set(PseudoDB._pages_cache_key, self.pages)

        # get tags
        self.tags_idx = {}

        entry_key = ndb.Key(CacheEntry, PseudoDB._tags_cache_key)
        entry = entry_key.get()

        if entry:
            self.tags_idx = entry.blob
        memcache.set(PseudoDB._tags_cache_key, self.tags_idx)
        logging.debug('read {} tags from db'.format(len(self.tags_idx)))

    @staticmethod
    def dump():
        '''Return raw DB dump'''
        logging.debug('dumping db')

        video_pages = []
        i = 0
        while True:
            entry_key = ndb.Key(CacheEntry, PseudoDB._cache_key_base+str(i))
            entry = entry_key.get()
            page = entry.blob

            if page:
                video_pages.append(page)
                # if page is not full, this must be the last one
                if len(page) < PseudoDB._page_size:
                    break
            else:
                break
            i += 1

        # tags index
        tags_idx = {}
        entry_key = ndb.Key(CacheEntry, PseudoDB._tags_cache_key)
        entry = entry_key.get()
        if entry:
            tags_idx = entry.blob

        # sources index
        sources = {}
        entry_key = ndb.Key(CacheEntry, Sources._sources_cache_key)
        entry = entry_key.get()
        if entry:
            sources = entry.blob

        # views
        views = {}
        entry_key = ndb.Key(CacheEntry, Views._views_cache_key)
        entry = entry_key.get()
        if entry:
            views = entry.blob

        return {'video_pages': video_pages, 'tags_idx': tags_idx,
                'sources': sources, 'views': views}

    def store_entries(self):
        '''Stores all videos and tags in DB and cache.'''
        to_put = []
        to_delete = []

        videos = self.videos

        # in the special case of no videos (i.e. the DB is being cleared), we
        # have to make sure we store an empty first page
        if not videos:
            entry_key = ndb.Key(CacheEntry, PseudoDB._cache_key_base+'0')
            entry = CacheEntry(blob=[], key=entry_key)
            to_put.append(entry)
            memcache.set(PseudoDB._cache_key_base+'0', [])
            logging.debug('db is empty; storing 0 entries')

        # count new number of pages
        full_pages, rest = divmod(len(videos), PseudoDB._page_size)
        pages = full_pages
        if rest > 0:
            pages +=1

        i = 0
        while videos:
            # select the first entry_size videos
            page = videos[:PseudoDB._page_size]
            videos = videos[PseudoDB._page_size:]

            entry_key = ndb.Key(CacheEntry, PseudoDB._cache_key_base+str(i))
            entry = CacheEntry(blob=page, key=entry_key)
            to_put.append(entry)

            # store in memcache
            memcache.set(PseudoDB._cache_key_base+str(i), page)

            logging.debug('storing {} entries in page {}'.format(len(page), i))

            i += 1

        # delete extra pages if necessary
        if pages < self.pages:
            for j in range(pages, self.pages):
                entry_key = ndb.Key(CacheEntry, PseudoDB._cache_key_base+str(j))
                to_delete.append(entry_key)

                # delete from memcache
                memcache.delete(PseudoDB._cache_key_base+str(j))

                logging.debug('deleting leftover page {}'.format(j))
        self.pages = pages

        logging.debug('stored {} pages'.format(pages))
        memcache.set(PseudoDB._pages_cache_key, pages)

        tags_entry = self.store_tags_index(insert_in_db=False)
        to_put.append(tags_entry)
        logging.debug('stored {} tags'.format(len(self.tags_idx)))

        try:
            if to_put:
                ndb.put_multi(to_put)
            if to_delete:
                ndb.delete_multi(to_delete)
        except apiproxy_errors.OverQuotaError:
            logging.debug("couldn't write entries to db: over quota")

    def store_tags_index(self, insert_in_db=True):
        # store in memcache
        memcache.set(PseudoDB._tags_cache_key, self.tags_idx)

        # store tags index
        entry_key = ndb.Key(CacheEntry, PseudoDB._tags_cache_key)
        entry = CacheEntry(blob=self.tags_idx, key=entry_key)
        # when not inserting in DB, return entry so the caller can do it
        if insert_in_db:
            entry.put()
        else:
            return entry

    @staticmethod
    def get_filter_params(user_params, strict=True, defaults=None,
                          accepted_vars=None):
        '''Turn information extracted from the path and parameters passed by the
        user into parameters for querying the DB.'''

        # accept all known keys by default
        if not accepted_vars:
            accepted_vars = PseudoDB._accepted_vars

        # defaults
        params = {
            # sort from newest to oldest
            'order': 'descending',
            # sort by post time
            'sort_by': 'posted_at',
            # accept all posted_at times
            'since': None,
            'until': None,
            'return_type': 'list',
            # start at the first entry
            'offset': None,
            # don't limit output to a number of entries
            'limit': None,
            # don't filter by tags
            'tags': None,
            # join tags with OR when multiple tags are present
            'tags_op': 'or',
            # don't filter by entry name
            'names': None,
            # don't filter by source
            'source': None,
            # don't filter by host
            'hosts': None,
            # don't filter keys
            'keys': None,
            # expand entries
            'expand': True,
            # entries will not be filtered in any way
            '_filtered' : False,
            }
        if defaults:
            for key in defaults:
                if key in params:
                    params[key] = defaults[key]

        # check all variables
        if strict:
            for var in user_params:
                if not var in PseudoDB._accepted_vars:
                    raise ValueError(u'Invalid argument: {}; expected {}'.format(var,
                        PseudoDB._accepted_vars))

        # order
        if 'order' in accepted_vars and 'order' in user_params:
            order = user_params['order']
            if not order in ['asc', 'ascending', 'desc', 'descending', 'rand',
                             'random', 'randomize']:
                # raise error or ignore
                if strict:
                    raise ValueError(u'Invalid odering: "{}". Expected "ascending" '
                        u'or "descending"'.format(order))
            elif order.startswith('asc'):
                params['order'] = 'ascending'
            elif order.startswith('rand'):
                params['order'] = 'random'

        if 'sort_by' in accepted_vars and 'sort_by' in user_params:
            sort_by = user_params['sort_by']
            if not sort_by in ['posted_at', 'views']:
                # raise error or ignore
                if strict:
                    raise ValueError(u'Invalid sort by key: "{}". Expected '
                            u'"posted_at" or "views"'.format(sort_by))
            else:
                params['sort_by'] = sort_by

        # since, until
        if 'since' in accepted_vars and 'since' in user_params:
            since = user_params['since']
            if since:
                if strict:
                    # convert to datetime then back so strptime can raise ValueError if
                    # necessary
                    since = datetime.datetime.strptime(since, '%Y-%m-%d %H:%M:%S.%f')
                    params['since'] = since.strftime('%Y-%m-%d %H:%M:%S.%f')
                else:
                    params['since'] = since

        if 'until' in accepted_vars and 'until' in user_params:
            until = user_params['until']
            if until:
                if strict:
                    # convert to datetime then back so strptime can raise ValueError if
                    # necessary
                    until = datetime.datetime.strptime(until, '%Y-%m-%d %H:%M:%S.%f')
                    params['until'] = until.strftime('%Y-%m-%d %H:%M:%S.%f')
                else:
                    params['until'] = until

        # number of entries, offset
        for var in ['limit', 'offset']:
            if var in accepted_vars and var in user_params:
                try:
                    params[var] = int(user_params[var])
                except ValueError:
                    if strict:
                        raise

        # return type
        if 'type' in accepted_vars and 'type' in user_params:
            return_type = user_params['type']
            if not return_type in ['list', 'dict']:
                # raise error or ignore
                if strict:
                    raise ValueError(u'Invalid return type: "{}". Expected "list" '
                            'or "dict"'.format(return_type))
            elif return_type == 'dict':
                params['return_type'] = 'dict'

        # names
        if 'names' in accepted_vars and 'names' in user_params and user_params['names']:
            params['names'] = PseudoDB.split_names_list(user_params['names'])

        # source
        if 'source' in accepted_vars and 'source' in user_params and user_params['source']:
            params['source'] = user_params['source']

        # tags, hosts, keys
        for var in ['tags', 'hosts', 'keys']:
            if var in accepted_vars and var in user_params and user_params[var]:
                params[var] = user_params[var].split(PseudoDB._list_separator)

        # fix or drop bad tags
        if params['tags']:
            params['tags'] = DBUpdater.validate_tags(params['tags'])

        # operation for joining multiple tags
        if 'tags_op' in accepted_vars and 'tags_op' in user_params:
            op = user_params['tags_op']
            if op not in ['or', 'and']:
                # raise error or ignore
                if strict:
                    raise ValueError(
                        u'Invalid operation: "{}". Expected "or" or "and"'
                        u''.format(op))
            elif op == 'and':
                params['tags_op'] = 'and'

        # check keys
        if params['keys']:
            for i in range(len(params['keys'])):
                key = params['keys'][i]
                if not key in PseudoDB._valid_entry_keys:
                   # raise error or ignore
                   if strict:
                       raise ValueError(u'Invalid key: {}; expected {}'.format(key,
                           PseudoDB._valid_entry_keys))

        # check if any of the filtering parameters have been set
        for var in ['since', 'until', 'tags', 'names', 'source', 'hosts']:
            if params[var]:
                params['_filtered'] = True
                break

        return params

    @staticmethod
    def split_names_list(list_string):
        if not list_string:
            return []

        # try to split with all possible separators
        names = None
        for separator in PseudoDB._names_list_separators:
            if separator in list_string:
                names = list_string.split(separator)
                break
        # no separators present, return list with a single item
        if not names:
            names = [list_string]
        return names

    @staticmethod
    def get_clean_params(params, accepted_vars, default=None):
        if default is None:
            default = {}

        clean_params = {}
        for var in accepted_vars:
            # only pass along value if it isn't the default
            if ((var in params and params[var])
                and not (var in default and params[var] == default[var])):

                # cast int params
                if var in ['limit', 'offset']:
                    params[var] = str(params[var])

                # join lists
                value = params[var]
                if isinstance(value, list):
                    if var == 'names':
                        value = PseudoDB.names_list_separator.join(value)
                    else:
                        value = PseudoDB._list_separator.join(value)

                clean_params[var] = value

        return clean_params

    def add_views(self, entries):
        '''Adds view count for every entry'''
        for entry in entries:
            entry['views'] = self._views.get_views(entry['name'])

        if self._views.increment_views:
            self._views.save()

    def fill_out_missing_views(self):
        '''Saves 0 views for all videos without views set.'''
        modified = False
        for video in self.videos:
            # tolerate both a list or a dict of videos
            if isinstance(video, dict):
                video = video['name']

            if not self._views.has_views(video):
                self._views.set_views(video, 0)
                modified = True

        if modified:
            self._views.save()

class UpdateThumbs:
    _task_url = '/blog/json/admin/update_thumbs'

    @staticmethod
    def post(params):
        if not (params and 'names' in params and params['names']):
            # should never happen
            webapp2.abort(500)

        payload = {'key': thumbs.secret_key}
        payload['names'] = Jinja.fix_urllib_str(params['names'])

        # we can't use urlfetch.fetch()'s 'payload' named argument if method is
        # GET, so we must build the url ourselves
        url = '{}?{}'.format(thumbs.url, urllib.urlencode(payload))

        result = urlfetch.fetch(url, method=urlfetch.GET, deadline=60)
        logging.debug('updating thumbs: got status {}'.format(result.status_code))

    @staticmethod
    def add_task(names):
        '''Add task for updating thumbnails for entries.'''
        if len(names) <= 0:
            return

        # FIXME: switch to new separator
        data = {'names': ';'.join(names)}
        taskqueue.add(url=UpdateThumbs._task_url, params=data)


class DBUpdater:
    _basic_entry_keys = ['name', 'posted_at', 'source_url', 'source_text', 'tags']
    _required_entry_keys = ['name', 'posted_at', 'source_text']
    _url_cleaner = re.compile('^(?:http://anonym\.to/\?)?(?P<url>.*)$')
    _youtube_id_regex = re.compile(r'^[0-9A-Za-z_-]{11}$')
    _youtube_url = 'https://www.youtube.com/watch?v='
    _valid_url_regex = re.compile(r'^[a-z]+://')
    _valid_tag_regex = re.compile('^[0-9a-z_-]+$')

    allowed_hosts = ['gc', 'kd', 'lv', 'ko']

    def __init__(self):
        self.db = PseudoDB()

    def replace_entries(self, entries=None):
        '''Replace all entries with the supplied dict.'''
        entries = PseudoDB.dict2list(entries)
        videos = []
        tags_idx = {}
        views = Views(clear=True)

        # validate entries
        for entry in entries:
            # this must be done before validation, or the inforrmation will be lost
            entry_views = None
            if 'views' in entry:
                entry_views = entry['views']

            # will raise error if something is wrong
            entry = DBUpdater.validate_entry(entry)
            videos.append(entry)
            DBUpdater.index_tags(entry, tags_idx)
            # will raise error if something is wrong
            views.set_views(entry['name'], entry_views)

        # sort videos before storing
        self.db.videos = PseudoDB.sort_entries(videos, force_list=True)
        self.db.tags_idx = tags_idx

        # store modified entries
        self.db.store_entries()

        # store views
        views.save()

        # build and store sources index
        sources = Sources()
        sources.build_sources_index(entries=self.db.videos)
        sources.save()

        # flush cache
        memcache.flush_all()

    def add_entries(self, videos=None, source_url=None, source_text=None,
            tags=None, posted_at=None):
        '''Adds new entries to the DB.'''
        # reverse videos so they'll show up in the expected order
        videos.reverse()

        videos_dict = PseudoDB.list2ordered_dict(self.db.videos)

        if posted_at:
            try:
                posted_at = datetime.datetime.strptime(posted_at, '%Y-%m-%d %H:%M:%S')
            except ValueError as e:
                raise DBError('Error converting posted_at time:\n{}'.format(e))

        entries_dict = {}
        for video in videos:
            # skip empty lines
            if not video:
                continue

            entry = {
                    'name': video,
                    'source_url': source_url,
                    'source_text': source_text,
                    'tags': tags,
                    }

            if posted_at:
                entry['posted_at'] = posted_at.strftime('%Y-%m-%d %H:%M:%S.%f')
                # space out entries by 1 ms so they'll retain their ordering
                posted_at += datetime.timedelta(seconds=0.001)
            else:
                entry['posted_at'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')

            # will raise error if something is wrong
            entry = DBUpdater.validate_entry(entry)

            # check if the host is allowed in the DB
            host = entry['name'][:2]
            if not host in self.allowed_hosts:
                raise DBError('sorry, but the only hosts allowed for blog entries '
                              'at the moment are:\n{}\n(contact loopvid@gmx.co.uk '
                              'if you need a host added)'.format(
                                  ', '.join(sorted(self.allowed_hosts))))

            if entry['name'] in videos_dict:
                raise DBError('video already in DB: {}'.format(entry['name']))

            # skip duplicates
            if entry['name'] in entries_dict:
                continue

            entries_dict[entry['name']] = entry
            self.db.videos.append(entry)
            DBUpdater.index_tags(entry, self.db.tags_idx)
            self.db.sources.index_entry(entry)

        if len(entries_dict) == 0:
            raise DBError('nothing to add')
        else:
            logging.info(u'adding entries: {}'.format(list(entries_dict.keys())))

        # sort videos before storing
        self.db.videos = PseudoDB.sort_entries(self.db.videos, force_list=True)

        # store modified entries
        self.db.store_entries()
        self.db.sources.save()

        # update thumbs
        UpdateThumbs.add_task(entries_dict.keys())

        # clear cached entries
        cache_keys = ['video:'+name for name in entries_dict]
        memcache.delete_multi(cache_keys)

        # set views for new entries
        self.db.fill_out_missing_views()

        # return list of new videos
        return entries_dict.keys()

    def update_entries(self, db_action=None, videos=None, source_url=None,
            source_text=None, tags=None, new_names=None):
        '''Updates entries according to the requested action.'''

        if not db_action in ['add_tags', 'remove_tags', 'change_source',
                'update_thumbs', 'remove_videos', 'rename_videos']:
            raise DBError('Unknown action: {}'.format(db_action))

        videos_dict = PseudoDB.list2ordered_dict(self.db.videos)

        # if adding tags, we must validate them beforehand, because we'll be
        # indexing them before validating the whole entry
        if db_action == 'add_tags':
            tags = DBUpdater.validate_tags(tags, fail=True)

        done = {}
        to_update_thumb = []
        for name in videos:
            # skip empty lines
            if not name:
                continue

            # skip duplicates
            if name in done:
                continue
            done[name] = None

            if not name in videos_dict:
                raise DBError('video not found in DB: {}'.format(name))

            entry = videos_dict[name]

            if db_action == 'change_source':
                if entry['source_text'] != source_text or entry['source_url'] != source_url:
                    logging.info(u'changing source for {}: from ({}, {}) to ({}, {})'.format(
                        name,
                        entry['source_text'], entry['source_url'],
                        source_text, source_url
                        ))

                    self.db.sources.unindex_entry(entry)

                    entry['source_url'] = source_url
                    entry['source_text'] = source_text

            elif db_action == 'add_tags':
                for tag in tags:
                    if tag not in entry['tags']:
                        logging.info(u'adding tag for {}: {}'.format(name, tag))
                        entry['tags'].append(tag)
                DBUpdater.index_tags(entry, self.db.tags_idx)

            elif db_action == 'remove_tags':
                for tag in tags:
                    if tag in entry['tags']:
                        logging.info(u'removing tag for {}: {}'.format(name, tag))
                        entry['tags'].remove(tag)
                DBUpdater.unindex_tags(name, tags, self.db.tags_idx)

            elif db_action == 'remove_videos':
                logging.info(u'removing video {}'.format(json.dumps(entry)))
                del videos_dict[name]
                DBUpdater.unindex_tags(name, entry['tags'], self.db.tags_idx)
                self.db.sources.unindex_entry(entry)
                self.db._views.delete_video(name)

            elif db_action == 'update_thumbs':
                logging.info(u'updating thumb for {}'.format(name))
                to_update_thumb.append(name)

            elif db_action == 'rename_videos':
                try:
                    new_name = new_names.pop(0)
                    # validate new name ahead of time in order to shorten it if
                    # necessary
                    new_name = DBUpdater.validate_value('name', new_name)

                    # check host
                    host = new_name[:2]
                    if not host in DBUpdater.allowed_hosts:
                        raise DBError('new entries from host {}/ are not '
                                      'allowed'.format(host))

                    logging.info(u'renaming video {} into {}'.format(name, new_name))
                except IndexError:
                    raise DBError('not enough new names')
                if new_name in videos_dict:
                    raise DBError('duplicated name or already in DB: {}'.format(new_name))

                entry = videos_dict[name]
                self.db.sources.unindex_entry(entry)

                entry['name'] = new_name
                self.db._views.rename_video(name, new_name)

                # remove entry
                del videos_dict[name]
                DBUpdater.unindex_tags(name, entry['tags'], self.db.tags_idx)

                # add back under new name
                videos_dict[new_name] = entry
                DBUpdater.index_tags(entry, self.db.tags_idx)

                # update name for validation below
                name = new_name

                # mark for generating new thumbnail
                to_update_thumb.append(new_name)

            # validate updated entry (except for removal, in which case there's
            # nothing to validate)
            if db_action != 'remove_videos':
                # will raise error if something is wrong
                videos_dict[name] = DBUpdater.validate_entry(entry)

                # index source if changed
                if db_action in ['change_source', 'rename_videos']:
                    self.db.sources.index_entry(entry)

        # update thumbs
        if to_update_thumb:
            UpdateThumbs.add_task(to_update_thumb)

        # if only updating thumbs, there's nothing else to do
        if db_action == 'update_thumbs':
            return

        # sort videos before storing (since this is an OrderedDict and we've
        # reassigned entries on validation)
        if db_action != 'remove_videos':
            self.db.videos = PseudoDB.sort_entries(videos_dict, force_list=True)
        else:
            self.db.videos = PseudoDB.dict2list(videos_dict)

        # store modified entries
        self.db.store_entries()
        if db_action in ['rename_videos', 'remove_videos']:
            self.db._views.save()
            self.db.sources.save()
        elif db_action == 'change_source':
            self.db.sources.save()

        # clear cached entries
        cache_keys = ['video:'+name for name in done]
        memcache.delete_multi(cache_keys)

    def low_level_rename(self, old_name, new_name):
        '''Rename video at a low level, without any checks. Use this function
        only to fix a mistakenly added bad name.'''

        logging.info(u'low level renaming {} into {}'.format(old_name, new_name))

        # remember changes to report them back to the user
        fixed = []

        # fix entry itself
        videos_dict = PseudoDB.list2ordered_dict(self.db.videos)
        if old_name in videos_dict:
            entry = videos_dict[old_name]
            entry['name'] = new_name
            del videos_dict[old_name]
            videos_dict[new_name] = entry
            fixed.append('entry')

        # fix tags index
        for tag_videos in self.db.tags_idx.values():
            if old_name in tag_videos:
                tag_videos.remove(old_name)
                tag_videos.append(new_name)
                if not 'tags' in fixed:
                    fixed.append('tags')

        # fix sources index
        for source_videos in self.db.sources._index.values():
            if isinstance(source_videos, list) and old_name in source_videos:
                source_videos.remove(old_name)
                source_videos.append(new_name)
                fixed.append('source')
                break

        # fix views
        # store views for the new name so we can add to those from the old name
        extra_views = self.db._views.get_views(new_name)
        if old_name in self.db._views._views_dict:
            self.db._views.rename_video(old_name, new_name)
            # only add extra views if there are any
            if extra_views > 0:
                self.db._views.increment_video_views(new_name, amount=extra_views,
                                                     auto_save=False)
            fixed.append('views')

        # save changes
        self.db.store_entries()
        self.db.sources.save()
        self.db._views.save()

        return {'success': True, 'fixed': fixed}

    _host_regex = re.compile(r'^[a-z]{2}/?$')
    def replace_host(self, old_host, new_host):
        '''Rename all videos from a certain host to a new host.

        For example, replacing ab/ with cd/ will cause all videos named ab/* to
        be renamed to cd/*.
        '''

        logging.info(u'low level replacing host {} into {}'.format(old_host, new_host))

        # validate hosts
        for host in [old_host, new_host]:
            if not self._host_regex.match(host):
                return {'success': False, 'error': 'Invalid host {}'.format(host)}

        # make user hosts end in a slash
        if not old_host.endswith('/'):
            old_host = old_host+'/'
        if not new_host.endswith('/'):
            new_host = new_host+'/'

        # generate regex for replacing the old host with the new one
        old_host_regex = re.compile('^{}'.format(old_host))

        # fix all entries
        fixed = 0
        for entry in self.db.videos:
            if entry['name'].startswith(old_host):
                old_name = entry['name']
                new_name = old_host_regex.sub(new_host, old_name)

                # remove from indexes, where applicaple
                self.db.sources.unindex_entry(entry)
                DBUpdater.unindex_tags(old_name, entry['tags'], self.db.tags_idx)

                # fix name
                entry['name'] = new_name

                # re-index
                self.db.sources.index_entry(entry)
                DBUpdater.index_tags(entry, self.db.tags_idx)
                self.db._views.rename_video(old_name, new_name)

                fixed += 1

        # save changes
        self.db.store_entries()
        self.db.sources.save()
        self.db._views.save()

        return {'success': True, 'fixed': fixed}

    @staticmethod
    def index_tags(entry, tags_idx):
        '''Adds entry's tags to tags index.'''
        if not ('tags' in entry and entry['tags']):
            return

        for tag in entry['tags']:
            if not tag in tags_idx:
                tags_idx[tag] = []
            if not entry['name'] in tags_idx[tag]:
                tags_idx[tag].append(entry['name'])

    @staticmethod
    def unindex_tags(video, tags, tags_idx):
        for tag in tags:
            if not tag in tags_idx:
                continue
            if video in tags_idx[tag]:
                tags_idx[tag].remove(video)
            # if tag is now empty, delete it
            if not tags_idx[tag]:
                del tags_idx[tag]

    @staticmethod
    def validate_entry(entry):
        '''Validates a single entry and raises DBError if anything is wrong.'''
        if not isinstance(entry, dict):
            raise DBError('invalid type for entry (expected dict): {}: {}'.format(type(entry), entry))

        validated_entry = {
                'name': '',
                'posted_at' : '',
                'source_url': '',
                'source_text': '',
                'tags': [],
                }

        for key in validated_entry:
            # will raise error if there's a problem
            validated_entry[key] = DBUpdater.validate_value(key, entry[key])

        for key in DBUpdater._required_entry_keys:
            if not validated_entry[key]:
                raise DBError('missing value for required key: {}'.format(key))

        return validated_entry

    @staticmethod
    def validate_value(key, value):
        '''Validate the value of a single basic key from an entry dict.'''
        # check key
        if not key in DBUpdater._basic_entry_keys+['tag']:
            raise DBError('invalid key: {}'.format(key))

        # recurse for tags list
        if key == 'tags' and isinstance(value, list):
            # using a set to remove duplicates
            tags_set = set([DBUpdater.validate_value('tag', item) for item in value])
            return list(tags_set)

        # replace None with empty string
        if value is None:
            value = ''

        # check type
        if not (isinstance(value, str) or isinstance(value, unicode)):
            raise DBError('invalid type for key (expected str): {}: {}'.format(key, type(value)))

        if key == 'name':
            expanded_urls = Shortener.expand_urls(value)
            shortened_urls = Shortener.shorten_urls(expanded_urls)
            if not shortened_urls:
                raise DBError(u'invalid name for entry: {}\n(must be a shortened url)'.format(value))
            value = shortened_urls
        elif key == 'posted_at':
            try:
                posted_at = datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S.%f')
            except ValueError:
                raise DBError(u'invalid post time: {}'.format(value))
        elif key == 'source_url':
            if not value:
                return ''

            # clean url
            match = DBUpdater._url_cleaner.match(value)
            if match:
                value = match.group('url')
            # expand youtube
            if DBUpdater._youtube_id_regex.match(value):
                value = DBUpdater._youtube_url + value
            # check url
            if not DBUpdater._valid_url_regex.match(value):
                raise DBError(u'invalid url: {}'.format(value))
        elif key == 'tag':
            value = value.strip().lower().replace(' ', '_')
            if not DBUpdater._valid_tag_regex.match(value):
                raise DBError(u'invalid tag name: {}'.format(value))

        return value

    @staticmethod
    def validate_tags(tags, fail=False):
        '''Returns list of fixed tags, without including malformed ones. If
        fail=True, raise error on malformed tags.'''
        good_tags = []
        for tag in tags:
            tag = tag.strip().lower().replace(' ', '_')
            if DBUpdater._valid_tag_regex.match(tag):
                good_tags.append(tag)
            elif fail:
                raise DBError('invalid tag name: {}'.format(tag))
        return good_tags
