# -*- coding: utf-8 -*-
from __future__ import print_function
import os
import argparse
import json
import string
import re
import glob
import emoji

import numpy as np
import pandas as pd

from scipy.sparse import csc_matrix
from sklearn.externals import joblib

from alcohol_surveillance_utils import alcohol_keywords, other_keywords
from alcohol_surveillance_utils import alcohol_emoji_dict

from keyword_matching_utils import make_aho_keyword_automaton
from keyword_matching_utils import find_keywords

from filter_alcohol_posts import InstagramAlcoholPostFilter

from tqdm import tqdm

import warnings


class InstagramAlcoholPostClassifier:

    def __init__(self):

        # classification model
        self.clf_model = None

        # create aho corasick keyword automaton
        self.keyword_list = alcohol_keywords + other_keywords
        self.kw_automaton = make_aho_keyword_automaton(self.keyword_list)
        self.kw_count_dict_init = {w.strip(): 0 for w in self.keyword_list}

        # set of printable characters
        self.printable = set(string.printable)

        # set of punctuations
        self.punctuation = set(string.punctuation)

        # regex for prices -- https://regex101.com/r/dE9dH1/3
        self.reg_price = re.compile(
            r"(?:dollar[s]?|\$)\s*(\d+)|(\d+)\s*(?:dollar[s]?|\$)")

        # regex for emails -- https://gist.github.com/dideler/5219706
        self.reg_email = re.compile(
            r"([a-z0-9!#$%&'*+\/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+\/=?^_`"
            "{|}~-]+)*(@|\sat\s)(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(\.|"
            "\sdot\s))+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)"
        )

        # regex for web urls -- http://ryancompton.net/2015/02/16/url-extraction-in-python/   # noqa
        self.reg_web_url = re.compile(r"""(?i)\b((?:https?:(?:/{1,3}|[a-z0-9%])|[a-z0-9.\-]+[.](?:com|net|org|edu|gov|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)/)(?:[^\s()<>{}\[\]]+|\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\))+(?:\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\)|[^\s`!()\[\]{};:'".,<>?«»“”‘’])|(?:(?<!@)[a-z0-9]+(?:[.\-][a-z0-9]+)*[.](?:com|net|org|edu|gov|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)\b/?(?!@)))""")  # noqa

        self.reg_com_url = re.compile(r"""(?i)\b((?:https?:(?:/{1,3}|[a-z0-9%])|[a-z0-9.\-]+[.](?:com)\b/?(?!@)))""")  # noqa

        # regex for phone numbers -- https://stackoverflow.com/a/3868861
        self.reg_phone = re.compile(r".*?(\(?\d{3}\)? ?[\.-]? ?\d{3} ?[\.-]? ?\d{4}).*?", re.S)  # noqa

        # regex for discount
        self.reg_discount = re.compile(r'\d+%')

        # iap filter
        self.iap_filter = InstagramAlcoholPostFilter()

    def set_model(self, clf_model):

        self.clf_model = clf_model

    def load_model_from_file(self, clf_model_file):

        self.clf_model = joblib.load(open(clf_model_file, 'rb'))

    def compute_text_features(self, cap_list, show_progress=False):
        """
        Computes features for classification from the caption of the post
        """

        assert isinstance(cap_list, list)

        features = {}

        if show_progress:
            cap_seq = tqdm(range(len(cap_list)))
        else:
            cap_seq = range(len(cap_list))

        for i in cap_seq:

            cap = cap_list[i]

            if isinstance(cap, str):
                cap_unicode = cap.decode('utf-8')
            elif isinstance(cap, unicode):
                cap_unicode = cap
            else:
                print(i, cap)
                raise ValueError('caption encoding must be ascii nor unicode')

            # convert to lower-case ascii
            cap_ascii = filter(lambda x: x in self.printable, cap).lower()
            cap_ascii = cap_ascii.encode('ascii', 'ignore')

            # drop puncutations and tokenize by white space
            cap_no_punc = ''.join(
                [x if x not in self.punctuation else ' ' for x in cap_ascii])
            cap_no_punc_words = cap_no_punc.split()

            # check for mentions of prices
            features.setdefault('has_price', []).append(
                self.reg_price.search(cap_ascii) is not None)

            # check for mentions of discounts
            features.setdefault('has_discount', []).append(
                self.reg_discount.search(cap_ascii) is not None)

            # check for mentions of email address
            features.setdefault('has_email_address', []).append(
                self.reg_email.search(cap_ascii) is not None)

            # check for mentions of websites
            features.setdefault('has_web_url', []).append(
                self.reg_web_url.search(cap_ascii) is not None)

            features.setdefault('has_com_url', []).append(
                self.reg_com_url.search(cap_ascii) is not None)

            # check presence of phone number
            features.setdefault('has_phone_number', []).append(
                self.reg_phone.search(cap_ascii) is not None)

            # compute count of exclamation mark
            features.setdefault('count_exclamation', []).append(
                cap_ascii.count('!'))

            # count emojis
            emoji_list = [c for c in cap_unicode if c in emoji.UNICODE_EMOJI]
            features.setdefault('count_emojis', []).append(len(emoji_list))

            # count alcohol emojis
            alcohol_emoji_count_dict = {v: 0 for k, v in
                                        alcohol_emoji_dict.iteritems()}

            for e in emoji_list:
                if e in alcohol_emoji_dict:
                    alcohol_emoji_count_dict[alcohol_emoji_dict[e]] += 1

            for k, v in alcohol_emoji_count_dict.iteritems():
                features.setdefault('count_emoji_%s' % k, []).append(v)

                # compute counts of keywords
            kw_found = find_keywords(cap_no_punc, self.kw_automaton)

            kw_count_dict = self.kw_count_dict_init.copy()
            for w in kw_found:
                kw_count_dict[w.strip()] += 1

            for k, v in kw_count_dict.iteritems():
                features.setdefault('count_%s' % k, []).append(v)

            # compute length of caption
            features.setdefault('num_characters', []).append(len(cap))
            features.setdefault('num_words', []).append(len(cap_no_punc_words))

        for k, v in features.iteritems():
            assert len(v) == len(
                cap_list), "Invalid number of samples for feature - %s, %d" % (
            k, len(v))

        features = pd.DataFrame.from_dict(features)

        return features

    def classify_post(self, post):
        """
        Detects if a given post involves alcohol consumption
        """

        # Discard irrelevant posts upfront
        if not self.iap_filter.filter_post(post):
            return False

        # Get caption
        caption = post.get('caption', {}).get('text')

        if not caption:
            return False

        # Compute features
        features = self.compute_text_features([caption])

        feature_vec = csc_matrix(
            np.array(features.values, dtype='float').reshape(1, -1))

        # apply classification model
        with warnings.catch_warnings():
            warnings.filterwarnings("ignore", category=DeprecationWarning)
            pred = self.clf_model.predict(feature_vec)[0]

        return pred

    def classify_posts(self, input_json_file, output_json_file=None,
                       limit=None, state=None):
        """
        Finds all the posts in the input json file that involve alcohol
        consumption using a machine learning model
        """

        assert self.clf_model, \
            "Classification model not set. " \
            "Use set_model() or load_model() to set the classification model "

        print('\n---------------------------------------')

        # read input json file
        print('\n>>> Loading input file - {}'.format(input_json_file))

        with open(input_json_file) as f:
            text = f.read()
            input_json = json.loads(text)

        if not text:
            raise Exception('Input file missing, empty, or unreadable ({})'
                            .format(input_json_file))

        print('Number of input posts: {}'.format(len(input_json)))

        # filter posts
        print('\n>>> Running filter')

        output_json = []

        if limit is not None:
            input_json = input_json[:limit]

        for i in tqdm(range(len(input_json))):

            post = input_json[i]

            pred = self.classify_post(post)

            if pred:
                if state:
                    post['state'] = state
                output_json.append(post)

        print('Number of positive posts: {}'.format(len(output_json)))

        # write to output file
        if output_json_file is not None:

            print('\n>>> Writing output file - {}'.format(output_json_file))

            with open(output_json_file, 'w') as f:
                f.write(json.dumps(output_json))
                f.write('\n')

        else:

            return output_json


if __name__ == '__main__':

    parser = argparse.ArgumentParser()

    parser.add_argument('-if', '--input_filename',
                        help='Input json file containing instagram posts')

    parser.add_argument('-id', '--input_dir',
                        help='Input directory containing json files with '
                             'instagram posts')

    parser.add_argument('-m', '--ml_model_file', required=True,
                        help='Path to the machine learning model file')

    parser.add_argument('-of', '--output_filename',
                        help='Output json file containing the alcohol posts')

    parser.add_argument('-od', '--output_dir',
                        help='Output directory for writing json files with '
                             'alcohol posts')

    parser.add_argument('-l', '--limit', type=int,
                        help='Limit number of items processed')

    parser.add_argument('-s', '--state', help='Add state field to data')

    options = parser.parse_args()

    print(options)

    # check args
    assert options.input_filename or options.input_dir, \
        "Either input_filename of input_dir must be provided"
    assert not (options.input_filename and options.input_dir), \
        "Only one of input_filename of input_dir must be provided"

    # filter posts
    if options.input_dir:

        assert options.output_dir, 'Output directory was not provided'

        # Create output dir if it doesnt exist
        if not os.path.isdir(options.output_dir):
            os.makedirs(options.output_dir)
        else:
            Warning('Output directory already exists!')

        # Look for json files in input directory
        print('>>> Found the following json files in the input directory:')

        json_file_list = sorted(glob.glob(
            os.path.join(options.input_dir, '*.json')))

        for f in json_file_list:
            print(f)

        # create classification filter
        iap_filter = InstagramAlcoholPostClassifier()
        iap_filter.load_model_from_file(options.ml_model_file)

        # Run the filter
        for input_json_file in json_file_list:

            output_json_file = os.path.join(
                options.output_dir,
                os.path.basename(input_json_file)
            )

            iap_filter.classify_posts(
                input_json_file,
                output_json_file=output_json_file,
                limit=options.limit
            )

    if options.input_filename:

        assert options.output_filename, 'Output filename was not provided'

        # create classification filter
        iap_filter = InstagramAlcoholPostClassifier(options.ml_model_file)

        # Run the filter
        iap_filter.classify_posts(
            options.input_filename,
            output_json_file=options.output_filename,
            limit=options.limit,
            state=options.state
        )
