import json
import logging
import os
import random
import time
import types
import urllib.error
import urllib.parse
import http.client

from concurrent.futures import ThreadPoolExecutor

import htmlmin
import requests
import wikipediaapi
from bs4 import BeautifulSoup
from wikidata.client import Client

from descriptions.exceptions import GettingError, ParseError

"""
This script downloads Wikipedia pages for different languages.
"""
log = logging.getLogger(__name__)

WORKERS = 80
REQUEST_ATTEMPTS = 8
ATTEMPTS_PAUSE_SECONDS = 4.0

HEADERS = {f"h{x}" for x in range(1, 7)}
BAD_SECTIONS = {
    "en": [
        "External links",
        "Sources",
        "See also",
        "Bibliography",
        "Further reading",
        "References",
    ],
    "de": [
        "Einzelnachweise",
        "Weblinks",
        "Literatur",
        "Siehe auch",
        "Anmerkungen",
        "Anmerkungen und Einzelnachweise",
        "Filme",
        "Einzelbelege",
    ],
    "fr": [
        "Bibliographie",
        "Lien externe",
        "Voir aussi",
        "Liens externes",
        "Références",
        "Notes et références",
        "Articles connexes",
    ],
    "es": ["Vínculos de interés", "Véase también", "Enlaces externos", "Referencias"],
    "ru": ["Литература", "Ссылки", "См. также", "Библиография", "Примечания"],
    "tr": ["Kaynakça", "Ayrıca bakınız", "Dış bağlantılar", "Notlar", "Dipnot"],
}


def try_get(obj, prop, *args, **kwargs):
    attempts = REQUEST_ATTEMPTS
    while attempts != 0:
        try:
            attr = getattr(obj, prop)
            is_method = isinstance(attr, types.MethodType)
            return attr(*args, **kwargs) if is_method else attr
        except (
            requests.exceptions.ConnectionError,
            requests.exceptions.ReadTimeout,
            json.decoder.JSONDecodeError,
            http.client.HTTPException,
        ) as e:
            log.debug(e)
        except urllib.error.HTTPError as e:
            if e.code == 404:
                raise GettingError(f"Page not found {e.msg}")
        except KeyError:
            raise GettingError(f"Getting {prop} field failed. {prop} not found.")
        except urllib.error.URLError:
            raise GettingError(f"URLError: {obj}, {prop}, {args}, {kwargs}")

        time.sleep(random.uniform(0.0, ATTEMPTS_PAUSE_SECONDS))
        attempts -= 1

    raise GettingError(f"Getting {prop} field failed")


def read_popularity(path):
    """
    :param path: a path of popularity file. A file contains '<id>,<rank>' rows.
    :return: a set of popularity object ids
    """
    ids = set()
    for line in open(path):
        try:
            ident = int(line.split(",", maxsplit=1)[0])
        except (AttributeError, IndexError):
            continue
        ids.add(ident)
    return ids


def should_download_page(popularity_set):
    def wrapped(ident):
        return popularity_set is None or ident in popularity_set

    return wrapped


def remove_bad_sections(soup, lang):
    if lang not in BAD_SECTIONS:
        return soup
    it = iter(soup.find_all())
    current = next(it, None)
    current_header_level = None
    while current is not None:
        if current.name in HEADERS and current.text.strip() in BAD_SECTIONS[lang]:
            current_header_level = current.name
            current.extract()
            current = next(it, None)
            while current is not None:
                if current.name == current_header_level:
                    break
                current.extract()
                current = next(it, None)
        else:
            current = next(it, None)
    return soup


def beautify_page(html, lang):
    soup = BeautifulSoup(html, "html.parser")
    for x in soup.find_all():
        if len(x.text.strip()) == 0:
            x.extract()
    soup = remove_bad_sections(soup, lang)
    html = str(soup.prettify())
    html = htmlmin.minify(html, remove_empty_space=True)
    return html


def need_lang(lang, langs):
    return lang in langs if langs else True


def get_page_info(url):
    url = urllib.parse.unquote(url)
    parsed = urllib.parse.urlparse(url)
    try:
        lang = parsed.netloc.split(".", maxsplit=1)[0]
    except (AttributeError, IndexError):
        raise ParseError(f"{parsed.netloc} is incorrect.")
    try:
        page_name = parsed.path.rsplit("/", maxsplit=1)[-1]
    except (AttributeError, IndexError):
        raise ParseError(f"{parsed.path} is incorrect.")
    return lang, page_name


def get_wiki_page(lang, page_name):
    wiki = wikipediaapi.Wikipedia(
        language=lang, extract_format=wikipediaapi.ExtractFormat.HTML
    )
    return wiki.page(page_name)


def download(directory, url):
    try:
        lang, page_name = get_page_info(url)
    except ParseError:
        log.exception(f"Parsing failed. {url} is incorrect.")
        return None

    path = os.path.join(directory, f"{lang}.html")
    if os.path.exists(path):
        log.debug(f"{path} already exists.")
        return None

    page = get_wiki_page(lang, page_name)
    try:
        text = try_get(page, "text")
    except GettingError as e:
        log.exception(f"Error: page {page_name} is not downloaded for lang {lang} and url {url} ({e}).")
        return None

    page_size = len(text)
    if page_size > 0:
        os.makedirs(directory, exist_ok=True)
        text = beautify_page(text, lang)
        log.info(f"Save to {path} {lang} {page_name} {page_size}.")
        with open(path, "w") as file:
            file.write(text)
    else:
        log.warning(f"Page {url} is empty. It has not been saved.")

    return text


def get_wiki_langs(url):
    lang, page_name = get_page_info(url)
    page = get_wiki_page(lang, page_name)

    curr_lang = [(lang, url)]
    try:
        langlinks = try_get(page, "langlinks")
        return (
            list(zip(langlinks.keys(), [link.fullurl for link in langlinks.values()]))
            + curr_lang
        )
    except GettingError as e:
        log.exception(f"Error: no languages for page {page_name} with url {url} ({e}).")
        return curr_lang


def download_all_from_wikipedia(path, url, langs):
    try:
        available_langs = get_wiki_langs(url)
    except ParseError:
        log.exception("Parsing failed. {url} is incorrect.")
        return
    available_langs = filter(lambda x: need_lang(x[0], langs), available_langs)
    for lang in available_langs:
        download(path, lang[1])


def wikipedia_worker(output_dir, checker, langs):
    def wrapped(line):
        if not line.strip():
            return
        try:
            # First param is mwm_path, which added this line entry.
            _, ident, url = line.split("\t")
            ident = int(ident)
            if not checker(ident):
                return
            url = url.strip()
        except (AttributeError, ValueError):
            log.exception(f"{line} is incorrect.")
            return
        parsed = urllib.parse.urlparse(url)
        path = os.path.join(output_dir, parsed.netloc, parsed.path[1:])
        download_all_from_wikipedia(path, url, langs)

    return wrapped


def download_from_wikipedia_tags(input_file, output_dir, langs, checker):
    with open(input_file) as file:
        _ = file.readline() # skip header
        with ThreadPoolExecutor(WORKERS) as pool:
            pool.map(wikipedia_worker(output_dir, checker, langs), file)


def get_wikidata_urls(entity, langs):
    try:
        keys = entity.data["sitelinks"].keys()
    except (KeyError, AttributeError):
        log.exception(f"Sitelinks not found for {entity.id}.")
        return None
    return [
        entity.data["sitelinks"][k]["url"]
        for k in keys
        if any([k.startswith(lang) for lang in langs])
    ]


def wikidata_worker(output_dir, checker, langs):
    def wrapped(line):
        if not line.strip():
            return
        try:
            ident, wikidata_id = line.split("\t")
            ident = int(ident)
            wikidata_id = wikidata_id.strip()
            if not checker(ident):
                return
        except (AttributeError, ValueError):
            log.exception(f"{line} is incorrect.")
            return
        client = Client()
        try:
            entity = try_get(client, "get", wikidata_id, load=True)
        except GettingError:
            log.exception(f"Error: page is not downloaded {wikidata_id}.")
            return
        urls = get_wikidata_urls(entity, langs)
        if not urls:
            return
        path = os.path.join(output_dir, wikidata_id)
        for url in urls:
            download(path, url)

    return wrapped


def download_from_wikidata_tags(input_file, output_dir, langs, checker):
    wikidata_output_dir = os.path.join(output_dir, "wikidata")
    os.makedirs(wikidata_output_dir, exist_ok=True)
    with open(input_file) as file:
        with ThreadPoolExecutor(WORKERS) as pool:
            pool.map(wikidata_worker(wikidata_output_dir, checker, langs), file)


def check_and_get_checker(popularity_file):
    popularity_set = None
    if popularity_file is None:
        log.warning(f"Popularity file not set.")
    elif os.path.exists(popularity_file):
        popularity_set = read_popularity(popularity_file)
        log.info(f"Popularity set size: {len(popularity_set)}.")
    else:
        log.error(f"Popularity file ({popularity_file}) not found.")
    return should_download_page(popularity_set)