# -*- coding: utf-8 -*-

u"""23 Nov 2016
Michał Marcińczuk <michal.marcinczuk_at_pwr.edu.pl,marcinczuk_at_gmail.com>
Michał Krautforst
Wrocław University of Science and Technology

The script splits the list of wikilinks into several groups 
according to their morphological and ortographical analysis.

The list of wikilinks is seperated into the following files:
A) wikilinks-difflen.txt — forms have different number of tokens,
B) wikilinks-samelen*    — forms have the same number of tokens,
 1. wikilinks-samelen-textbase.txt         — each pair of tokens share at least one base form,
 1.a) wikilinks-samelen-textbase-one.txt   — links contain only a signel token,
 1.b) wikilinks-samelen-textbase-multi.txt — multi-token links,
 2. wikilinks-samelen-rules.txt            — at least one pair of token was not matched by their bases
                                             but using a suffix substitution (conjugation).
 3. wikilinks-samelen-different.txt        — the forms were not matched.

usage: python %prog -i FILENAME -o FILENAME"""
from docopt import docopt
import platform
import ctypes
import sys
import os

if 'Linux' in platform.system():
    dlflags = sys.getdlopenflags()
    sys.setdlopenflags(dlflags | ctypes.RTLD_GLOBAL)

import maca
import corpus2
import re

#------------------------------------------------------------------------------
# Options
#------------------------------------------------------------------------------
def parse_options():    
    from optparse import OptionParser
    optparser = OptionParser(usage="usage: python %prog -i FILENAME -o FILENAME\n\n" + re.sub("^", ">> ", __doc__, flags=re.MULTILINE))

    optparser.add_option("-i", "--input", action="store", dest="input", metavar="FILE",
                         help=u"Nazwa pliku wjeściowego")
    optparser.add_option("-o", "--output", action="store", dest="output", metavar="FOLDER",
                         help=u"Nazwa katalogu wyjściowego")
    
    (options, _) = optparser.parse_args()
    
    if not options.input:
        optparser.error("missing --input (-i) parameter")
        sys.exit()

    if not options.output:
        optparser.error("missing --output (-o) parameter")
        sys.exit()
  
    return (options, optparser)

#------------------------------------------------------------------------------
# MAIN
#------------------------------------------------------------------------------
def main():
    options, optparser = parse_options()

    tagset = corpus2.get_named_tagset('nkjp')

    conjugation_pairs = (('o', 'em'), ('o', 'owi'), ('o', 'a'), ('', 'a'), ('a', 'ą'), ('a', 'ę'), ('a', 'y'),
                     ('ski', 'skiego'), ('ski', 'skiemu'), ('cki', 'ckiego'), ('cki', 'ckiemu'),
                     ('cht', 'chta'), ('cht', 'chtem'), ('cht', 'chcie'))

    if 'Linux' in platform.system():
        # get back to default dlopen policy
        sys.setdlopenflags(dlflags)

    tagset = corpus2.get_named_tagset('nkjp')
    links_set = set()

    print "Reading links ..."
    with open(options.input, 'r') as wikilinks:
        for line in wikilinks.readlines():
            links_set.update(line.strip('\n').split('\t'))

    maca_input = ''
    with open(options.input, 'r') as wikilinks:
        maca_input = ('\n').join([line.replace('|', ' | ') for line in wikilinks.readlines()])

    same_textbase, same_rules, diff_textbase, diff_length = analyze(maca_input, tagset, conjugation_pairs)

    with open(os.path.join(options.output, 'wikilinks-samelen-textbase.txt'), 'w') as output:
        for uri, forms in same_textbase.iteritems():
            output.write(('\t').join([uri]+[f for f in forms])+'\n')

    with open(os.path.join(options.output, 'wikilinks-samelen-rules.txt'), 'w') as output:
        for uri, forms in same_rules.iteritems():
            output.write(('\t').join([uri]+[f for f in forms])+'\n')

    with open(os.path.join(options.output, 'wikilinks-samelen-diffrent.txt'), 'w') as output:
        for uri, forms in diff_textbase.iteritems():
            output.write(('\t').join([uri]+[f for f in forms])+'\n')

    with open(os.path.join(options.output, 'wikilinks-difflen.txt'), 'w') as output:
        for uri, forms in diff_length.iteritems():
            output.write(('\t').join([uri]+[f for f in forms])+'\n')

    one = open(os.path.join(options.output, 'wikilinks-samelen-textbase-one.txt'), 'w')
    multi = open(os.path.join(options.output, 'wikilinks-samelen-textbase-multi.txt'), 'w')
    for uri, forms in same_textbase.iteritems():
        if ' ' in uri:
            multi.write(('\t').join([uri] + [f for f in forms])+'\n')
        else:
            one.write(('\t').join([uri] + [f for f in forms])+'\n')
    one.close()
    multi.close()


#------------------------------------------------------------------------------
# Methods
#------------------------------------------------------------------------------
def analyze(string, tagset, conjugation_pairs):
    same_textbase, same_rules, diff_textbase, diff_length = {}, {}, {}, {}
    print "Tagging ..."
    reader = maca.PlainTextReader.create_string_reader(string, 'morfeusz-nkjp-official')
    i=0
    for link in get_wikilinks(reader):
        i+=1
        sys.stdout.write("\rProcessing line %d" % i)
        analyze_wikilink(link, same_textbase, same_rules, diff_textbase, diff_length, tagset, conjugation_pairs)
    del reader
    return same_textbase, same_rules, diff_textbase, diff_length


def analyze_wikilink(link, same_dict, same_rules_dict, diff_dict, diff_length, tagset, conjugation_pairs):
    check_ign(tagset, link['base'])
    base_orth = ('').join([(' ' + t.orth_utf8() if t.after_space() else t.orth_utf8()) for t in link['base']]).strip()
    for form in link['forms']:

        form_orth = ('').join([(' ' + t.orth_utf8() if t.after_space() else t.orth_utf8()) for t in form]).strip()
    
        if len(form) != len(link['base']):
            if base_orth not in diff_length:
                diff_length[base_orth] = set([form_orth])
            else:
                diff_length[base_orth].add(form_orth)
            continue
    
        conjugation_rules_match = False
        common_base_form = True
        for token_b, token_f in zip(link['base'], form):
            if not has_common_base(token_b.lexemes(), token_f.lexemes()):
                common_base_form = False
                conjugation_rules_match = check_conjugation_rules(token_b, token_f, tagset, conjugation_pairs)

        if common_base_form:
            if base_orth in same_dict: 
                same_dict[base_orth].add(form_orth)
            else:
                same_dict[base_orth] = set([form_orth])
                
        elif conjugation_rules_match:
            if base_orth in same_rules_dict:
                same_rules_dict[base_orth].add(form_orth)
            else:
                same_rules_dict[base_orth] = set([form_orth])

        else:
            if base_orth in diff_dict:
                diff_dict[base_orth].add(form_orth)
            else:
                diff_dict[base_orth] = set([form_orth])


def has_common_base(lexemes1, lexemes2):
    """
    Check if two lexems share at least one base form.
    """
    return set([l.lemma_utf8() for l in lexemes1]).intersection(set([l.lemma_utf8() for l in lexemes2]))


def check_ign(tagset, phrase):
    for token in phrase:
        if token.lexemes().size() > 1:
            for lex in token.lexemes():
                if tagset.tag_to_string(lex.tag()) == 'ign':
                    print (' | ').join([token.orth_utf8(), token.lexemes().size()]+[l.lemma_utf8() for l in token.lexemes()])


def is_ign(tagset, token):
    """
    Checks if the token has only the 'ign' interpretation.
    """
    return token.lexemes().size() == 1 and tagset.tag_to_string(token.lexemes()[0].tag()) == 'ign'


def check_conjugation_rules(base_token, form_token, tagset, conjugation_pairs):
    if is_ign(tagset, base_token) or is_ign(tagset, form_token):
        base_orth = base_token.orth_utf8()
        form_orth = form_token.orth_utf8()
        for base_ending, form_ending in conjugation_pairs:
            base_prefix = base_orth[:-len(base_ending)] if base_ending else base_orth
            form_prefix = form_orth[:-len(form_ending)] if form_ending else form_orth
            if base_prefix == form_prefix:
                if base_orth.endswith(base_ending) and form_orth.endswith(form_ending):
                    return True
    return False


def get_wikilinks(reader):
    """
    Reads link forms from the maca output.
    """
    while True:
        chunk = reader.get_next_chunk()
        if not chunk:
            break
        chunk_tokens = []
        for s in chunk.sentences():
            for t in s.tokens():
                chunk_tokens.append(t)

        base_tokens = []
        for idx, tok in enumerate(chunk_tokens):
            if tok.orth_utf8() == '|':
                chunk_tokens = chunk_tokens[idx+1:]
                break
            else:
                base_tokens.append(tok)

        forms = [[]]
        for tok in chunk_tokens:
            if tok.orth_utf8() == '|':
                forms.append([])
            else:
                forms[-1].append(tok)

        yield {'base': base_tokens, 'forms': forms}

#------------------------------------------------------------------------------
# ENTRY
#------------------------------------------------------------------------------
if __name__ == '__main__':
    main()
