# -*- coding: utf-8 -*-

u"""23 Nov 2016
Michał Marcińczuk <michal.marcinczuk_at_pwr.edu.pl,marcinczuk_at_gmail.com>
Wrocław University of Science and Technology

The script cleans and filters a list of internal links extracted from Wikipedia dump.

The input file must contain a list of links, every link in a seperate file. 
The link parameters are joined with a pipe (|).

For example:
<pre>
Almeida|Almeidy
islam|muzułmańskie
Kilwa Kisiwani|Kilwa
Tanzania|Tanzanii
</pre>

The script removes:
1. Links without a pipe (|)
2. Links with more than one pipe (|)
3. Links which starts with XXX:, where XXX can be Image, File, etc.
4. Removes texts in brackets (including the brackets)
"""

import sys
import codecs
import re


#------------------------------------------------------------------------------
# Options
#------------------------------------------------------------------------------
def parse_options():    
    from optparse import OptionParser
    optparser = OptionParser(usage="usage: python %prog -i FILENAME -o FILENAME\n\n" + re.sub("^", ">> ", __doc__, flags=re.MULTILINE))

    optparser.add_option("-i", "--input", action="store", dest="input", metavar="FILE",
                         help=u"Nazwa pliku wjeściowego (plik z meta wzorcami fraz).")
    optparser.add_option("-o", "--output", action="store", dest="output", metavar="FILE",
                         help=u"Nazwa pliku do którego zostaną zapisane reguły")
    
    (options, _) = optparser.parse_args()
    
    if not options.input:
        optparser.error("missing --input (-i) parameter")
        sys.exit()

    if not options.output:
        optparser.error("missing --output (-o) parameter")
        sys.exit()
  
    return (options, optparser)


#------------------------------------------------------------------------------
# MAIN
#------------------------------------------------------------------------------
def main():        
    options, optparser = parse_options()

    pattern_link_type = re.compile("^[a-zA-Z]*:")
    pattern_link_brackets = re.compile(" [(][^)]*[)]");
    
    out = codecs.open(options.output, "w", "utf8")
    i = 0

    for line in codecs.open(options.input, "r", "utf8"):
        i+=1
        if i%1000 == 0:
            sys.stdout.write("\r%d line(s) processed" % i)
        line = line.strip()
        pipes = line.count("|")
        
        # Ignore links without a pipe or with more than one pipe.
        if pipes != 1:
            continue
        
        # Ignore links with defined link types
        if pattern_link_type.match(line):
            continue
        
        # Remove brackets with their content
        line = pattern_link_brackets.sub("", line)
        
        out.write(line)
        out.write("\n")
    
    out.close()

    
#------------------------------------------------------------------------------
# Entry
#------------------------------------------------------------------------------
if __name__ == '__main__':
    main()