#!/usr/bin/env python

# $LastChangedDate: 2006-08-01 18:03:21 +0200 (Tue, 01 Aug 2006) $
# $Rev: 23 $
# $Author: paul $

"""
SYNOPSIS

tv_grab_nl_py is a python script that trawls tvgids.nl for TV
programming information and outputs it in XMLTV-formatted output (see
http://membled.com/work/apps/xmltv). Users of MythTV
(http://www.mythtv.org) will appreciate the output generated by this
grabber, because it fills the category fields, i.e. colors in the EPG,
and has logos for most channels automagically available. Check the
website below for screenshots.  The newest version of this script can be
found here: 

     http://visualisation.tudelft.nl/~paul/grabber

USAGE

Check the web site above and/or run script with --help and start from there

HISTORY

tv_grab_nl_py used to be called tv_grab_nl_pdb, first released on
2003/07/09. The name change was necessary because more and more people
are actively contributing to this script and I always disliked using my
initials (I was just too lazy to change it). At the same time I switched
from using CVS to SVN and as a result the version numbering scheme has
changed. The lastest official release of tv_grab_nl_pdb is 0.48. The
first official release of tv_grab_nl_py is 6.

QUESTIONS

Questions (and patches) are welcome at: paul at pwdebruin dot net.

IMPORTANT NOTES

If you were using tv_grab_nl from the XMLTV bundle then enable the
compat flag or use the --compat command-line option.  Otherwise, the
xmltvid's are wrong and you will not see any new data in MythTV.

CONTRIBUTORS

Main author: Paul de Bruin (paul at pwdebruin dot net)

Michel van der Laan made available his extensive collection of
high-quality logos that is used by this script. 

Michael Heus has taken the effort to further enhance this script so that
it now also includes:
 - Credit info: directors, actors, presenters and writers
 - removal of programs that are actually just groupings/broadcasters 
   (e.g. "KETNET", "Wild Friday", "Z@pp")
 - Star-rating for programs tipped by tvgids.nl
 - Black&White, Stereo and URL info
 - Better detection of Movies
 - and more... 

Several other people have provided feedback and patches (these are the
people I could find in my email archive, if you are missing from this
list let me know):
Huub Bouma, Roy van der Kuil, Remco Rotteveel, Mark Wormgoor, Dennis van
Onselen, Hugo van der Kooij

""" 

# Modules we need
import re, urllib, getopt, sys
import time, random 
import htmlentitydefs, os, os.path, pickle
from string import replace, split, strip
from threading import Thread

# do extra debug stuff
debug = 1

try:
    import redirect
except:
    debug = 0
    pass

# globals

tvgids = 'http://www.tvgids.nl/'
uitgebreid_zoeken = tvgids + 'zoeken'

# how many seconds to wait before we timeout on a 
# url fetch, 10 seconds seems reasonable
global_timeout = 10

# Wait a random number of seconds between each page fetch.
# We want to be nice and not hammer tvgids.nl (these are the 
# friendly people that provide our data...).
# Also, it appears tvgids.nl throttles its output.
# So there, there is not point in lowering these numbers, if you 
# are in a hurry, use the (default) fast mode.
nice_time = [1, 3]

# Maximum number of characters to use for program description.
# This is a MythTV-specific setting.
desc_len = 475

# Maximum length in minutes of overlapping programming to correct
max_overlap = 10

# Create a category translation dictionary
# Look in mythtv/themes/blue/ui.xml for all category names
# The keys are the categories used by tvgids.nl (lowercase please)
cattrans = { 'amusement'        : 'Talk',
             'animatie'         : 'Animated',
             'comedy'           : 'Comedy',
             'documentaire'     : 'Documentary',
             'educatief'        : 'Educational',
             'erotiek'          : 'Adult',
             'film'             : 'Movies',
             'muziek'           : 'Music',
             'informatief'      : 'Educational',
             'jeugd'            : 'Kids',
             'kunst/cultuur'    : 'arts/culture',
             'misdaad'          : 'Crime/Mystery',
             'muziek'           : 'Music',
             'natuur'           : 'Nature',
             'nieuws/actualiteiten' : 'News',
             'overige'          : 'Unknown',
             'religieus'        : 'Religion',
             'serie/soap'       : 'SERIE',
             'sport'            : 'Sports',
             'theater'          : 'music/ballet/dance',
             'wetenschap'       : 'Science/Nature'}

# Create a role translation dictionary for the xmltv credits part
# The keys are the roles used by tvgids.nl (lowercase please)
roletrans = {'regie'             : 'director',
             'acteurs'           : 'actor',
             'presentatie'       : 'presenter',
             'scenario'          : 'writer'}

# We have two sources of logos, the first provides the nice ones, but is not 
# complete. We use the tvgids logos to fill the missing bits.
logo_provider = [ 'http://visualisation.tudelft.nl/~paul/logos/gif/64x64/',
                  'http://static.tvgids.nl/gfx/zenders/' ]

logo_names = { 
            1 : [0, 'ned1'],
            2 : [0, 'ned2'],
            3 : [0, 'ned3'],
            4 : [0, 'rtl4'],
            5 : [0, 'een'],
            6 : [0, 'canvas_color'],
            7 : [0, 'bbc1'],
            8 : [0, 'bbc2'],
            9 : [0,'ard'],
            10 : [0,'zdf'],
            11 : [1, 'rtl'],
            12 : [0, 'wdr'],
            13 : [1, 'ndr'],
            14 : [1, 'srsudwest'],
            15 : [1, 'rtbf1'],
            16 : [1, 'rtbf2'],
            17 : [0, 'tv5'],
            18 : [0, 'ngc'],
            19 : [1, 'eurosport'],
            20 : [1, 'tcm'],
            24 : [0, 'canal+red'],
            25 : [0, 'mtv-color'],
            26 : [0, 'cnn'],
            27 : [0, 'rai'],
            28 : [1, 'sat1'],
            29 : [0, 'discover-spacey'],
            31 : [0, 'rtl5'],
            32 : [1, 'trt'],
            34 : [0, 'veronica'],
            35 : [0, 'tmf'],
            36 : [0, 'sbs6'],
            37 : [0, 'net5'],
            38 : [1, 'arte'],
            39 : [0, 'canal+blue'],
            40 : [0, 'at5'],
            46 : [0, 'rtl7'],
            49 : [1, 'vtm'],
            50 : [1, '3sat'],
            58 : [1, 'pro7'],
            59 : [1, 'kanaal2'],
            60 : [1, 'vt4'],
            65 : [0, 'animal-planet'],
            73 : [1, 'mezzo'],
            86 : [0, 'bbc-world'],
            87 : [1, 'tve'],
            89 : [1, 'nick'],
            90 : [1, 'bvn'],
            92 : [0, 'talpa-mono'],
            100 : [1, 'tvutrecht'],
            101 : [1, 'tvwest'],
            102 : [1, 'tvrijnmond'],
            103 : [1, 'tvnoordholland'],
            105 : [1, 'spice'],
            107 : [0, 'canal+yellow']}

# Work in progress, the idea is to cache program categories and
# descriptions to eliminate a lot of page fetches from tvgids.nl
# for programs that do not have interesting/changing descriptions

class ProgramCache:
    """
    A cache to hold program name and category info.
    TVgids stores the detail for each program on a separate URL with an
    (apparently unique) ID. This cache stores the fetched info with the ID.
    New fetches will use the cached info instead of doing an (expensive)
    page fetch.
    """
    def __init__(self, filename=None):
        """
        Create a new ProgramCache object, optionally from file 
        """

        # where we store our info
        self.filename  = filename

        if filename == None:
            self.pdict = {}
        else:
            if os.path.isfile(filename):
                self.load(filename)
            else:
                self.pdict = {}


    def load(self, filename):
        """
        Loads a pickled cache dict from file
        """
        self.pdict = pickle.load(open(filename,'r'))

    def dump(self, filename):
        """
        Dumps a pickled cache
        """
        pickle.dump(self.pdict, open(filename, 'w'))

    
    def query(self, program_id):
        """
        Updates/gets/whatever.
        """

        try:
            return self.pdict[program_id]
        except:
            return None

    def add(self, program):
        """
        Adds a program
        """
        self.pdict[program['ID']] = program

    def clear(self):
        """
        Clears the cache (i.e. empties it)
        """
        self.pdict = {}

    def clean(self):
        """
        Removes all cached programming before today.
        Also removes erroneously cached programming.
        """
        now = int(time.strftime('%Y%m%d'))
        for key in self.pdict.keys():
                if int(self.pdict[key]['stop'][0:8]) < now:
                    del self.pdict[key]
                # normally, these entries should not be present in the 
                # cache, but people may have caches filled with these
                # entries before they upgraded to a newer version of this
                # script
                elif self.pdict[key]['name'].lower() == 'onbekend':
                    del self.pdict[key]


def usage():
    print 'tv_grab_nl_py: A grabber that grabs tvguide data from tvgids.nl\n'
    print 'and stores it in XMLTV-combatible format.\n'
    print 'Usage:'
    print '--help, -h    = print this info'
    print '--configure   = create configfile (overwrites existing file)'
    print '--config-file = name of the configuration file (default = ~/.xmltv/tv_grab_py.conf'
    print '--output      = file where to put the output'
    print '--days        = # number of days to grab'
    print '--offset      = # day offset from where to grab (0 is today, 1 tomorow, etc)'
    print '--slow        = also grab descriptions of programming'
    print '--quiet       = suppress all output'
    print '----'
    print '--compat      = append tvgids.nl to the xmltv id (use this if you were using tv_grab_nl)'
    print '--logos       = insert urls to channel icons (mythfilldatabase will then use these)'
    print '--nocattrans  = do not translate the grabbed genres into MythTV-genres'
    print '--cache       = cache descriptions and use the file to store'
    print '--clean_cache = clean the cache file before fetching'
    print '--clear_cache = empties the cache file before fetching data'
    print '--slowdays    = grab slowdays initial days and the rest in fast mode'

def filter_line_identity(m, defs=htmlentitydefs.entitydefs):
    # callback: translate one entity to its ISO Latin value
    k = m.group(1)
    if k.startswith("#"):
        return chr(int(k[1:]))

    try:
        return defs[k]
    except KeyError:
        return m.group(0) # use as is

def filter_line(s):
    """
    Removes unwanted stuff in strings (adapted from tv_grab_be)
    """

    # do the latin1 stuff
    pattern = re.compile("&(\S+?);")
    s = pattern.sub(filter_line_identity, s)

    s = replace(s,'&nbsp;',' ')
    s = replace(s,'\r',' ')
    x = re.compile('(<.*?>)')
    s = x.sub('', s)

    # A couple of characters which are not legal in Latin-1, we have
    # to guess what they are.
    #
    s = replace(s, '~Q', "'")
    s = replace(s, '~R', "'")

    # Hmm, not sure if I understand this. Without it, mythfilldatabase barfs
    # on program names like "Steinbrecher &..."
    s = replace(s,'&','&amp;')

    return s

def calc_timezone(t):
    """
    Takes a time from tvgids.nl and formats it with all the required
    timezone conversions.
    in: '20050429075000'
    out:'20050429075000 (CET|CEST)'

    Until I have figured out how to correctly do timezoning in python this method
    will bork if you are not in a zone that has the same DST rules as 'Europe/Amsterdam'.

    """

    year = int(t[0:4])
    month = int(t[4:6])
    day = int(t[6:8])
    hour = int(t[8:10])
    minute = int(t[10:12])

    pt = time.mktime((year,month,day,hour,minute,0,0,0,-1))
    timezone=''
    try:
        timezone = time.tzname[(time.localtime(pt))[-1]]
    except:
        sys.stderr.write('Cannot convert time to timezone')

    return t+' %s' % timezone

def duration(t1,t2):
    """
    Calculates the duration of a program (24h times)
    in minutes. [h2,m2] can be on the next day.

    duration(23,10,23,15) -> 5
    duration(23,10,0,20)  -> 70
    """
    h1 = int(t1[0:2])
    m1 = int(t1[3:5])
    h2 = int(t2[0:2])
    m2 = int(t2[3:5])
    if h2<h1:
        hd = 23-h1 + h2
        md = 60-m1 + m2
    else:
        hd = h2-h1
        md = m2-m1
        if md<0:
            md = 60+md
            hd = hd - 1
    return hd*60+md

def time_adjust(t1, minutes):
    """
    Adds (or subtracts) minutes from time t1
    time_adjust('11:35', -5) --> '11:30'
    """
    h = int(t1[0:2])
    m = int(t1[3:5])+minutes
    nh,nm = divmod(h*60+m,60)
    return '%02d:%02d' % (nh % 24,nm)

def get_page_internal(url, quiet=0):
    """
    Retrieves the url and returns a string with the contents.
    Optionally, returns None if processing takes longer than
    the specified number of timeout seconds.
    """
    try:
        fp = urllib.urlopen(url)
        lines = fp.readlines()
        page = "".join(lines)
        return page
    except:
        if not quiet:
            sys.stderr.write('Cannot open url: %s\n' % url)
        return None

class FetchURL(Thread):
    """
    A simple thread to fetch a url with a timeout
    """
    def __init__ (self, url, quiet=0):
        Thread.__init__(self)
        self.quiet = quiet
        self.url = url
        self.result = None

    def run(self):
        self.result = get_page_internal(self.url, self.quiet)

def get_page(url, quiet=0):
    """
    Wrapper around get_page_internal to catch the
    timeout exception
    """
    try: 
        fu = FetchURL(url, quiet)
        fu.start()
        fu.join(global_timeout)
        return fu.result
    except:
        if not quiet:
            sys.stderr.write('get_page timed out on (>%s s): %s\n' % (global_timeout, url))
        return None

def get_channels(file, quiet=0):
    """
    Get a list of all available channels and store these
    in a file.
    """
    # store channels in a dict
    channels = {}

    # tvgids stores several instances of channels, we want to
    # find all the possibile channels
    channel_get = re.compile('<optgroup label=.*?>(.*?)</optgroup>', re.DOTALL)

    # this is how we will find a (number, channel) instance
    channel_re  = re.compile('<option value="([0-9]+)" >(.*?)</option>', re.DOTALL)

    # this is where we will try to find our channel list
    total = get_page(uitgebreid_zoeken, quiet)
    if total == None:
        return

    # get a list of match objects of all the <select blah station>
    stations = channel_get.finditer(total)

    # and create a dict of number, channel_name pairs
    # we do this this way because several instances of the 
    # channel list are stored in the url and not all of the 
    # instances have all the channels, this way we get them all.
    for station in stations:
        m = channel_re.finditer(station.group(0))           
        for p in m:
            a = int(p.group(1))
            b = p.group(2)
            # just to make sure we receive sane values, 
            # there are no channels above 200 (yet). 
            if a>0 and a<200:
                channels[a] = b

    # sort on channel number (arbitrary but who cares)
    keys = channels.keys()
    keys.sort()

    # and create a file with the channels
    f = open(file,'w')
    for k in keys:
        f.write("%s %s\n" % (k, channels[k]))
    f.close()

def get_channel_all_days(channel, quiet=0):
    """
    Get all available days of programming for channel number

    The output is a list of programming in order where each row
    contains a dictionary with program information.
    """

    programs = []

    channel_url = 'http://www.tvgids.nl/zoeken/?trefwoord=Titel+of+trefwoord&interval=0&timeslot='+\
                  '&station=%s&periode=6&genre=&order=0' % channel

    # get the raw programming for a week
    total = get_page(channel_url, quiet)
    if total == None:
        return programs

    # Setup a number of regexps

    # match  <tr .*>.*</tr> 
    # 1 = optional "class=progTip"
    # 2 = program info
    getrow = re.compile('<[tT][Rr](.*?)>(.*?)</[tT][rR]>',re.DOTALL)

    # match the required program info
    # 1 = times
    # 2 = ID
    # 3 = program name
    parserow = re.compile('<th.*?>(.*?)</th>.*?ID=(.*?)">(.*?)</a', re.DOTALL)

    #  normal begin and end times
    times = re.compile('([0-9]+:[0-9]+) - ([0-9]+:[0-9]+)?')

    # day switch
    dayswitch = re.compile('<h4>.*?</h4>')

    # and find relevant programming info
    allrows = getrow.finditer(total)

    offset = -1

    for r in allrows:

        found_new_day = dayswitch.search(r.group(2))
        if found_new_day != None:
            offset = offset + 1

        detail = parserow.search(r.group(2))

        if detail != None: 

            # default times
            start_time = None
            stop_time  = None

            # parse for begin and end times
            t  = times.search(detail.group(1))

            if t != None:
                start_time = t.group(1)
                stop_time  = t.group(2)

            program_url  = 'http://www.tvgids.nl/programmadetail/?ID=' + detail.group(2)
            program_name = detail.group(3)

            # store time, name and detail url in a dictionary 
            tdict = {}
            tdict['start'] = start_time
            tdict['stop']  = stop_time
            tdict['name']  = program_name
            tdict['url']   = program_url
            tdict['ID']    = detail.group(2)
            tdict['offset'] = offset

            #Add star rating if tipped by tvgids.nl
            tdict['star-rating'] = '';
            if r.group(1).find('Tip') != -1:
                tdict['star-rating'] = '4/5'

            # and append the program to the list of programs
            programs.append(tdict)

    # done
    return programs


def parse_programs(programs, offset=0, quiet=0):
    """
    Parse a list of programs as generated by get_channel_all_days()  and
    convert begin and end times to xmltv compatible times.  
    """

    # good programs
    good_programs = []

    for i in range(len(programs)):

        # Try to correct missing end time by taking start time from next program on schedule
        if (programs[i]['stop'] == None and i < len(programs)-1):
            if not quiet:
                sys.stderr.write('Oops, "%s" has no end time. Trying to fix...\n' % programs[i]['name'])
            programs[i]['stop'] = programs[i+1]['start']

        # PdB: Fix tvgids start-before-end x minute interval overlap.  An overlap (positive or
        # negative) is halved and each half is assigned to the adjacent programmes. The maximum
        # overlap length between programming is set by the global variable 'max_overlap' and is 
        # default 10 minutes. Example:
        #
        # 10:55 - 12:00 Lala
        # 11:55 - 12:20 Wawa
        # is transformed in:
        # 10:55 - 11.57 Lala
        # 11:57 - 12:20 Wawa
        # 
        # and also:
        # 
        # 10:55 - 11:50 Lala
        # 12:00 - 12:20 Wawa
        # is transformed in:
        # 10:55 - 11.55 Lala
        # 11:55 - 12:20 Wawa
         

        if (programs[i]['stop'] != None and i < len(programs)-1):
            if programs[i+1]['start'] != None:
                stop  = programs[i]['stop']
                start = programs[i+1]['start']
                interval = duration(stop, start)
                if 0 < abs(interval) <= max_overlap:
                    if not quiet:
                        sys.stderr.write('"%s" and "%s" overlap %s minutes. Adjusting times.\n' % \
                            (programs[i]['name'],programs[i+1]['name'], interval))
                    # strategy 1: use stop time
                    #programs[i+1]['start'] = programs[i]['stop']
                    # strategy 2: use start time
                    #programs[i]['stop'] = programs[i+1]['start']
                    # strategy 3: subdivide the interval
                    programs[i]['stop']    = time_adjust(programs[i]['stop'],abs((interval/2))) 
                    programs[i+1]['start'] = programs[i]['stop']
                    # strategy 4: clump the programming?

        # The common case: start and end times are present and are not
        # equal to each other (yes, this can happen)
        if programs[i]['start'] != None and \
           programs[i]['stop']  != None and \
           programs[i]['start'] != programs[i]['stop']:
            good_programs.append(programs[i])

    # store enough dates such that an offset is an index into a list of actual dates
    dates = [time.strftime('%Y%m%d', time.gmtime(time.time()+x*86400)) for x in range(0,20)]

    # and finally, modify the start/end times to xmltv format
    for poffset in range(0,7):
        add_offset = 0

        for c in range(len(good_programs)):

            if good_programs[c]['offset'] != poffset:
                continue
            start = good_programs[c]['start'].replace(':','')
            stop  = good_programs[c]['stop'].replace(':','')
            offset = good_programs[c]['offset']

            # check for clumped programming
            # crude solution for now, make it more elegant later
            if c < len(good_programs)-1:
                nstart = good_programs[c+1]['start'].replace(':','')
                nstop  = good_programs[c+1]['stop'].replace(':','')
                noffset = good_programs[c+1]['offset']
                if nstart == start and nstop == stop and offset==noffset:
                    good_programs[c]['clumpidx'] = '0/2'
                    good_programs[c+1]['clumpidx'] = '1/2'


            if stop=='0000' or (int(stop)<int(start)):
                good_programs[c]['start'] = dates[poffset+add_offset]+start +'00'
                good_programs[c]['stop']  = dates[poffset+add_offset+1]+stop +'00'

                if (c < len(good_programs)-1) and (start > good_programs[c+1]['start'].replace(':','')):
                   add_offset = 1

            elif start=='0000':
                add_offset = 1
                # assign the new times
                good_programs[c]['start'] = dates[poffset+add_offset]+start+'00'
                good_programs[c]['stop']  = dates[poffset+add_offset]+stop +'00'

            else:
                # assign base converted times
                good_programs[c]['start'] = dates[poffset+add_offset]+start+'00'
                good_programs[c]['stop']  = dates[poffset+add_offset]+stop +'00'

    # Try to exclude programs that only identify a group or broadcaster and have overlapping start/end times with
    # the actual programs
    for i in range(len(good_programs)-2,-1,-1):
          
        if good_programs[i]['start'] <= good_programs[i+1]['start'] and good_programs[i]['stop'] >= good_programs[i+1]['stop']:
            if not quiet:
                sys.stderr.write('Deleting grouping/broadcaster: %s\n' % good_programs[i]['name'])
            del good_programs[i]

    # done, nothing to see here, please move on 
    return good_programs

def get_descriptions(programs, program_cache=None, nocattrans=0, quiet=0, slowdays=0):
    """
    Given a list of programs, from get_channel, retrieve program information
    """

    # detail regexp now also allows detection of <td class=personen> tags
    detail      = re.compile('<th>(.*?):</th>.*?<td.*?>(.*?)</td>', re.DOTALL)
    detaildiv   = re.compile('<div>(.*?)</div>', re.DOTALL)
    description = re.compile('<p class="inleiding">(.*?)</p>.*?<p>(.*?)</p>.*?<p>(.*?)</p>',re.DOTALL)
    # regexp for finding persons in list of roles
    person      = re.compile('(.+?)(\.\ [^:]*\:\ |en\/of|e\.a.*|\/|\ en\ |<br\ />|;\ )',re.DOTALL)

    # randomize detail requests
    nprograms = len(programs)
    fetch_order = range(0,nprograms)
    random.shuffle(fetch_order)

    counter = 0
    for i in fetch_order:
        counter += 1
        if programs[i]['offset'] >= slowdays:
                continue
        
        if not quiet:
            sys.stderr.write('\n(%3.0f%%) %s: %s ' % (100*float(counter)/float(nprograms), i, programs[i]['name']))

        # check the cache for this program's ID
        cached_program = program_cache.query(programs[i]['ID'])
        if (cached_program != None):
                if not quiet:
                    sys.stderr.write(' [cached]')
                # copy the cached information, except the start/end times and rating, 
                # these may have changed.
                tstart = programs[i]['start']
                tstop  = programs[i]['stop']
                offset = programs[i]['offset']
                rating = programs[i]['star-rating']
                programs[i] = cached_program
                programs[i]['start'] = tstart
                programs[i]['stop']  = tstop
                programs[i]['offset'] = offset
                programs[i]['star-rating'] = rating
                continue
        else:
            # be nice to tvgids.nl
            time.sleep(random.randint(nice_time[0], nice_time[1]))

        # get the details page, and get all the detail nodes
        descriptions = ()
        try:
            if not quiet:
                sys.stderr.write(' [normal fetch]')
            total = get_page(programs[i]['url'])
            details = detail.finditer(total)
            descriptions = description.finditer(total)
        except:
            # if we cannot find the description page, 
            # go to next in the loop
            if not quiet:
                sys.stderr.write(' [fetch failed or timed out]')
                continue
        # define containers
        programs[i]['credits'] = {}
        programs[i]['video']   = {}

        # now parse the details
        for descript in descriptions:
            programs[i]['detail1'] = filter_line(descript.group(1)).strip()
            programs[i]['detail2'] = filter_line(descript.group(2)).strip()
            programs[i]['detail3'] = filter_line(descript.group(3)).strip()

        for d in details:
            type = d.group(1).strip().lower()
            content_asis = d.group(2).strip()

            content = filter_line(content_asis).strip()
            if content == '':
                continue

            elif type == 'genre':

                # Fix detection of movies based on description as tvgids.nl sometimes 
                # categorises a movie as e.g. "Komedie". 
                genre = content;
                if     programs[i]['detail1'].lower().find('film')      != -1 \
                   or  programs[i]['detail1'].lower().find('komedie')   ==  0 \
                   and programs[i]['detail1'].lower().find('tekenfilm') == -1 \
                   and programs[i]['detail1'].lower().find('filmpje')   == -1:
                    genre = 'film'

                if nocattrans:
                    programs[i]['genre'] = genre.title()
                else:
                    try:
                        programs[i]['genre'] = cattrans[genre.lower()]
                    except:
                        programs[i]['genre'] = ''


            # Parse persons and their roles for credit info
            elif roletrans.has_key(type):
                programs[i]['credits'][roletrans[type]] = []
                persons = person.finditer(detaildiv.search(content_asis).group(1))
                for name in persons:
                    name = name.group(1)
                    if name.find(':') != -1:
                        name = name.split(':')[1]
                    if name.find('-') != -1:
                        name = name.split('-')[0]
                    programs[i]['credits'][roletrans[type]].append(filter_line(name.strip()))

            elif type == 'bijzonderheden':
                if content.find('Breedbeeld') != -1:
                    programs[i]['video']['breedbeeld'] = 1
                if content.find('Zwart') != -1: 
                    programs[i]['video']['blackwhite'] = 1
                if content.find('Teletekst') != -1: 
                    programs[i]['teletekst'] = 1
                if content.find('Stereo') != -1: 
                    programs[i]['stereo'] = 1
            elif type == 'url':
                programs[i]['infourl'] = content
            else:
                programs[i][type] = content

        # do not cache programming that is unknown at the time
        # of fetching.
        if programs[i]['name'].lower() != 'onbekend':
            program_cache.add(programs[i])

    if not quiet:
        sys.stderr.write('\ndone...\n\n')
                    
    # done
      
def title_kludge(program):
    """
    Discovery, National Geographic and Animal Planet have the annoying
    habit of adding the subtitle to the title of a program. 
    This function attempts to fix this, by splitting the name at a ':'.
    """
    name = program['name'].split(':')

    if len(name)==2:
        title = name[0].strip()
        subtitle = name[1].strip()
        program['name'] = title
        program['titel aflevering'] = subtitle

    elif len(name)==3:
        title = name[0].strip()
        subtitle = name[1].strip() + ': ' + name[2].strip()
        program['name'] = title
        program['titel aflevering'] = subtitle


def xmlefy_programs(programs, channel, compat=0, nocattrans=0):
    """
    Given a list of programming (from get_channels())
    returns a string with the xml equivalent
    """
    output = []
    for program in programs:

        clumpidx = ''
        try:
            if program.has_key('clumpidx'):
                clumpidx = 'clumpidx="'+program['clumpidx']+'"'
        except:
            print program

        output.append('  <programme start="%s" stop="%s" channel="%s%s" %s> \n' % \
            (calc_timezone(program['start']), calc_timezone(program['stop']),\
             channel, compat and '.tvgids.nl' or '', clumpidx))

        output.append('    <title lang="nl">%s</title>\n' % filter_line(program['name']))

        if program.has_key('titel aflevering') and program['titel aflevering'] != '':
                output.append('    <sub-title lang="nl">%s</sub-title>\n' % filter_line(program['titel aflevering']))

        desc = []
        for detail_row in ['detail1','detail2','detail3']:
                if program.has_key(detail_row) and program[detail_row] != '':
                        desc.append('%s ' % program[detail_row])
        if desc != []:
                # join and remove newlines from descriptions
                desc_line = "".join(desc).strip()
                desc_line.replace('\n', ' ')
                if len(desc_line) > desc_len: 
                    spacepos = desc_line[0:desc_len-3].rfind(' ') 
                    desc_line = desc_line[0:spacepos] + '...'
                output.append('    <desc lang="nl">%s</desc>\n' % desc_line)
        
        # Process credits section if present.
        # This will generate director/actor/presenter info.
        if program.has_key('credits') and program['credits'] != {}:
            output.append('    <credits>\n')
            for role in program['credits']:
                for name in program['credits'][role]:
                    if name != '':
                        output.append('       <%s>%s</%s>\n' % (role, name, role))
            output.append('    </credits>\n')

        if program.has_key('jaar van premiere') and program['jaar van premiere'] != '':
                output.append('    <date>%s</date>\n' % program['jaar van premiere'])

        if program.has_key('genre') and program['genre'] != '':
                output.append('    <category')
                if nocattrans:
                   output.append(' lang="nl"')
                output.append ('>%s</category>\n' % program['genre'])
        
        if program.has_key('infourl') and program['infourl'] != '':
                output.append('    <url>%s</url>\n' % program['infourl']) 

        if program.has_key('aflevering') and program['aflevering'] != '':
                output.append('    <episode-num system="onscreen">%s</episode-num>\n' % filter_line(program['aflevering']))

        # Process video section if present
        if program.has_key('video') and program['video'] != {}:
            output.append('    <video>\n');
            if program['video'].has_key('breedbeeld'):
                output.append('           <aspect>"16:9"</aspect>\n')
            if program['video'].has_key('blackwhite'):
                output.append('           <colour>no</colour>\n')
            output.append('    </video>\n')

        if program.has_key('stereo'):
            output.append('    <audio><stereo>yes</stereo></audio>\n')
 
        if program.has_key('teletekst'):
            output.append('    <subtitles type="teletext" />\n')

        # Set star-rating if applicable
        if program['star-rating'] != '':
             output.append('    <star-rating><value>%s</value></star-rating>\n' % program['star-rating'])
                
        output.append('  </programme>\n')
    
    return "".join(output)

def main():

    # Parse command line options
    try:
        opts, args = getopt.getopt(sys.argv[1:], "h", ["help", "output=", 
                                                       "offset=", "days=", 
                                                       "configure", "slow",
                                                       "cache=", "clean_cache", 
                                                       "slowdays=","compat",
                                                       "nocattrans","config-file=",
                                                       "clear_cache", "quiet","logos"])
    except getopt.GetoptError:
        usage()
        sys.exit(2)

    # DEFAULT OPTIONS - Edit if you know what you are doing

    # where the output goes
    output      = None
    output_file = None

    # offset from today, i.e. today = 0, tomorrow = 1, etc 
    offset      = 0

    # the total number of days to fetch 
    days        = 7

    # fetch data in slow mode, i.e. grab all the detail information,
    # slow means slow, because for each program a web page needs to be fetched
    slow        = 1

    # number of days to fetch in slow mode. For example: --days 5 --slowdays 2, will 
    # fetch the first two days in slow mode (with all the details) and the remaining three
    # days in fast mode.
    slowdays    = 4

    # no output 
    quiet       = 0

    # insert url of channel logo into the xml data, this will be picked up by mythfilldatabase
    logos       = 1

    # enable this option if you were using tv_grab_nl, it adjusts the generated
    # xmltvid's so that everything works.
    compat      = 0
    
    # enable this option if you do not want the tvgids categories being translated into
    # MythTV-categories (genres)
    nocattrans  = 0
    
    # default configuration file locations
    xmltv_dir   = os.environ['HOME']+'/.xmltv'
    program_cache_file = xmltv_dir+'/program_cache'
    config_file = xmltv_dir+'/tv_grab_nl_py.conf'

    # cache the detail information. 
    program_cache = None
    clean_cache = 1
    clear_cache = 0

    # seed the random generator
    random.seed()

    for o, a in opts:
        if o in ("-h", "--help"):
            usage()
            sys.exit(1)

        if o == "--quiet":
            quiet = 1;

    for o, a in opts:
        if o == "--config-file":
            # use the provided name for configuration
            config_file = a
            if not quiet:
                sys.stderr.write('Using config file: %s\n' % config_file)

    for o, a in opts:
        if o == "--configure":
            # check for the ~.xmltv dir
            if not os.path.exists(xmltv_dir):
                if not quiet:
                    sys.stderr.write('You do not have the ~/.xmltv directory,')
                    sys.stderr.write('I am going to make a shiny new one for you...')
                os.mkdir(xmltv_dir)
            if not quiet:
                sys.stderr.write('Creating config file: %s\n' % config_file)
            get_channels(config_file)
            sys.exit(2)

        if o == "--days":
            days = int(a)

        if o == "--compat":
            compat = 1

        if o == "--nocattrans":
            nocattrans = 1

        if o == "--offset":
            offset = int(a)

        if o == "--slow":
            slow = 1

        if o == "--output":
            output_file = a
            try:
                output = open(output_file,'w')
                # and redirect output
                if debug:
                    debug_file = open('/tmp/kaas.xml','w')
                    blah = redirect.Tee(output, debug_file) 
                    sys.stdout = blah
                else:
                    sys.stdout = output
            except:
                if not quiet:
                    sys.stderr.write('Cannot write to outputfile: %s\n' % output_file)
                sys.exit(2)

        if o == "--slowdays":
            slowdays = int(a)
            # slowdays implies slow == 1
            slow = 1

        if o == "--clean_cache":
            clean_cache = 1
        if o == "--clear_cache":
            clear_cache = 1
        if o == "--cache":
            program_cache_file = a

    # get configfile if available
    try:
        f = open(config_file,'r')
    except:
        sys.stderr.write('Config file %s not found.\n' % config_file)
        sys.stderr.write('Re-run me with the --configure flag.\n')
        sys.exit(1)

    #check for cache
    program_cache = ProgramCache(program_cache_file)
    if clean_cache != 0:
        program_cache.clean()
    if clear_cache != 0:
        program_cache.clear()

    # Go!
    channels = {}

    # Read the channel stuff
    for blah in f.readlines():
        blah = blah.lstrip()
        blah = blah.replace('\n','')
        if blah:
            if blah[0] != '#':
                channel = blah.split()
                channels[channel[0]] = " ".join(channel[1:])

    # channels are now in channels dict keyed on channel id

    # print header stuff
    print '<?xml version="1.0" encoding="ISO-8859-1"?>'
    print '<!DOCTYPE tv SYSTEM "xmltv.dtd">'
    print '<tv generator-info-name="tv_grab_nl_py $Rev: 23 $">'

    # first do the channel info
    for key in channels.keys():
        print '  <channel id="%s%s">' % (key, compat and '.tvgids.nl' or '')
        print '    <display-name lang="nl">%s</display-name>' % channels[key]
        if (logos):
            ikey = int(key)
            if logo_names.has_key(ikey):
                full_logo_url = logo_provider[logo_names[ikey][0]]+logo_names[ikey][1]+'.gif'
                print '    <icon src="%s" />' % full_logo_url
        print '  </channel>'

    num_chans = len(channels.keys())
    cur_day = -1

    fluffy = channels.keys()
    nfluffy = len(fluffy)
    for id in fluffy:
        cur_day += 1
        if not quiet:
                sys.stderr.write('Now fetching %s(xmltvid=%s%s) (channel %s of %s)\n' % \
                    (channels[id], id, (compat and '.tvgids.nl' or ''), cur_day, nfluffy))
        info = get_channel_all_days(id,  quiet)
        blah = parse_programs(info, None, quiet)
        # check for discovery, ngc and animal
        if int(id) in [18, 29, 65]:
            for program in blah:
                    title_kludge(program)

        # fetch descriptions
        if slow:
           get_descriptions(blah, program_cache, nocattrans, quiet, slowdays)

        # check for discovery, ngc and animal (there was a good reason for duplicating
        # this check, but I forgot what is was :-) )
        if int(id) in [18, 29, 65]:
            for program in blah:
                    title_kludge(program)

        print xmlefy_programs(blah, id, compat, nocattrans)
    
        # be nice to tvgids.nl
        time.sleep(random.randint(nice_time[0], nice_time[1]))

    # print footer stuff
    print "</tv>"

    # close the outputfile if necessary
    if output != None:
        output.close()

    # save the cache if necessary
    if program_cache != None:
        program_cache.clean()
        program_cache.dump(program_cache_file)

    # and return success
    sys.exit(0)

# allow this to be a module
if __name__ == '__main__':
    main()

# vim:tw=0:et:sw=4