#!/usr/bin/env python

# Downloads REMIB data for all nodes

# Usage: self

import csv

import itertools

import os.path

import StringIO

import sys

import urllib2

sys.path.append(os.path.dirname(__file__)+"/../../../lib")

import exc

import profiling

import streams

import strings

import timeout

import util

timeout_ = 20 # sec

alphabet = map(chr, xrange(ord('A'), ord('Z')+1))

class InputException(Exception): pass

def is_ignore(line):

    line = strings.remove_line_ending(line)

    return line == '' or line.startswith('\t') or line.find(',') < 0

def main():

    def clear_line(): sys.stderr.write('\n')

    log_indent = 0

    def log(msg, line_ending='\n'):

        sys.stderr.write(('    '*log_indent)+msg+line_ending)

    os.chdir(os.path.dirname(__file__)) # dir of output files

    # Get by family ('familia') because that is the most general level at which

    # an identification can be made. This assumes all records have a family.

    url_template = ('http://www.conabio.gob.mx/remib/cgi-bin/'

        'remib_distribucion.cgi?lengua=EN&niveltax=familia&taxon=[prefix]%25&'

        'pais=Todos&pais_otro=&estado=100&formato=csv&mapa=no&mapabase=estados'

        '&coleccion=id%3D[node_id]')

    done = False

    for node_id in itertools.count(1):

        if done: break

        log('Processing node #'+str(node_id)+'...')

        log_indent += 1

        profiler = profiling.ItersProfiler(start_now=True, iter_text='row')

        filename = 'node.'+str(node_id)+'.specimens.csv'

        out = streams.LineCountOutputStream(open(filename, 'w'))

        start_line_num = out.line_num

        node_url_template = url_template.replace('[node_id]', str(node_id))

        for prefix_chars in itertools.product(alphabet, repeat=2):

            if done: break

            prefix = ''.join(prefix_chars)

            log('Processing prefix '+prefix+'...')

            row_ct = 0

            def print_status(line_ending='\n'):

                log('Processed '+str(row_ct)+' row(s)', line_ending)

            log_indent += 1

            url = node_url_template.replace('[prefix]', prefix)

            stream = streams.StreamIter(streams.TimeoutInputStream(

                urllib2.urlopen(url), timeout_))

            # Copy lines

            try:

                util.skip(stream, is_ignore) # skip header

                try:

                    metadata_row = csv.reader(stream).next()

                    if metadata_row[0] != 'COLLECTION': raise InputException(

                        'Invalid metadata row: '+str(metadata_row))

                except StopIteration:

                    done = True # empty response means no more nodes

                for line in stream:

                    if is_ignore(line):

                        error = strings.remove_prefix('\t\t', line)

                        if len(error) != len(line): raise InputException(error)

                        break

                    if row_ct % 100 == 0: print_status('\r')

                        # CR at end so next print overwrites msg

                    out.write(line)

                    row_ct += 1

            except (InputException, timeout.TimeoutException), e:

                clear_line()

                log('! Output line '+str(out.line_num)+': '+exc.str_(e))

            stream.close()

            profiler.add_iters(row_ct)

            print_status()

            log_indent -= 1

        profiler.stop()

        log(profiler.msg())

        log_indent -= 1

main()