#!/usr/bin/env python

# A DiGIR client

import os

import os.path

import sys

import urllib

import urllib2

import xml.dom.minidom as minidom

sys.path.append(os.path.dirname(__file__)+"/../lib")

import dates

import http

import opts

import profiling

import streams

import util

import xml_dom

import xpath

# Config

timeout = 20 # sec

default_chunk_size = 10000 # records

schema = 'http://digir.net/schema/conceptual/darwin/full/2003/1.0/darwin2full.xsd'

request_xml_template = '''\

<?xml version="1.0" encoding="UTF-8"?>

<request

    xmlns="http://digir.net/schema/protocol/2003/1.0"

    xmlns:xsd="http://www.w3.org/2001/XMLSchema"

    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

    xmlns:digir="http://digir.net/schema/protocol/2003/1.0"

    xmlns:darwin="http://digir.net/schema/conceptual/darwin/2003/1.0"

    xmlns:dwc="http://digir.net/schema/conceptual/darwin/2003/1.0"

    xsi:schemaLocation="http://digir.net/schema/protocol/2003/1.0 

      http://digir.sourceforge.net/schema/protocol/2003/1.0/digir.xsd 

      http://digir.net/schema/conceptual/darwin/2003/1.0 

      http://digir.sourceforge.net/schema/conceptual/darwin/2003/1.0/darwin2.xsd">

    <header>

        <version>1.0</version>

        <sendTime>[time]</sendTime>

        <source>[source]</source>

        <destination resource="[resource]">[url]</destination>

        <type>search</type>

    </header>

    <search>

        <filter>

            <equals>

                <darwin:Kingdom>plantae</darwin:Kingdom>

            </equals>

        </filter>

        <records limit="[count]" start="[start]">

            <structure schemaLocation="[schema]"/>

        </records>

        <count>true</count>

    </search>

</request>

'''

diags_start = '<diagnostics>'

diags_end = '</diagnostics>'

class InputError(Exception): pass

def main():

    # Usage

    env_names = []

    def usage_err():

        raise SystemExit('Usage: '+opts.env_usage(env_names, True)+' '

            +sys.argv[0]+' 2>>log')

    # Get config from env vars

    url = opts.get_env_var('url', None, env_names)

    resource = opts.get_env_var('resource', None, env_names)

    start = util.cast(int, opts.get_env_var('start', 0, env_names))

    count = util.cast(int, opts.get_env_var('n', None, env_names))

    chunk_size = util.cast(int, opts.get_env_var('chunk_size',

        default_chunk_size, env_names))

    debug = opts.env_flag('debug', False, env_names)

    if url == None or resource == None: usage_err()

    # Logging

    def clear_line(): sys.stderr.write('\n')

    log_indent = 0

    def log(msg, line_ending='\n'): sys.stderr.write(msg+line_ending)

    def debug_log(str_, label=None):

        if debug:

            if label != None: sys.stderr.write(label+':\n')

            sys.stderr.write(str_+'\n')

    # Request XML

    self_dir = os.path.dirname(__file__)

    source = os.popen(self_dir+"/local_ip").read().strip()

    this_request_xml_template = (request_xml_template

        .replace('[source]', source)

        .replace('[url]', url)

        .replace('[resource]', resource)

        .replace('[schema]', schema)

        )

    # Stats

    total = 0

    def print_status(line_ending='\n'):

        log('Processed '+str(total)+' record(s)', line_ending)

    match_ct = None

    profiler = profiling.ItersProfiler(start_now=True, iter_text='record')

    # Retrieve data

    while count == None or total < count:

        # Adjust chunk size if last chunk

        this_count = chunk_size

        if count != None: this_count = min(this_count, count - total)

        # Request XML

        time = dates.strftime('%Y-%m-%d %H:%M:%S %Z', dates.now())

        request_xml = (this_request_xml_template

            .replace('[count]', str(this_count))

            .replace('[start]', str(start))

            .replace('[time]', time)

            )

        debug_log(request_xml, 'request')

        # Send request

        this_url = url+'?'+urllib.urlencode({'request': request_xml})

        stream = streams.CaptureStream(streams.TimeoutInputStream(

            urllib2.urlopen(this_url), timeout), diags_start, diags_end)

        # Retrieve response

        streams.copy(stream, sys.stdout)

        # Make sure output ends in a newline so that consecutive XML documents

        # are on different lines

        sys.stdout.write('\n')

        stream.close()

        # Parse diagnostics

        diags_str = stream.matches[0]

        debug_log(diags_str, 'diagnostics')

        diags = xml_dom.parse_str(diags_str)

        def get_diag(name):

            return xpath.get_value(diags, 'diagnostic[@code='+name+']')

        # Process match count

        this_match_ct = util.cast(int, get_diag('MATCH_COUNT'))

        if this_match_ct != match_ct: # first or updated match count

            match_ct = this_match_ct

            log('Found '+str(match_ct)+' record(s)')

        # Process record count

        this_ct = util.cast(int, get_diag('RECORD_COUNT'))

        if this_ct == None: raise InputError('Missing RECORD_COUNT diagnostic')

        total += this_ct

        start += this_ct # advance start to fetch next set

        print_status('\r') # CR at end so next print overwrites msg

        # Decide if done

        if this_ct == 0 or get_diag('END_OF_RECORDS') == 'true': break

    print_status()

    profiler.stop(total)

    log(profiler.msg())

main()