<?php

#############################################################################

# Driver file for taiper2flatClient, a simple script to 

# consume taiper web services and save the results in a delimited file,

# one record per line.  The current delimiter is comma ',' but

# that can be changed by changing the $seperator variable located

# in configurableParams.php.

#

# Command line useage:

#   php taiper2flat.php

#

# dependencies:

# php 5+

# pear

# HTTP_Request

# I had to install pear and HTTP_Request before it would work (on Ubuntu):

#   sudo apt-get install php-http-request

# This should install all dependencies, but just in case, the full

# dependency list is:

#   php-net-socket 

#   php-net-url

#   php-pear

#   php-http-request

#############################################################################

require_once('HTTP/Request.php'); // pear package

require_once('tapirRequestTemplate.php'); 

require_once('getAllConcepts.php'); 

global $url;

global $seperator;

$flat_filename = "specieslink.txt";

$record_num_filename = "recordnum.dat";

$error_log_filename = "error.log";

$supportedConcepts = getAllConcepts();

if(!is_array($supportedConcepts)){

  $error_log = fopen($error_log_filename,"a");

  fwrite($error_log, "Unable to retrieve supported concepts from service, failing.\n");

  fclose($error_log);

  exit;

}

$successive_failures = 0;

$start_record = 0;	// Record to begin first request (ordinal position, not ID)

if(file_exists($record_num_filename)) {

  $start_record = file_get_contents($record_num_filename);

}

$request_limit = 1000;		// Number of records per request; cannot exceed 1000

//According to TAPIR docs, the estimate from the service 

//is an upper bound on the number of records.  Start out high.

$estimated_max_from_service = 4000000;  

if(!file_exists($flat_filename)) {

  $flat_file = fopen($flat_filename,"a");

  $file_header = '';

  foreach(array_keys($supportedConcepts) as $key){

    $file_header .= $key . $seperator;

  }

  $file_header = substr_replace($file_header ,"",-1);

  $file_header .= "\n";

  fwrite($flat_file,$file_header);

  fclose($flat_file);

}

$finished = false;

$itrNum = 0;

$http_request = new HTTP_Request();

while(!$finished && $start_record < $estimated_max_from_service) {

    $itrNum++;

    $body = buildRequest($start_record,$request_limit,$supportedConcepts);

    $http_request->setMethod( 'POST' );

    $http_request->addHeader('Content-Type', 'text/xml');

    $http_request->addRawPostData( $body );

    $http_request->setURL( $url );

    $http_request->_timeout = 300;

    $http_request->_readTimeout = 300;

    // This can be used to see the entire request

    #$raw_request = $http_request->_buildRequest();

    #echo "\n\n" . $raw_request;

    $http_request->sendRequest();

    $response = $http_request->getResponseBody();

    $code = $http_request->getResponseCode();

    if ( $code != 200 ) // 200 = OK

    {

        $label = 'Unknown Error';

        switch ( $code )

        {

            case 201: $label = 'Created'; break;

            case 202: $label = 'Accepted'; break;

            case 203: $label = 'Non-Authoritative Information'; break;

            case 204: $label = 'No Content'; break;

            case 205: $label = 'Reset Content'; break;

            case 206: $label = 'Partial Content'; break;

            case 300: $label = 'Multiple Choices'; break;

            case 301: $label = 'Moved Permanently'; break;

            case 302: $label = 'Found'; break;

            case 303: $label = 'See Other'; break;

            case 304: $label = 'Not Modified'; break;

            case 305: $label = 'Use Proxy'; break;

            case 307: $label = 'Temporary Redirect'; break;

            case 400: $label = 'Bad Request'; break;

            case 401: $label = 'Unauthorized'; break;

            case 402: $label = 'Payment Required'; break;

            case 403: $label = 'Forbidden'; break;

            case 404: $label = 'Not Found'; break;

            case 405: $label = 'Method Not Allowed'; break;

            case 406: $label = 'Not Acceptable'; break;

            case 407: $label = 'Proxy Authentication Required'; break;

            case 408: $label = 'Request Timeout'; break;

            case 409: $label = 'Conflict'; break;

            case 410: $label = 'Gone'; break;

            case 411: $label = 'Length Required'; break;

            case 412: $label = 'Precondition Failed'; break;

            case 413: $label = 'Request Entity Too Large'; break;

            case 414: $label = 'Request-URI Too Long'; break;

            case 415: $label = 'Unsupported Media Type'; break;

            case 416: $label = 'Requested Range Not Satisfiable'; break;

            case 417: $label = 'Expectation Failed'; break;

            case 500: $label = 'Internal Server Error'; break;

            case 501: $label = 'Not Implemented'; break;

            case 502: $label = 'Bad Gateway'; break;

            case 503: $label = 'Service Unavailable'; break;

            case 504: $label = 'Gateway Timeout'; break;

            case 505: $label = 'HTTP Version Not Supported'; break;

	}

       $error_log = fopen($error_log_filename,"a");

       fwrite($error_log, "Service responded with HTTP ".$code." code: ".$label."\n");

       $successive_failures += 1;

       if($successive_failures >= 3) {

         fwrite($error_log, "3 successive failures, quitting\n");

         fclose($error_log);

         exit;

       }

       fclose($error_log);

       //Should repeat the same request

       continue;

    }

    #Weird encoding bug - need to remove the ^F and ^L characters so

    #that the xml parser won't choke.

    $xmlDoc = new DOMDocument();

    $response = preg_replace("//","6",$response);

    $response = preg_replace("//","12",$response);

    $xmlDoc->loadXML($response);

    $errors = $xmlDoc->getElementsByTagName("error");

    if($errors->length > 0) {

       $error_log = fopen($error_log_filename,"a");

       foreach($errors as $error) {

         fwrite($error_log, $error->nodeValue . "\n");

       }

       $successive_failures += 1;

       if($successive_failures >= 3) {

         fwrite($error_log, "3 successive failures, quitting\n");

         fclose($error_log);

         exit;

       }

       fclose($error_log);

       //Should repeat the same request

       continue;

    }

    $summary = $xmlDoc->getElementsByTagName("summary");

    if($summary->length == 0) {

       $error_log = fopen($error_log_filename,"a");

       fwrite($error_log, "No summary node, assuming there's a missed error.\n");

       $successive_failures += 1;

       if($successive_failures >= 3) {

         fwrite($error_log, "3 successive failures, quitting\n");

         fclose($error_log);

         exit;

       }

       fclose($error_log);

       continue;

    } else {

      $start_record_temp = $summary->item(0)->getAttribute("next");

      $estimated_max_from_service = $summary->item(0)->getAttribute("totalMatched");

    }

    $flat_file = fopen($flat_filename,"a");

    $records = $xmlDoc->getElementsByTagName("record");

    $recordsStr = '';

    foreach($records as $record) {

      $fields = array();

      foreach(array_keys($supportedConcepts) as $key){

        $element = $record->getElementsByTagName($key);

        if($element->length > 0) {

          $fields[] = $element->item(0)->nodeValue;

        }

      }

      fputcsv($flat_file,$fields,$seperator);

    }

    fwrite($flat_file,$recordsStr);

    fclose($flat_file);

    $successive_failures = 0;

    $record_file = fopen($record_num_filename,"w");

    fwrite($record_file,$start_record_temp);

    fclose($record_file);

    $start_record = $start_record_temp;    

    //If number of records is less than request_limit, it means

    //the service is out of records.

    $finished = ($request_limit > $records->length);

    if($itrNum == 1) {

      print "Extimated number of records:  $estimated_max_from_service\n";

    } elseif($itrNum % 10 == 0) {

      print "Pulled $start_record records out of $estimated_max_from_service ".

            "estimated total records.\n";

    }

}

?>

<?php

#############################################################################

# Builds the xml body to be sent to Tapir service.  Only put in a 

# seperate file because there is a lot of code-invarient text that would

# just make other parts of the script difficult to read.

#############################################################################

require_once('configurableParams.php');

function buildStructure($supportedConcepts) {

  $structureStr = '';

  foreach(array_keys($supportedConcepts) as $key){

    $structureStr .= 

      "              <xs:element name=\"$key\" type=\"xs:string\" minOccurs=\"0\"/>\n";

  }

  return $structureStr;

}

function buildMap($supportedConcepts) {

  $mapStr = '';

  foreach(array_keys($supportedConcepts) as $key){

    $mapStr .=

      "        <node path=\"/records/record/$key\">\n".

      "          <concept id=\"$supportedConcepts[$key]\"/>\n".

      "        </node>\n";

  }

  return $mapStr;

}

function buildRequest($start,$limit,$supportedConcepts) {

  $xmlRequest = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n".

  "<request \n".

  "    xmlns=\"http://rs.tdwg.org/tapir/1.0\"\n".

  "    xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n".

  "    xsi:schemaLocation=\"http://rs.tdwg.org/tapir/1.0 \n".

  "                        http://rs.tdwg.org/tapir/1.0/schema/tapir.xsd\">\n".

  "  <header>\n".

  "  </header>\n".

  "  <search count=\"true\" start=\"$start\" limit=\"$limit\" envelope=\"true\">\n".

  "    <outputModel>\n".

  "      <structure>\n".

  "        <xs:schema targetNamespace=\"http://example.net/simple_specimen\" xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xsi:schemaLocation=\"http://www.w3.org/2001/XMLSchema http://www.w3.org/2001/XMLSchema.xsd\">\n".

  "          <xs:element name=\"records\">\n".

  "            <xs:complexType>\n".

  "              <xs:sequence>\n".

  "                <xs:element name=\"record\" minOccurs=\"0\" maxOccurs=\"unbounded\" type=\"unitType\">\n".

  "                </xs:element>\n".

  "              </xs:sequence>\n".

  "            </xs:complexType>\n".

  "          </xs:element>\n".

  "          <xs:complexType name=\"unitType\">\n".

  "            <xs:sequence>\n".

               buildStructure($supportedConcepts).

  "            </xs:sequence>\n".

  "          </xs:complexType>\n".

  "        </xs:schema>\n".

  "      </structure>\n".

  "      <indexingElement path=\"/records/record\"/>\n".

  "      <mapping>\n".

         buildMap($supportedConcepts).

  "      </mapping>\n".

  "    </outputModel>\n".

  "    <filter>\n".

         buildFilter().

  "    </filter>\n".

  "  </search>\n".

  "</request>\n";

  return $xmlRequest;

}

function buildCapabilitiesRequest() {

  $xmlRequest = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n".

  "<request \n".

  "    xmlns=\"http://rs.tdwg.org/tapir/1.0\"\n".

  "    xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n".

  "    xsi:schemaLocation=\"http://rs.tdwg.org/tapir/1.0 \n".

  "                        http://rs.tdwg.org/tapir/1.0/schema/tapir.xsd\">\n".

  "  <header>\n".

  "  </header>\n".

  "  <capabilities />\n".

  "</request>\n";

  return $xmlRequest;

}

?>

<?php

#############################################################################

# File containing all the paramaters that could potentially need to be

# configured for the job.

#

#  $url:  The url for the desired Taiper service

#  $seperator:  The delimiter for the flat file.

#  buildFilter():  Function to specify the search paramaters.

#  desiredConcepts:  Desired data to be returned by the service.

#

#############################################################################

# The url of the desired Taiper service

$url = 'http://tapir.cria.org.br/tapirlink/tapir.php/specieslink';

# The seperator to be used in the flat file to seperate the 

# data fields of each record.

$seperator = ',';

# Tapir services have an extremely flexable syntax to search for

# records with the desired properties.  However, this flexibility

# makes it rather difficult to generalize the search possibilities

# and quickly come up with appropriate xml.  As such, the current

# solution is to build the filter node here.  See the associated

# documentation on how to build the filter:

#   http://www.tdwg.org/dav/subgroups/tapir/1.0/docs/tdwg_tapir_specification_2010-05-05.htm#toc47

function buildFilter() {

  $filter = 

        '<equals>' .

          '<concept id="http://rs.tdwg.org/dwc/dwcore/Kingdom"/>' .

          '<literal value="Plantae"/>' .

        '</equals>';

  return $filter;

}

?>

<?php

#############################################################################

# This file holds the utility function getAllConcepts which calls the 

# tapir service to find out all the concepts (data units) that are 

# supported.  This list is necessary so that we know what to ask for

# when making the rquest.

#############################################################################

require_once('HTTP/Request.php'); // pear package

require_once('tapirRequestTemplate.php'); 

function getAllConcepts() {

    global $url;

    $error_log_filename = "error.log";

    $http_request = new HTTP_Request();

    $body = buildCapabilitiesRequest();

    $http_request->setMethod( 'POST' );

    $http_request->addHeader('Content-Type', 'text/xml');

    $http_request->addRawPostData( $body );

    $http_request->setURL( $url );

    $http_request->_timeout = 300;

    $http_request->_readTimeout = 300;

    // This can be used to see the entire request

    #$raw_request = $http_request->_buildRequest();

    #echo "\n\n" . $raw_request;

    $http_request->sendRequest();

    $response = $http_request->getResponseBody();

    $code = $http_request->getResponseCode();

    if ( $code != 200 ) // 200 = OK

    {

        $label = 'Unknown Error';

        switch ( $code )

        {

            case 201: $label = 'Created'; break;

            case 202: $label = 'Accepted'; break;

            case 203: $label = 'Non-Authoritative Information'; break;

            case 204: $label = 'No Content'; break;

            case 205: $label = 'Reset Content'; break;

            case 206: $label = 'Partial Content'; break;

            case 300: $label = 'Multiple Choices'; break;

            case 301: $label = 'Moved Permanently'; break;

            case 302: $label = 'Found'; break;

            case 303: $label = 'See Other'; break;

            case 304: $label = 'Not Modified'; break;

            case 305: $label = 'Use Proxy'; break;

            case 307: $label = 'Temporary Redirect'; break;

            case 400: $label = 'Bad Request'; break;

            case 401: $label = 'Unauthorized'; break;

            case 402: $label = 'Payment Required'; break;

            case 403: $label = 'Forbidden'; break;

            case 404: $label = 'Not Found'; break;

            case 405: $label = 'Method Not Allowed'; break;

            case 406: $label = 'Not Acceptable'; break;

            case 407: $label = 'Proxy Authentication Required'; break;

            case 408: $label = 'Request Timeout'; break;

            case 409: $label = 'Conflict'; break;

            case 410: $label = 'Gone'; break;

            case 411: $label = 'Length Required'; break;

            case 412: $label = 'Precondition Failed'; break;

            case 413: $label = 'Request Entity Too Large'; break;

            case 414: $label = 'Request-URI Too Long'; break;

            case 415: $label = 'Unsupported Media Type'; break;

            case 416: $label = 'Requested Range Not Satisfiable'; break;

            case 417: $label = 'Expectation Failed'; break;

            case 500: $label = 'Internal Server Error'; break;

            case 501: $label = 'Not Implemented'; break;

            case 502: $label = 'Bad Gateway'; break;

            case 503: $label = 'Service Unavailable'; break;

            case 504: $label = 'Gateway Timeout'; break;

            case 505: $label = 'HTTP Version Not Supported'; break;

	}

       $error_log = fopen($error_log_filename,"a");

       fwrite($error_log, "Service responded with HTTP ".$code." code: ".$label."\n".

              "while attempting to request supported concepts\n");

       fclose($error_log);

       echo( "Service responded with HTTP ".$code." code: ".$label."\n".

              "while attempting to request supported concepts\n");

       return 0;

    }

    #Weird encoding bug - need to remove the ^F and ^L characters so

    #that the xml parser won't choke.

    $xmlDoc = new DOMDocument();

    $response = preg_replace("//","6",$response);

    $response = preg_replace("//","12",$response);

    $xmlDoc->loadXML($response);

    $errors = $xmlDoc->getElementsByTagName("error");

    if($errors->length > 0) {

       $error_log = fopen($error_log_filename,"a");

       foreach($errors as $error) {

         fwrite($error_log, $error->nodeValue . "\n");

       }

       fclose($error_log);

       return 0;

    }

    $concepts = $xmlDoc->getElementsByTagName("mappedConcept");

    $concepts_array = array();

    if($concepts->length == 0) {

       $error_log = fopen($error_log_filename,"a");

       fwrite($error_log, "No mappedConcept node, assuming there's a missed error.\n");

       fclose($error_log);

       return 0;

    } else {

      foreach($concepts as $concept) {

        $concept_id = $concept->getAttribute("id");

        $concept_key = preg_replace("/\//","_",$concept_id);

        $concept_key = preg_replace("/\./","_",$concept_key);

        $concept_key = preg_replace("/:/","",$concept_key);

        $concepts_array[$concept_key] = $concept_id;

      }

    }

    return $concepts_array;

}

?>

tapir2flat.php:  a simple script to 

consume taiper web services and save the results in a delimited file,

one record per line.  The current delimiter is backtick '`' but

that can be changed by changing the $seperator variable located

in configurableParams.php.

Command line useage:

  php taiper2flat.php

dependencies:

  php 5+

  pear

  HTTP_Request

I had to install pear and HTTP_Request before it would work (on Ubuntu):

  sudo apt-get install php-http-request

This should install all dependencies, but just in case, the full dependency list is:

  php-net-socket 

  php-net-url

  php-pear

  php-http-request

Necessary files:

  tapir2flat.php: Driver file

  configurableParams.php: File containing paramaters that can be configured

  tapirRequestTemplate.php: Holds the template required to send a tapir request.

  getAllConcepts.php: holds utility function to get all possible return values from tapir service.

Possible output files:

  specieslink.txt:  Where the retrieved records will be stored.  Each line represents one record, delimited by whatever is configured as the delimitation paramater.

  recordnum.dat:  Holds the starting record number for the most recently sent request.

  error.log:  Where any errors will be recorded.

Possible configurations: see configurableParams.php as well.

  The url of desired tapir service

  Delimiter to be used to sperate data fields within each record

  The filter sent to tapir service that specifies search parameters, in xml format.  See http://www.tdwg.org/dav/subgroups/tapir/1.0/docs/tdwg_tapir_specification_2010-05-05.htm#toc47 for documentation on how to do this.