Project

General

Profile

« Previous | Next » 

Revision 14

Added by Aaron Marcuse-Kubitza about 13 years ago

Changed xml2db to avoid inserting duplicate rows

View differences:

scripts/xml2db/db_util.py
6 6 

  		




  7
  7
  
    
import ex_util


  		




  8
  8
  
    

  		




  
  9
  
    
def _add_cursor_info(ex, cur): ex_util.add_msg(ex, 'query: '+cur.query)


  		




  
  10
  
    

  		




  9
  11
  
    
class NameException(Exception): pass


  		




  10
  12
  
    

  		




  11
  
  
    
class ExceptionWithColumn(ex_util.ExceptionWithCause):


  		




  
  13
  
    
class DbException(ex_util.ExceptionWithCause):


  		




  
  14
  
    
    def __init__(self, msg, cause=None, cur=None):


  		




  
  15
  
    
        ex_util.ExceptionWithCause.__init__(self, msg, cause)


  		




  
  16
  
    
        if cur != None: _add_cursor_info(self, cur)


  		




  
  17
  
    

  		




  
  18
  
    
class ExceptionWithColumn(DbException):


  		




  12
  19
  
    
    def __init__(self, col, cause=None):


  		




  13
  
  
    
        ex_util.ExceptionWithCause.__init__(self, 'column: '+col, cause)


  		




  
  20
  
    
        DbException.__init__(self, 'column: '+col, cause)


  		




  14
  21
  
    
        self.col = col


  		




  15
  22
  
    

  		




  16
  23
  
    
class DuplicateKeyException(ExceptionWithColumn): pass


  		




  ......




  25
  32
  
    
    cur = db.cursor()


  		




  26
  33
  
    
    try: cur.execute(query, params)


  		




  27
  34
  
    
    except Exception, ex:


  		




  28
  
  
    
        ex_util.add_msg(ex, 'query: '+cur.query)


  		




  
  35
  
    
        _add_cursor_info(ex, cur)


  		




  29
  36
  
    
        raise


  		




  30
  37
  
    
    return cur


  		




  31
  38
  
    

  		




  32
  
  
    
def row(cur): return cur.fetchone()


  		




  
  39
  
    
def row(cur): return iter(lambda: cur.fetchone(), None).next()


  		




  33
  40
  
    

  		




  34
  41
  
    
def value(cur): return row(cur)[0]


  		




  35
  42
  
    

  		




  ......




  66
  73
  
    

  		




  67
  74
  
    
def last_insert_id(db): return value(run_query(db, 'SELECT lastval()'))


  		




  68
  75
  
    

  		




  69
  
  
    
def insert_ignore(db, table, row):


  		




  
  76
  
    
def try_insert(db, table, row):


  		




  70
  77
  
    
    try: return with_savepoint(db, lambda: insert(db, table, row))


  		




  71
  78
  
    
    except Exception, ex:


  		




  72
  79
  
    
        msg = str(ex)


  		




  ......




  79
  86
  
    
        raise # no specific exception raised


  		




  80
  87
  
    

  		




  81
  88
  
    
def insert_or_get(db, table, row, pkey, row_ct_ref=None):


  		




  82
  
  
    
    try:


  		




  83
  
  
    
        row_ct = insert_ignore(db, table, row).rowcount


  		




  84
  
  
    
        if row_ct_ref != None and row_ct >= 0: row_ct_ref[0] += row_ct


  		




  85
  
  
    
        return last_insert_id(db)


  		




  86
  
  
    
    except DuplicateKeyException, ex:


  		




  87
  
  
    
        return value(select(db, table, [pkey], {ex.col: row[ex.col]}))


  		




  
  89
  
    
    try: return value(select(db, table, [pkey], row))


  		




  
  90
  
    
    except StopIteration:


  		




  
  91
  
    
        try:


  		




  
  92
  
    
            row_ct = try_insert(db, table, row).rowcount


  		




  
  93
  
    
            if row_ct_ref != None and row_ct >= 0: row_ct_ref[0] += row_ct


  		




  
  94
  
    
            return last_insert_id(db)


  		




  
  95
  
    
        except DuplicateKeyException, ex:


  		




  
  96
  
    
            return value(select(db, table, [pkey], {ex.col: row[ex.col]}))


  		












  

    
      scripts/xml2db/xml_db.py
    
  





  74
  74
  
    
                break


  		




  75
  75
  
    
            except db_util.NullValueException, ex:


  		




  76
  76
  
    
                if try_num > 0: raise # exception still raised after retry


  		




  
  77
  
    
                # Search for required column in ancestors and their children


  		




  77
  78
  
    
                target = find_by_name(node, ptr_type(ex.col))


  		




  78
  79
  
    
                if target == None: raise


  		




  79
  80
  
    
                row[ex.col] = xml_util.get_id(target)


  		












Also available in:  Unified diff