Project

General

Profile

1
# CSV I/O
2

    
3
import csv
4
import _csv
5
import StringIO
6

    
7
import streams
8
import strings
9
import util
10

    
11
delims = ',;\t|`'
12
tab_padded_delims = ['\t|\t']
13
tsv_delim = '\t'
14
escape = '\\'
15

    
16
ending_placeholder = r'\n'
17

    
18
def is_tsv(dialect): return dialect.delimiter.startswith(tsv_delim)
19

    
20
def sniff(line):
21
    '''Automatically detects the dialect'''
22
    line, ending = strings.extract_line_ending(line)
23
    dialect = csv.Sniffer().sniff(line, delims)
24
    
25
    if is_tsv(dialect):
26
        # TSVs usually don't quote fields (nor doublequote embedded quotes)
27
        dialect.quoting = csv.QUOTE_NONE
28
        
29
        # Check multi-char delims using \t
30
        delim = strings.find_any(line, tab_padded_delims)
31
        if delim:
32
            dialect.delimiter = delim
33
            line_suffix = delim.rstrip('\t')
34
            if line.endswith(line_suffix): ending = line_suffix+ending
35
    else: dialect.doublequote = True # Sniffer doesn't turn this on by default
36
    dialect.lineterminator = ending
37
    
38
    return dialect
39

    
40
def stream_info(stream, parse_header=False):
41
    '''Automatically detects the dialect based on the header line.
42
    Uses the Excel dialect if the CSV file is empty.
43
    @return NamedTuple {header_line, header, dialect}'''
44
    info = util.NamedTuple()
45
    info.header_line = stream.readline()
46
    info.header = None
47
    if info.header_line != '':
48
        info.dialect = sniff(info.header_line)
49
    else: info.dialect = csv.excel # line of '' indicates EOF = empty stream
50
    
51
    if parse_header:
52
        try: info.header = reader_class(info.dialect)(
53
            StringIO.StringIO(info.header_line), info.dialect).next()
54
        except StopIteration: info.header = []
55
    
56
    return info
57

    
58
tsv_encode_map = strings.json_encode_map[:]
59
tsv_encode_map.append(('\t', r'\t'))
60
tsv_decode_map = strings.flip_map(tsv_encode_map)
61

    
62
class TsvReader:
63
    '''Unlike csv.reader, for TSVs, interprets \ as escaping a line ending but
64
    ignores it before everything else (e.g. \N for NULL).
65
    Also expands tsv_encode_map escapes.
66
    '''
67
    def __init__(self, stream, dialect):
68
        assert is_tsv(dialect)
69
        self.stream = stream
70
        self.dialect = dialect
71
    
72
    def __iter__(self): return self
73
    
74
    def next(self):
75
        record = ''
76
        ending = None
77
        while True:
78
            line = self.stream.readline()
79
            if line == '': raise StopIteration
80
            
81
            line = strings.remove_suffix(self.dialect.lineterminator, line)
82
            contents = strings.remove_suffix(escape, line)
83
            record += contents
84
            if len(contents) == len(line): break # no line continuation
85
            record += ending_placeholder
86
        
87
        # Prevent "new-line character seen in unquoted field" errors
88
        record = record.replace('\r', ending_placeholder)
89
        
90
        row = record.split(self.dialect.delimiter)
91
        return [strings.replace_all(tsv_decode_map, v) for v in row]
92

    
93
def reader_class(dialect):
94
    if is_tsv(dialect): return TsvReader
95
    else: return csv.reader
96

    
97
def make_reader(stream, dialect): return reader_class(dialect)(stream, dialect)
98

    
99
def reader_and_header(stream):
100
    '''Automatically detects the dialect based on the header line
101
    @return tuple (reader, header)'''
102
    info = stream_info(stream, parse_header=True)
103
    return (make_reader(stream, info.dialect), info.header)
104

    
105
##### csv modifications
106

    
107
# Note that these methods only work on *instances* of Dialect classes
108
csv.Dialect.__eq__ = lambda self, other: self.__dict__ == other.__dict__
109
csv.Dialect.__ne__ = lambda self, other: not (self == other)
110

    
111
__Dialect__validate_orig = csv.Dialect._validate
112
def __Dialect__validate(self):
113
        try: __Dialect__validate_orig(self)
114
        except _csv.Error, e:
115
            if str(e) == '"delimiter" must be an 1-character string': pass # OK
116
            else: raise
117
csv.Dialect._validate = __Dialect__validate
118

    
119
##### Row filters
120

    
121
class Filter:
122
    '''Wraps a reader, filtering each row'''
123
    def __init__(self, filter_, reader):
124
        self.reader = reader
125
        self.filter = filter_
126
    
127
    def __iter__(self): return self
128
    
129
    def next(self): return self.filter(self.reader.next())
130
    
131
    def close(self): pass # support using as a stream
132

    
133
std_nulls = [r'\N']
134
empty_nulls = [''] + std_nulls
135

    
136
class NullFilter(Filter):
137
    '''Translates special string values to None'''
138
    def __init__(self, reader, nulls=std_nulls):
139
        map_ = dict.fromkeys(nulls, None)
140
        def filter_(row): return [map_.get(v, v) for v in row]
141
        Filter.__init__(self, filter_, reader)
142

    
143
class StripFilter(Filter):
144
    '''Strips whitespace'''
145
    def __init__(self, reader):
146
        def filter_(row): return [v.strip() for v in row]
147
        Filter.__init__(self, filter_, reader)
148

    
149
class ColCtFilter(Filter):
150
    '''Gives all rows the same # columns'''
151
    def __init__(self, reader, cols_ct):
152
        def filter_(row): return util.list_as_length(row, cols_ct)
153
        Filter.__init__(self, filter_, reader)
154

    
155
##### Translators
156

    
157
class StreamFilter(Filter):
158
    '''Wraps a reader in a way that's usable to a filter stream that does not
159
    require lines to be strings. Reports EOF as '' instead of StopIteration.'''
160
    def __init__(self, reader):
161
        Filter.__init__(self, None, reader)
162
    
163
    def readline(self):
164
        try: return self.reader.next()
165
        except StopIteration: return '' # EOF
166

    
167
class ColInsertFilter(Filter):
168
    '''Adds a column to each row
169
    @param mk_value(row, row_num)
170
    '''
171
    def __init__(self, reader, mk_value, index=0):
172
        def filter_(row):
173
            row = list(row) # make sure it's mutable; don't modify input!
174
            row.insert(index, mk_value(row, self.reader.line_num))
175
            return row
176
        Filter.__init__(self, filter_,
177
            streams.LineCountInputStream(StreamFilter(reader)))
178

    
179
class RowNumFilter(ColInsertFilter):
180
    '''Adds a row # column at the beginning of each row'''
181
    def __init__(self, reader):
182
        def mk_value(row, row_num): return row_num
183
        ColInsertFilter.__init__(self, reader, mk_value, 0)
184

    
185
class InputRewriter(StreamFilter):
186
    '''Wraps a reader, writing each row back to CSV'''
187
    def __init__(self, reader, dialect=csv.excel):
188
        StreamFilter.__init__(self, reader)
189
        
190
        self.dialect = dialect
191
    
192
    def readline(self):
193
        row = self.reader.readline()
194
        if row == '': return row # EOF
195
        
196
        line_stream = StringIO.StringIO()
197
        csv.writer(line_stream, self.dialect).writerow(row)
198
        return line_stream.getvalue()
199
    
200
    def read(self, n): return self.readline() # forward all reads to readline()
(10-10/45)