Project

General

Profile

1 1388 aaronmk
# CSV I/O
2
3
import csv
4 5431 aaronmk
import _csv
5 1388 aaronmk
import StringIO
6
7 14594 aaronmk
import dicts
8 8069 aaronmk
import exc
9 14592 aaronmk
import lists
10 5593 aaronmk
import streams
11 1623 aaronmk
import strings
12 1444 aaronmk
import util
13
14 5426 aaronmk
delims = ',;\t|`'
15 5439 aaronmk
tab_padded_delims = ['\t|\t']
16 1623 aaronmk
tsv_delim = '\t'
17
escape = '\\'
18 1442 aaronmk
19 1623 aaronmk
ending_placeholder = r'\n'
20
21 5437 aaronmk
def is_tsv(dialect): return dialect.delimiter.startswith(tsv_delim)
22 1623 aaronmk
23 1621 aaronmk
def sniff(line):
24
    '''Automatically detects the dialect'''
25 5435 aaronmk
    line, ending = strings.extract_line_ending(line)
26 9961 aaronmk
    try: dialect = csv.Sniffer().sniff(line, delims)
27
    except _csv.Error, e:
28
        if exc.e_msg(e) == 'Could not determine delimiter': dialect = csv.excel
29
        else: raise
30 5435 aaronmk
31 5439 aaronmk
    if is_tsv(dialect):
32 8070 aaronmk
        dialect.quoting = csv.QUOTE_NONE
33 5439 aaronmk
        # Check multi-char delims using \t
34
        delim = strings.find_any(line, tab_padded_delims)
35
        if delim:
36
            dialect.delimiter = delim
37
            line_suffix = delim.rstrip('\t')
38
            if line.endswith(line_suffix): ending = line_suffix+ending
39 1621 aaronmk
    else: dialect.doublequote = True # Sniffer doesn't turn this on by default
40 5435 aaronmk
    dialect.lineterminator = ending
41
42 1621 aaronmk
    return dialect
43
44 8202 aaronmk
def has_unbalanced_quotes(str_): return str_.count('"') % 2 == 1 # odd # of "
45
46
def has_multiline_column(str_): return has_unbalanced_quotes(str_)
47
48 1923 aaronmk
def stream_info(stream, parse_header=False):
49 6589 aaronmk
    '''Automatically detects the dialect based on the header line.
50
    Uses the Excel dialect if the CSV file is empty.
51 1923 aaronmk
    @return NamedTuple {header_line, header, dialect}'''
52 1444 aaronmk
    info = util.NamedTuple()
53
    info.header_line = stream.readline()
54 8202 aaronmk
    if has_multiline_column(info.header_line): # 1st line not full header
55
        # assume it's a header-only csv with multiline columns
56
        info.header_line += ''.join(stream.readlines()) # use entire file
57 1923 aaronmk
    info.header = None
58
    if info.header_line != '':
59
        info.dialect = sniff(info.header_line)
60 6589 aaronmk
    else: info.dialect = csv.excel # line of '' indicates EOF = empty stream
61
62
    if parse_header:
63
        try: info.header = reader_class(info.dialect)(
64
            StringIO.StringIO(info.header_line), info.dialect).next()
65
        except StopIteration: info.header = []
66
67 1444 aaronmk
    return info
68
69 5170 aaronmk
tsv_encode_map = strings.json_encode_map[:]
70
tsv_encode_map.append(('\t', r'\t'))
71
tsv_decode_map = strings.flip_map(tsv_encode_map)
72 5146 aaronmk
73 1623 aaronmk
class TsvReader:
74
    '''Unlike csv.reader, for TSVs, interprets \ as escaping a line ending but
75 5145 aaronmk
    ignores it before everything else (e.g. \N for NULL).
76 5170 aaronmk
    Also expands tsv_encode_map escapes.
77 5145 aaronmk
    '''
78 1623 aaronmk
    def __init__(self, stream, dialect):
79
        assert is_tsv(dialect)
80
        self.stream = stream
81
        self.dialect = dialect
82
83
    def __iter__(self): return self
84
85
    def next(self):
86
        record = ''
87
        ending = None
88
        while True:
89
            line = self.stream.readline()
90
            if line == '': raise StopIteration
91
92 5438 aaronmk
            line = strings.remove_suffix(self.dialect.lineterminator, line)
93 5433 aaronmk
            contents = strings.remove_suffix(escape, line)
94 1623 aaronmk
            record += contents
95 5433 aaronmk
            if len(contents) == len(line): break # no line continuation
96 1623 aaronmk
            record += ending_placeholder
97 3055 aaronmk
98
        # Prevent "new-line character seen in unquoted field" errors
99
        record = record.replace('\r', ending_placeholder)
100
101 7211 aaronmk
        # Split line
102 8071 aaronmk
        if record == '': row = [] # csv.reader would interpret as EOF
103
        elif len(self.dialect.delimiter) > 1: # multi-char delims
104 7211 aaronmk
            row = record.split(self.dialect.delimiter)
105
        else: row = csv.reader(StringIO.StringIO(record), self.dialect).next()
106
107 5170 aaronmk
        return [strings.replace_all(tsv_decode_map, v) for v in row]
108 1623 aaronmk
109
def reader_class(dialect):
110
    if is_tsv(dialect): return TsvReader
111
    else: return csv.reader
112
113
def make_reader(stream, dialect): return reader_class(dialect)(stream, dialect)
114
115 1388 aaronmk
def reader_and_header(stream):
116
    '''Automatically detects the dialect based on the header line
117
    @return tuple (reader, header)'''
118 1923 aaronmk
    info = stream_info(stream, parse_header=True)
119 1958 aaronmk
    return (make_reader(stream, info.dialect), info.header)
120 1446 aaronmk
121 14577 aaronmk
def header(stream):
122
    '''fetches just the header line of a stream'''
123
    reader, header = reader_and_header(stream)
124
    return header
125
126 1446 aaronmk
##### csv modifications
127
128
# Note that these methods only work on *instances* of Dialect classes
129
csv.Dialect.__eq__ = lambda self, other: self.__dict__ == other.__dict__
130
csv.Dialect.__ne__ = lambda self, other: not (self == other)
131 2114 aaronmk
132 5430 aaronmk
__Dialect__validate_orig = csv.Dialect._validate
133
def __Dialect__validate(self):
134
        try: __Dialect__validate_orig(self)
135
        except _csv.Error, e:
136
            if str(e) == '"delimiter" must be an 1-character string': pass # OK
137
            else: raise
138
csv.Dialect._validate = __Dialect__validate
139
140 2114 aaronmk
##### Row filters
141
142
class Filter:
143
    '''Wraps a reader, filtering each row'''
144
    def __init__(self, filter_, reader):
145
        self.reader = reader
146
        self.filter = filter_
147
148
    def __iter__(self): return self
149
150
    def next(self): return self.filter(self.reader.next())
151 5574 aaronmk
152
    def close(self): pass # support using as a stream
153 2114 aaronmk
154 14599 aaronmk
class MultiFilter(Filter):
155
    '''enables applying multiple filters by nesting
156
    @param wrapped_reader outermost_filter(innermost_filter(reader))
157
    '''
158
    def __init__(self, wrapped_reader):
159
        def filter_(row): return row # no-op b/c filtering applied by nesting
160
        Filter.__init__(self, filter_, wrapped_reader)
161
162 2114 aaronmk
std_nulls = [r'\N']
163
empty_nulls = [''] + std_nulls
164
165
class NullFilter(Filter):
166
    '''Translates special string values to None'''
167
    def __init__(self, reader, nulls=std_nulls):
168
        map_ = dict.fromkeys(nulls, None)
169
        def filter_(row): return [map_.get(v, v) for v in row]
170
        Filter.__init__(self, filter_, reader)
171
172
class StripFilter(Filter):
173
    '''Strips whitespace'''
174
    def __init__(self, reader):
175
        def filter_(row): return [v.strip() for v in row]
176
        Filter.__init__(self, filter_, reader)
177 5570 aaronmk
178
class ColCtFilter(Filter):
179
    '''Gives all rows the same # columns'''
180
    def __init__(self, reader, cols_ct):
181
        def filter_(row): return util.list_as_length(row, cols_ct)
182
        Filter.__init__(self, filter_, reader)
183 5571 aaronmk
184
##### Translators
185
186 5586 aaronmk
class StreamFilter(Filter):
187
    '''Wraps a reader in a way that's usable to a filter stream that does not
188
    require lines to be strings. Reports EOF as '' instead of StopIteration.'''
189
    def __init__(self, reader):
190
        Filter.__init__(self, None, reader)
191
192
    def readline(self):
193
        try: return self.reader.next()
194
        except StopIteration: return '' # EOF
195
196 14586 aaronmk
class ProgressInputFilter(streams.ProgressInputStream): # is also a reader
197
    # ProgressInputStream extends StreamIter, so this can be used as a reader
198
    '''wraps a reader, reporting the # rows read every n rows and after the last
199
    row is read
200
    @param log the output stream for progress messages
201
    '''
202
    def __init__(self, reader, log, msg='Read %d row(s)', n=100):
203
        streams.ProgressInputStream.__init__(self, StreamFilter(reader), log,
204
            msg, n)
205
206 5735 aaronmk
class ColInsertFilter(Filter):
207 7290 aaronmk
    '''Adds column(s) to each row
208 9509 aaronmk
    @param mk_value(row, row_num) | literal_value
209 5735 aaronmk
    '''
210 14592 aaronmk
    def __init__(self, reader, mk_value, index=0, n=1, col_names=None):
211
        line_num_skip = 0
212
        if col_names != None:
213
            col_names = lists.mk_seq(col_names)
214
            n = len(col_names)
215
            line_num_skip = 1
216
217 9509 aaronmk
        if not callable(mk_value):
218
            value = mk_value
219
            def mk_value(row, row_num): return value
220
221 5735 aaronmk
        def filter_(row):
222
            row = list(row) # make sure it's mutable; don't modify input!
223 14592 aaronmk
224
            if self.is_header and col_names != None:
225
                values = col_names
226
                self.is_header = False
227
            else: values = n*[mk_value(row, self.reader.line_num-line_num_skip)]
228
229
            for i in xrange(len(values)): row.insert(index+i, values[i])
230 5735 aaronmk
            return row
231
        Filter.__init__(self, filter_,
232
            streams.LineCountInputStream(StreamFilter(reader)))
233 14592 aaronmk
        self.is_header = True
234 5735 aaronmk
235 5736 aaronmk
class RowNumFilter(ColInsertFilter):
236 5593 aaronmk
    '''Adds a row # column at the beginning of each row'''
237 14593 aaronmk
    def __init__(self, reader, col_name=None):
238 5736 aaronmk
        def mk_value(row, row_num): return row_num
239 14593 aaronmk
        ColInsertFilter.__init__(self, reader, mk_value, col_names=col_name)
240 5593 aaronmk
241 14591 aaronmk
class InputRewriter(StreamFilter): # is also a stream
242 5571 aaronmk
    '''Wraps a reader, writing each row back to CSV'''
243
    def __init__(self, reader, dialect=csv.excel):
244 5587 aaronmk
        StreamFilter.__init__(self, reader)
245
246 5571 aaronmk
        self.dialect = dialect
247
248
    def readline(self):
249 8069 aaronmk
        try:
250 14590 aaronmk
            row = StreamFilter.readline(self) # translate EOF
251 8069 aaronmk
            if row == '': return row # EOF
252
253
            line_stream = StringIO.StringIO()
254
            csv.writer(line_stream, self.dialect).writerow(row)
255
            return line_stream.getvalue()
256
        except Exception, e:
257
            exc.print_ex(e)
258
            raise
259 5571 aaronmk
260
    def read(self, n): return self.readline() # forward all reads to readline()
261 14594 aaronmk
262
def row_dict_to_list(dict_, col_order=[]):
263
    '''translates a CSV dict-based row to a list-based one
264
    @param dict_ {'col': 'value', __}
265
    @return (header, row) = (['col', __], ['value', __])
266
    '''
267
    dict_ = dict_.copy() # don't modify input!
268
    pairs = []
269
    for col in col_order: pairs.append((col, dict_.pop(col))) # ordered cols 1st
270
    pairs += sorted(dict_.items()) # then remaining cols in alphabetical order
271
    return (dicts.pair_keys(pairs), dicts.pair_values(pairs))
272 14595 aaronmk
273 14600 aaronmk
class row_dict_to_list_reader(Filter):
274
    '''reads dict-based rows as list-based rows
275
    @param reader [{'col': 'value', __}, __]
276 14595 aaronmk
    '''
277 14600 aaronmk
    def __init__(self, reader, col_order=[]):
278 14595 aaronmk
        self.header = None
279
280
        def filter_(row_dict):
281
            header, row = row_dict_to_list(row_dict, col_order)
282
283 14600 aaronmk
            if self.header == None: # 1st dict row: header
284 14595 aaronmk
                self.header = header
285
                self.next_row = row
286
                row = header
287 14600 aaronmk
            elif self.next_row != None: # 1st dict row: data
288 14595 aaronmk
                row = self.next_row
289
                self.next_row = None
290 14600 aaronmk
            else: # remaining dict rows
291 14595 aaronmk
                assert header == self.header # all rows must have same cols
292
293
            return row
294 14600 aaronmk
        Filter.__init__(self, filter_, reader)
295 14595 aaronmk
        self.next_row = None
296 14600 aaronmk
297
class JsonReader(MultiFilter):
298
    '''reads parsed JSON data as row tuples
299
    @param json_data [{'col': 'value', __}, __]
300
    '''
301
    def __init__(self, json_data, col_order=[]):
302
        def preprocess(json_row): return json_row
303
        MultiFilter.__init__(self, row_dict_to_list_reader(Filter(preprocess,
304
            iter(json_data))))