Project

General

Profile

1 1388 aaronmk
# CSV I/O
2
3
import csv
4 5431 aaronmk
import _csv
5 1388 aaronmk
import StringIO
6
7 5593 aaronmk
import streams
8 1623 aaronmk
import strings
9 1444 aaronmk
import util
10
11 5426 aaronmk
delims = ',;\t|`'
12 5439 aaronmk
tab_padded_delims = ['\t|\t']
13 1623 aaronmk
tsv_delim = '\t'
14
escape = '\\'
15 1442 aaronmk
16 1623 aaronmk
ending_placeholder = r'\n'
17
18 5437 aaronmk
def is_tsv(dialect): return dialect.delimiter.startswith(tsv_delim)
19 1623 aaronmk
20 1621 aaronmk
def sniff(line):
21
    '''Automatically detects the dialect'''
22 5435 aaronmk
    line, ending = strings.extract_line_ending(line)
23 1623 aaronmk
    dialect = csv.Sniffer().sniff(line, delims)
24 5435 aaronmk
25 5439 aaronmk
    if is_tsv(dialect):
26
        # Check multi-char delims using \t
27
        delim = strings.find_any(line, tab_padded_delims)
28
        if delim:
29
            dialect.delimiter = delim
30
            line_suffix = delim.rstrip('\t')
31
            if line.endswith(line_suffix): ending = line_suffix+ending
32 1621 aaronmk
    else: dialect.doublequote = True # Sniffer doesn't turn this on by default
33 5435 aaronmk
    dialect.lineterminator = ending
34
35 1621 aaronmk
    return dialect
36
37 1923 aaronmk
def stream_info(stream, parse_header=False):
38 6589 aaronmk
    '''Automatically detects the dialect based on the header line.
39
    Uses the Excel dialect if the CSV file is empty.
40 1923 aaronmk
    @return NamedTuple {header_line, header, dialect}'''
41 1444 aaronmk
    info = util.NamedTuple()
42
    info.header_line = stream.readline()
43 1923 aaronmk
    info.header = None
44
    if info.header_line != '':
45
        info.dialect = sniff(info.header_line)
46 6589 aaronmk
    else: info.dialect = csv.excel # line of '' indicates EOF = empty stream
47
48
    if parse_header:
49
        try: info.header = reader_class(info.dialect)(
50
            StringIO.StringIO(info.header_line), info.dialect).next()
51
        except StopIteration: info.header = []
52
53 1444 aaronmk
    return info
54
55 5170 aaronmk
tsv_encode_map = strings.json_encode_map[:]
56
tsv_encode_map.append(('\t', r'\t'))
57
tsv_decode_map = strings.flip_map(tsv_encode_map)
58 5146 aaronmk
59 1623 aaronmk
class TsvReader:
60
    '''Unlike csv.reader, for TSVs, interprets \ as escaping a line ending but
61 5145 aaronmk
    ignores it before everything else (e.g. \N for NULL).
62 5170 aaronmk
    Also expands tsv_encode_map escapes.
63 5145 aaronmk
    '''
64 1623 aaronmk
    def __init__(self, stream, dialect):
65
        assert is_tsv(dialect)
66
        self.stream = stream
67
        self.dialect = dialect
68
69
    def __iter__(self): return self
70
71
    def next(self):
72
        record = ''
73
        ending = None
74
        while True:
75
            line = self.stream.readline()
76
            if line == '': raise StopIteration
77
78 5438 aaronmk
            line = strings.remove_suffix(self.dialect.lineterminator, line)
79 5433 aaronmk
            contents = strings.remove_suffix(escape, line)
80 1623 aaronmk
            record += contents
81 5433 aaronmk
            if len(contents) == len(line): break # no line continuation
82 1623 aaronmk
            record += ending_placeholder
83 3055 aaronmk
84
        # Prevent "new-line character seen in unquoted field" errors
85
        record = record.replace('\r', ending_placeholder)
86
87 7211 aaronmk
        # Split line
88
        if len(self.dialect.delimiter) > 1: # multi-char delims
89
            row = record.split(self.dialect.delimiter)
90
        else: row = csv.reader(StringIO.StringIO(record), self.dialect).next()
91
92 5170 aaronmk
        return [strings.replace_all(tsv_decode_map, v) for v in row]
93 1623 aaronmk
94
def reader_class(dialect):
95
    if is_tsv(dialect): return TsvReader
96
    else: return csv.reader
97
98
def make_reader(stream, dialect): return reader_class(dialect)(stream, dialect)
99
100 1388 aaronmk
def reader_and_header(stream):
101
    '''Automatically detects the dialect based on the header line
102
    @return tuple (reader, header)'''
103 1923 aaronmk
    info = stream_info(stream, parse_header=True)
104 1958 aaronmk
    return (make_reader(stream, info.dialect), info.header)
105 1446 aaronmk
106
##### csv modifications
107
108
# Note that these methods only work on *instances* of Dialect classes
109
csv.Dialect.__eq__ = lambda self, other: self.__dict__ == other.__dict__
110
csv.Dialect.__ne__ = lambda self, other: not (self == other)
111 2114 aaronmk
112 5430 aaronmk
__Dialect__validate_orig = csv.Dialect._validate
113
def __Dialect__validate(self):
114
        try: __Dialect__validate_orig(self)
115
        except _csv.Error, e:
116
            if str(e) == '"delimiter" must be an 1-character string': pass # OK
117
            else: raise
118
csv.Dialect._validate = __Dialect__validate
119
120 2114 aaronmk
##### Row filters
121
122
class Filter:
123
    '''Wraps a reader, filtering each row'''
124
    def __init__(self, filter_, reader):
125
        self.reader = reader
126
        self.filter = filter_
127
128
    def __iter__(self): return self
129
130
    def next(self): return self.filter(self.reader.next())
131 5574 aaronmk
132
    def close(self): pass # support using as a stream
133 2114 aaronmk
134
std_nulls = [r'\N']
135
empty_nulls = [''] + std_nulls
136
137
class NullFilter(Filter):
138
    '''Translates special string values to None'''
139
    def __init__(self, reader, nulls=std_nulls):
140
        map_ = dict.fromkeys(nulls, None)
141
        def filter_(row): return [map_.get(v, v) for v in row]
142
        Filter.__init__(self, filter_, reader)
143
144
class StripFilter(Filter):
145
    '''Strips whitespace'''
146
    def __init__(self, reader):
147
        def filter_(row): return [v.strip() for v in row]
148
        Filter.__init__(self, filter_, reader)
149 5570 aaronmk
150
class ColCtFilter(Filter):
151
    '''Gives all rows the same # columns'''
152
    def __init__(self, reader, cols_ct):
153
        def filter_(row): return util.list_as_length(row, cols_ct)
154
        Filter.__init__(self, filter_, reader)
155 5571 aaronmk
156
##### Translators
157
158 5586 aaronmk
class StreamFilter(Filter):
159
    '''Wraps a reader in a way that's usable to a filter stream that does not
160
    require lines to be strings. Reports EOF as '' instead of StopIteration.'''
161
    def __init__(self, reader):
162
        Filter.__init__(self, None, reader)
163
164
    def readline(self):
165
        try: return self.reader.next()
166
        except StopIteration: return '' # EOF
167
168 5735 aaronmk
class ColInsertFilter(Filter):
169
    '''Adds a column to each row
170
    @param mk_value(row, row_num)
171
    '''
172
    def __init__(self, reader, mk_value, index=0):
173
        def filter_(row):
174
            row = list(row) # make sure it's mutable; don't modify input!
175
            row.insert(index, mk_value(row, self.reader.line_num))
176
            return row
177
        Filter.__init__(self, filter_,
178
            streams.LineCountInputStream(StreamFilter(reader)))
179
180 5736 aaronmk
class RowNumFilter(ColInsertFilter):
181 5593 aaronmk
    '''Adds a row # column at the beginning of each row'''
182
    def __init__(self, reader):
183 5736 aaronmk
        def mk_value(row, row_num): return row_num
184
        ColInsertFilter.__init__(self, reader, mk_value, 0)
185 5593 aaronmk
186 5587 aaronmk
class InputRewriter(StreamFilter):
187 5571 aaronmk
    '''Wraps a reader, writing each row back to CSV'''
188
    def __init__(self, reader, dialect=csv.excel):
189 5587 aaronmk
        StreamFilter.__init__(self, reader)
190
191 5571 aaronmk
        self.dialect = dialect
192
193
    def readline(self):
194 5587 aaronmk
        row = self.reader.readline()
195 5585 aaronmk
        if row == '': return row # EOF
196
197 5571 aaronmk
        line_stream = StringIO.StringIO()
198
        csv.writer(line_stream, self.dialect).writerow(row)
199
        return line_stream.getvalue()
200
201
    def read(self, n): return self.readline() # forward all reads to readline()