inbus

view inbus.py @ 6:59a9ca55098c

find changesets by author, revision, files, or words in the commit message
Changed name of struct_unpacker to pystruct_unpacker to be slightly less confusing (struct and tuple are easily confused).
author Eric Hopper <hopper@omnifarious.org>
date Tue, 26 Feb 2008 08:29:13 -0800
parents 8ecb08252c8a
children 7bf7acf4225e
line source
1 from struct import pack as _pack
2 from struct import unpack as _unpack
3 from functools import partial as _partial
4
5 def unpack(s, numtypes=1):
6 up, data_offset = create_unpacker(s, numtypes)
7 result = up(([], s, data_offset))
8 return tuple(result[0])
9
10 def unpack_off(s, numtypes=1):
11 up, data_offset = create_unpacker(s, numtypes)
12 result = up(([], s, data_offset))
13 return tuple(result[0]), result[2]
14
15 def create_unpacker(s, numtypes=1):
16 try:
17 numtypes = int(numtypes)
18 except (ValueError, TypeError):
19 raise TypeError("numtypes must be an integer")
20 if numtypes < 0:
21 raise TypeError("numtypes must be > 0")
22 if not isinstance(s, (str, buffer)):
23 raise TypeError("s must be a string or buffer")
24 def nullunpacker(dectup):
25 return dectup
26 def composeunpacker(f1, f2):
27 def compose(dectup):
28 return f1(f2(dectup))
29 return compose
30 unpacker = nullunpacker
31 offset = 0
32 for i in xrange(0, numtypes):
33 tag = s[offset]
34 offset += 1
35 if tag == 'c':
36 unpacker = composeunpacker(count_unpacker, unpacker)
37 elif tag == 'm':
38 if (len(s) > offset) and (s[offset] == '<'):
39 raise NotImplementedError("Length modifiers not yet supported for multi-precision integer ('m') type.")
40 unpacker = composeunpacker(multi_int_unpacker, unpacker)
41 elif tag == 'w':
42 if (len(s) > offset) and (s[offset] == '<'):
43 raise NotImplementedError("Length modifiers not yet supported for unsigned multi-precision integer ('w') type.")
44 unpacker = composeunpacker(multi_uint_unpacker, unpacker)
45 elif tag == 'y':
46 unpacker = composeunpacker(bool_unpacker, unpacker)
47 elif tag == 'b':
48 if (len(s) > offset) and (s[offset] == '<'):
49 raise NotImplementedError("Length modifiers not yet supported for binary blob ('b') type.")
50 unpacker = composeunpacker(blob_unpacker, unpacker)
51 elif tag == 's':
52 if (len(s) > offset) and (s[offset] == '<'):
53 raise NotImplementedError("Length modifiers not yet supported for string ('s') type.")
54 unpacker = composeunpacker(string_unpacker, unpacker)
55 elif tag == 'k':
56 unpacker = composeunpacker(byte_unpacker, unpacker)
57 elif tag == 'p':
58 unpacker = composeunpacker(ubyte_unpacker, unpacker)
59 elif tag == 'j':
60 unpacker = composeunpacker(short_unpacker, unpacker)
61 elif tag == 'o':
62 unpacker = composeunpacker(ushort_unpacker, unpacker)
63 elif tag == 'i':
64 unpacker = composeunpacker(int_unpacker, unpacker)
65 elif tag == 'u':
66 unpacker = composeunpacker(uint_unpacker, unpacker)
67 elif tag == 'l':
68 unpacker = composeunpacker(long_unpacker, unpacker)
69 elif tag == 'n':
70 unpacker = composeunpacker(ulong_unpacker, unpacker)
71 elif tag == 'f':
72 unpacker = composeunpacker(double_unpacker, unpacker)
73 elif tag == 'h':
74 unpacker = composeunpacker(float_unpacker, unpacker)
75 elif tag == 'g':
76 raise NotImplementedError("The arbitrary size floating point ('g') type is not yet supported.")
77 elif tag == 'v':
78 unpacker = composeunpacker(variant_unpacker, unpacker)
79 elif tag in ['t', 'a', 'd']:
80 if tag == 't':
81 subunpacker, suboffset = make_tuple_unpacker(buffer(s, offset))
82 elif tag == 'a':
83 subunpacker, suboffset = make_array_unpacker(buffer(s, offset))
84 elif tag == 'd':
85 subunpacker, suboffset = make_dict_unpacker(buffer(s, offset))
86 else:
87 assert False, "Getting here should be impossible."
88 offset += suboffset
89 unpacker = composeunpacker(subunpacker, unpacker)
90 else:
91 raise ValueError("Unknown type tag '%s' encountered." % (tag,))
92 return unpacker, offset
93
94 def count_unpacker(dectup):
95 val, newoffset = unpackcount_off(dectup[1], dectup[2])
96 dectup[0].append(val)
97 return dectup[0], dectup[1], newoffset
98
99 def multi_int_unpacker(dectup):
100 result, s, offset = dectup
101 mlen, newoffset = unpackcount_off(dectup[1], dectup[2])
102 if (newoffset + mlen) > len(s):
103 raise ValueError("The string passed in is too short to unpack.")
104 if mlen <= 0:
105 result.append(0L)
106 return (result, s, newoffset)
107
108 mint = unpackl(buffer(s, newoffset, mlen))
109 if ord(s[newoffset]) & 0x80:
110 mint -= 2**(mlen * 8)
111 result.append(mint)
112 return (result, s, newoffset + mlen)
113
114 def multi_uint_unpacker(dectup):
115 result, s, offset = dectup
116 mlen, newoffset = unpackcount_off(dectup[1], dectup[2])
117 if (newoffset + mlen) > len(s):
118 raise ValueError("The string passed in is too short to unpack.")
119 if mlen <= 0:
120 result.append(0L)
121 return (result, s, newoffset)
122 mint = unpackl(buffer(s, newoffset, mlen))
123 result.append(mint)
124 return (result, s, newoffset + mlen)
125
126 def blob_unpacker(dectup):
127 result, s, offset = dectup
128 blen, newoffset = unpackcount_off(dectup[1], dectup[2])
129 if (newoffset + blen) > len(s):
130 raise ValueError("The string passed in is too short to unpack.")
131 if blen <= 0:
132 result.append('')
133 return result, s, newoffset
134 result.append(buffer(s, newoffset, blen))
135 return result, s, newoffset + blen
136
137 def string_unpacker(dectup):
138 result, s, offset = dectup
139 slen, newoffset = unpackcount_off(dectup[1], dectup[2])
140 if (newoffset + slen) > len(s):
141 raise ValueError("The string passed in is too short to unpack.")
142 if slen <= 0:
143 result.append('')
144 return result, s, newoffset
145 result.append(unicode(buffer(s, newoffset, slen), 'utf-8'))
146 return result, s, newoffset + slen
147
148 def pystruct_unpacker(spec, len, dectup):
149 result, s, offset = dectup
150 (upresult,) = _unpack(spec, buffer(s, offset, len))
151 result.append(upresult)
152 return result, s, offset + len
153
154 byte_unpacker = _partial(pystruct_unpacker, '>b', 1)
155 ubyte_unpacker = _partial(pystruct_unpacker, '>b', 1)
156 short_unpacker = _partial(pystruct_unpacker, '>h', 2)
157 ushort_unpacker = _partial(pystruct_unpacker, '>H', 2)
158 int_unpacker = _partial(pystruct_unpacker, '>i', 4)
159 uint_unpacker = _partial(pystruct_unpacker, '>I', 4)
160 long_unpacker = _partial(pystruct_unpacker, '>q', 8)
161 ulong_unpacker = _partial(pystruct_unpacker, '>Q', 8)
162 float_unpacker = _partial(pystruct_unpacker, '>f', 4)
163 double_unpacker = _partial(pystruct_unpacker, '>d', 8)
164
165 def variant_unpacker(dectup):
166 result, s, offset = dectup
167 val, uplen = unpack_off(buffer(s, offset), 1)
168 (val,) = val
169 result.append(val)
170 return result, s, offset + uplen
171
172 def make_tuple_unpacker(s):
173 if s[0] != '(':
174 raise ValueError("Tuple type tag ('t') must be followed by '('")
175 offset = 1
176 def nullunpacker(dectup):
177 return dectup
178 def composeunpacker(f1, f2):
179 def compose(dectup):
180 return f1(f2(dectup))
181 return compose
182 def tuplefinishunpacker(dectup):
183 return [tuple(dectup[0])], dectup[1], dectup[2]
184 unpacker = nullunpacker
185 while (offset < len(s)) and (s[offset] != ')'):
186 elunpacker, consumed = create_unpacker(buffer(s, offset))
187 offset += consumed
188 unpacker = composeunpacker(elunpacker, unpacker)
189 if offset >= len(s):
190 raise ValueError("Ran off the end of the string to unpack while parsing tuple")
191 return composeunpacker(tuplefinishunpacker, unpacker), offset + 1
192
193 def make_array_unpacker(s):
194 if s[0] != '[':
195 raise ValueError("Array type tag ('a') must be followed by '['")
196 offset = 1
197 elunpacker, consumed = create_unpacker(buffer(s, offset))
198 offset += consumed
199 if (offset >= len(s)) or (s[offset] != ']'):
200 raise ValueError("Array type tag must end with ']' after one type tag.")
201 def unpack_array(dectup):
202 result, s, offset = dectup
203 array = []
204 if len(s) <= offset:
205 raise ValueError("End of string while parsing array.")
206 while s[offset] != '\0':
207 out, s, offset = elunpacker(([], s, offset + 1))
208 if len(s) <= offset:
209 raise ValueError("End of string while parsing array.")
210 array.extend(out)
211 result.append(array)
212 return result, s, offset + 1
213 return unpack_array, offset + 1
214
215 def make_dict_unpacker(s):
216 if s[0] != '{':
217 raise ValueError("Dictionary type tag ('d') must be followed by '{'")
218 offset = 1
219 keyunpacker, consumed = create_unpacker(buffer(s, offset))
220 offset += consumed
221 if offset >= len(s):
222 raise ValueError("Dictionary type tags must contain two subtypes.")
223 valunpacker, consumed = create_unpacker(buffer(s, offset))
224 offset += consumed
225 if (offset >= len(s)) or (s[offset] != '}'):
226 print "offset = %r / consumed = %r / s = %r / len(s) = %r" % \
227 (offset, consumed, s, len(s))
228 raise ValueError("Dictionary type tag must end with '}' after two type tag.")
229 def unpack_dict(dectup):
230 result, s, offset = dectup
231 dictionary = {}
232 if len(s) <= offset:
233 raise ValueError("End of string while parsing dictionary.")
234 while s[offset] != '\0':
235 out, s, offset = valunpacker(keyunpacker(([], s, offset + 1)))
236 if len(s) <= offset:
237 raise ValueError("End of string while parsing dictionary.")
238 dictionary[out[0]] = out[1]
239 result.append(dictionary)
240 return result, s, offset + 1
241 return unpack_dict, offset + 1
242
243 def packl(lnum, pad = 1):
244 if lnum < 0:
245 raise RangeError("Cannot use packl to convert a negative integer "
246 "to a string.")
247 count = 0
248 l = []
249 while lnum > 0:
250 l.append(lnum & 0xffffffffffffffffL)
251 count += 1
252 lnum >>= 64
253 if count <= 0:
254 return '\0' * pad
255 elif pad >= 8:
256 lens = 8 * count % pad
257 pad = ((lens != 0) and (pad - lens)) or 0
258 l.append('>' + 'x' * pad + 'Q' * count)
259 l.reverse()
260 return _pack(*l)
261 else:
262 l.append('>' + 'Q' * count)
263 l.reverse()
264 s = _pack(*l).lstrip('\0')
265 lens = len(s)
266 if (lens % pad) != 0:
267 return '\0' * (pad - lens % pad) + s
268 else:
269 return s
270
271 def unpackl(s):
272 n = 0L
273 if len(s) <= 0:
274 return n
275 count8 = len(s) // 8
276 count1 = len(s) % 8
277 upfmt = '>' + 'Q' * count8 + 'B' * count1
278 l = _unpack(upfmt, s)
279 for val in l[0:count8]:
280 n <<= 64
281 n |= val
282 for val in l[count8:]:
283 n <<= 8
284 n |= val
285 return n
286
287 def packcount(count):
288 """packcount(non-negative integer) -> string representing the count
289
290 The count cannot be less than 0. The encoding is designed so that if
291 the count is a message length, a very small percentage of the total
292 message will be consumed by the count.
293
294 It is suggested that the message length itself not be counted as part
295 of the message length. This means that counts of 0 should be
296 sensible, and indicate a message with no content.
297
298 This function should NOT be used for values that are really arbitrary
299 sized integers. The maximum sized integer that this function can
300 represent is 4080 bits. Public key values will most likely exceed
301 this. To store a public key (or other arbitrary sized integer) value,
302 encode that value in some other way (perhaps using packl), then use
303 this function to encode the length of that encoding."""
304 if count < 0:
305 raise RangeError("A count cannot be negative!")
306 if count < 223:
307 return chr(count)
308 elif count < (223 + 8192):
309 count -= 223
310 upper = count // 256
311 assert upper < 32
312 lower = count % 256
313 return chr(upper + 223) + chr(lower)
314 else:
315 s = packl(count, 2)
316 assert len(s) % 2 == 0
317 countlen = len(s) // 2
318 if countlen > 255:
319 raise RangeError("A count must take fewer than 4080 bits to "
320 "represent!")
321 return chr(255) + chr(countlen) + s
322
323 def unpackcount_off(s, off = 0):
324 """unpackcount(a string, offset in string of something created by packcount)
325 -> a tuple
326 the tuple is (count, offset of byte after count).
327
328 See the documentation for packcount for a little more of an
329 explanation. A ValueError exception will be thrown for strings that
330 don't start with a valid count value."""
331 fc = ord(s[off])
332 if fc < 223:
333 # One byte count
334 return (fc, off + 1)
335 elif fc < (223 + 32):
336 # Two byte count
337 if 2 > (len(s) - off):
338 raise ValueError("The passed in string is too short, and isn't "
339 "a valid count.")
340 upper_5bits = fc - 223
341 lower_8bits = ord(s[off + 1])
342 count = ((upper_5bits << 8) | lower_8bits) + 223
343 return (count, off + 2)
344 else:
345 # Variable length count, length in second octet
346 if 2 > (len(s) - off):
347 raise ValueError("The passed in string is too short, and isn't "
348 "a valid count.")
349 length = ord(s[off + 1]) * 2
350 if length <= 0:
351 raise ValueError("The passed in string isn't a valid count!")
352 # Skip past length of count itself
353 off += 2
354 # Enough octets for rest of count?
355 if length > (len(s) - off):
356 raise ValueError("The passed in string is too short, and isn't "
357 "a valid count.")
358 # Use the handy unpackl function to turn the octet string into a number
359 count = unpackl(s[off:(off + length)])
360 # If count fits into an int, turn it into an int
361 if count < 2**31:
362 count = int(count)
363 return (count, off + length)
364
365 def unpackcount(s):
366 """unpackcount(a string beginning with something created by packcount) -> a tuple
367 the tuple is (count, remainder of s).
368
369 See the documentation for packcount for a little more of an
370 explanation. A ValueError exception will be thrown for strings that
371 don't start with a valid count value."""
372 (count, newoff) = unpackcount_off(s, 0)
373 return (count, s[newoff:])