Package dbf :: Module ver_2
[hide private]
[frames] | no frames]

Source Code for Module dbf.ver_2

   1  from __future__ import with_statement 
   2  """ 
   3  ========= 
   4  Copyright 
   5  ========= 
   6   
   7      - Portions copyright: 2008-2012 Ad-Mail, Inc -- All rights reserved. 
   8      - Portions copyright: 2012-2014 Ethan Furman -- All rights reserved. 
   9      - Author: Ethan Furman 
  10      - Contact: ethan@stoneleaf.us 
  11   
  12  Redistribution and use in source and binary forms, with or without 
  13  modification, are permitted provided that the following conditions are met: 
  14      - Redistributions of source code must retain the above copyright 
  15        notice, this list of conditions and the following disclaimer. 
  16      - Redistributions in binary form must reproduce the above copyright 
  17        notice, this list of conditions and the following disclaimer in the 
  18        documentation and/or other materials provided with the distribution. 
  19      - Neither the name of Ad-Mail, Inc nor the 
  20        names of its contributors may be used to endorse or promote products 
  21        derived from this software without specific prior written permission. 
  22   
  23  THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, 
  24  INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY 
  25  AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 
  26  ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 
  27  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 
  28  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 
  29  OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
  30  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 
  31  OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 
  32  ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
  33  """ 
  34   
  35  import codecs 
  36  import collections 
  37  import csv 
  38  import datetime 
  39  import os 
  40  import struct 
  41  import sys 
  42  import time 
  43  import weakref 
  44   
  45  from array import array 
  46  from bisect import bisect_left, bisect_right 
  47  import decimal 
  48  from decimal import Decimal 
  49  from enum import Enum, IntEnum 
  50  from glob import glob 
  51  from math import floor 
  52  import types 
  53  from types import NoneType 
  54   
  55  py_ver = sys.version_info[:2] 
  56   
  57  # Flag for behavior if bad data is encountered in a logical field 
  58  # Return None if True, else raise BadDataError 
  59  LOGICAL_BAD_IS_NONE = True 
  60   
  61  # treat non-unicode data as ... 
  62  input_decoding = 'ascii' 
  63   
  64  # if no codepage specified on dbf creation, use this 
  65  default_codepage = 'ascii' 
  66   
  67  # default format if none specified 
  68  default_type = 'db3' 
  69   
  70  temp_dir = os.environ.get("DBF_TEMP") or os.environ.get("TMP") or os.environ.get("TEMP") or "" 
  71   
  72   
  73  # user-defined pql functions  (pql == primitive query language) 
  74  # it is not real sql and won't be for a long time (if ever) 
  75  pql_user_functions = dict() 
  76   
  77  # signature:_meta of template records 
  78  _Template_Records = dict() 
  79   
  80  # dec jan feb mar apr may jun jul aug sep oct nov dec jan 
  81  days_per_month = [31, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 31] 
  82  days_per_leap_month = [31, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 31] 
83 84 -class IsoDay(IntEnum):
85 MONDAY = 1 86 TUESDAY = 2 87 WEDNESDAY = 3 88 THURSDAY = 4 89 FRIDAY = 5 90 SATURDAY = 6 91 SUNDAY = 7 92
93 - def next_delta(self, day):
94 """Return number of days needed to get from self to day.""" 95 if self == day: 96 return 7 97 delta = day - self 98 if delta < 0: 99 delta += 7 100 return delta
101
102 - def last_delta(self, day):
103 """Return number of days needed to get from self to day.""" 104 if self == day: 105 return -7 106 delta = day - self 107 if delta > 0: 108 delta -= 7 109 return delta
110
111 -class RelativeDay(Enum):
112 LAST_SUNDAY = () 113 LAST_SATURDAY = () 114 LAST_FRIDAY = () 115 LAST_THURSDAY = () 116 LAST_WEDNESDAY = () 117 LAST_TUESDAY = () 118 LAST_MONDAY = () 119 NEXT_MONDAY = () 120 NEXT_TUESDAY = () 121 NEXT_WEDNESDAY = () 122 NEXT_THURSDAY = () 123 NEXT_FRIDAY = () 124 NEXT_SATURDAY = () 125 NEXT_SUNDAY = () 126
127 - def __new__(cls):
128 result = object.__new__(cls) 129 result._value = len(cls.__members__) + 1 130 return result
131
132 - def days_from(self, day):
133 target = IsoDay[self.name[5:]] 134 if self.name[:4] == 'LAST': 135 return day.last_delta(target) 136 return day.next_delta(target)
137 globals().update(RelativeDay.__members__)
138 139 -class IsoMonth(IntEnum):
140 JANUARY = 1 141 FEBRUARY = 2 142 MARCH = 3 143 APRIL = 4 144 MAY = 5 145 JUNE = 6 146 JULY = 7 147 AUGUST = 8 148 SEPTEMBER = 9 149 OCTOBER = 10 150 NOVEMBER = 11 151 DECEMBER = 12 152
153 - def next_delta(self, month):
154 """Return number of months needed to get from self to month.""" 155 if self == month: 156 return 12 157 delta = month - self 158 if delta < 0: 159 delta += 12 160 return delta
161
162 - def last_delta(self, month):
163 """Return number of months needed to get from self to month.""" 164 if self == month: 165 return -12 166 delta = month - self 167 if delta > 0: 168 delta -= 12 169 return delta
170
171 -class RelativeMonth(Enum):
172 LAST_DECEMBER = () 173 LAST_NOVEMBER = () 174 LAST_OCTOBER = () 175 LAST_SEPTEMBER = () 176 LAST_AUGUST = () 177 LAST_JULY = () 178 LAST_JUNE = () 179 LAST_MAY = () 180 LAST_APRIL = () 181 LAST_MARCH= () 182 LAST_FEBRUARY = () 183 LAST_JANUARY = () 184 NEXT_JANUARY = () 185 NEXT_FEBRUARY = () 186 NEXT_MARCH = () 187 NEXT_APRIL = () 188 NEXT_MAY = () 189 NEXT_JUNE = () 190 NEXT_JULY = () 191 NEXT_AUGUST = () 192 NEXT_SEPTEMBER = () 193 NEXT_OCTOBER = () 194 NEXT_NOVEMBER = () 195 NEXT_DECEMBER = () 196
197 - def __new__(cls):
198 result = object.__new__(cls) 199 result._value = len(cls.__members__) + 1 200 return result
201
202 - def months_from(self, month):
203 target = IsoMonth[self.name[5:]] 204 if self.name[:4] == 'LAST': 205 return month.last_delta(target) 206 return month.next_delta(target)
207 globals().update(RelativeMonth.__members__)
208 209 -def is_leapyear(year):
210 if year % 400 == 0: 211 return True 212 elif year % 100 == 0: 213 return False 214 elif year % 4 == 0: 215 return True 216 else: 217 return False
218 219 # 2.7 constructs 220 if py_ver < (2, 7):
221 # Decimal does not accept float inputs until 2.7 222 - def Decimal(val=0):
223 if isinstance(val, float): 224 return decimal.Decimal(str(val)) 225 return decimal.Decimal(val)
226 227 bytes = str 228 229 230 # 2.6 constructs 231 if py_ver < (2, 6):
232 233 # define next() 234 - def next(iterator):
235 return iterator.next()
236
237 238 # 2.6+ property for 2.5-, 239 # define our own property type 240 - class property(object):
241 """ 242 2.6 properties for 2.5- 243 """ 244
245 - def __init__(self, fget=None, fset=None, fdel=None, doc=None):
246 self.fget = fget 247 self.fset = fset 248 self.fdel = fdel 249 self.__doc__ = doc or fget.__doc__
250
251 - def __call__(self, func):
252 self.fget = func 253 if not self.__doc__: 254 self.__doc__ = func.__doc__
255
256 - def __get__(self, obj, objtype=None):
257 if obj is None: 258 return self 259 if self.fget is None: 260 raise AttributeError("unreadable attribute") 261 return self.fget(obj)
262
263 - def __set__(self, obj, value):
264 if self.fset is None: 265 raise AttributeError("can't set attribute") 266 self.fset(obj, value)
267
268 - def __delete__(self, obj):
269 if self.fdel is None: 270 raise AttributeError("can't delete attribute") 271 self.fdel(obj)
272
273 - def setter(self, func):
274 self.fset = func 275 return self
276
277 - def deleter(self, func):
278 self.fdel = func 279 return self
280 281 282 # 2.5 constructs 283 284 try: 285 all 286 except NameError:
287 288 - def all(iterable):
289 for element in iterable: 290 if not element: 291 return False 292 return True
293
294 - def any(iterable):
295 for element in iterable: 296 if element: 297 return True 298 return False
299 300 SEEK_SET, SEEK_CUR, SEEK_END = range(3) 301 302 else: 303 304 SEEK_SET, SEEK_CUR, SEEK_END = os.SEEK_SET, os.SEEK_CUR, os.SEEK_END 305 306 # collections.defaultdict for <= 2.4 307 try: 308 from collections import defaultdict 309 except ImportError:
310 311 - class defaultdict(dict):
312
313 - def __init__(self, default_factory=None, *a, **kw):
314 if (default_factory is not None and 315 not hasattr(default_factory, '__call__')): 316 raise TypeError('first argument must be callable') 317 dict.__init__(self, *a, **kw) 318 self.default_factory = default_factory
319
320 - def __getitem__(self, key):
321 try: 322 return dict.__getitem__(self, key) 323 except KeyError: 324 return self.__missing__(key)
325
326 - def __missing__(self, key):
327 if self.default_factory is None: 328 raise KeyError(key) 329 self[key] = value = self.default_factory() 330 return value
331
332 - def __reduce__(self):
333 if self.default_factory is None: 334 args = tuple() 335 else: 336 args = self.default_factory, 337 return type(self), args, None, None, self.iteritems()
338
339 - def copy(self):
340 return self.__copy__()
341
342 - def __copy__(self):
343 return type(self)(self.default_factory, self)
344
345 - def __deepcopy__(self, memo):
346 import copy 347 return type(self)(self.default_factory, 348 copy.deepcopy(self.items()))
349
350 - def __repr__(self):
351 return 'defaultdict(%s, %s)' % (self.default_factory, 352 dict.__repr__(self))
353
354 # other constructs 355 356 -class LazyAttr(object):
357 """ 358 doesn't create object until actually accessed 359 """ 360
361 - def __init__(yo, func=None, doc=None):
362 yo.fget = func 363 yo.__doc__ = doc or func.__doc__
364
365 - def __call__(yo, func):
366 yo.fget = func
367
368 - def __get__(yo, instance, owner):
369 if instance is None: 370 return yo 371 return yo.fget(instance)
372
373 374 -class MutableDefault(object):
375 """ 376 Lives in the class, and on first access calls the supplied factory and 377 maps the result into the instance it was called on 378 """ 379
380 - def __init__(self, func):
381 self._name = func.__name__ 382 self.func = func
383
384 - def __call__(self):
385 return self
386
387 - def __get__(self, instance, owner):
388 result = self.func() 389 if instance is not None: 390 setattr(instance, self._name, result) 391 return result
392
393 - def __repr__(self):
394 result = self.func() 395 return "MutableDefault(%r)" % (result, )
396
397 398 399 -def none(*args, **kwargs):
400 """ 401 because we can't do `NoneType()` 402 """ 403 return None
404 405 # Constants 406 SYSTEM = 0x01 407 NULLABLE = 0x02 408 BINARY = 0x04 409 #AUTOINC = 0x0c # not currently supported (not vfp 6) 410 411 TYPE = 0 412 START = 1 413 LENGTH = 2 414 END = 3 415 DECIMALS = 4 416 FLAGS = 5 417 CLASS = 6 418 EMPTY = 7 419 NULL = 8 420 421 FIELD_FLAGS = { 422 'null' : NULLABLE, 423 'binary' : BINARY, 424 'nocptrans' : BINARY, 425 #'autoinc' : AUTOINC, 426 427 NULLABLE : 'null', 428 BINARY : 'binary', 429 SYSTEM : 'system', 430 #AUTOINC : 'autoinc', 431 } 432 433 IN_MEMORY = 0 434 ON_DISK = 1 435 436 CLOSED = 'closed' 437 READ_ONLY = 'read-only' 438 READ_WRITE = 'read-write'
439 440 441 # warnings and errors 442 443 -class DbfError(Exception):
444 """ 445 Fatal errors elicit this response. 446 """
447
448 449 -class DataOverflowError(DbfError):
450 """ 451 Data too large for field 452 """ 453
454 - def __init__(self, message, data=None):
455 DbfError.__init__(self, message) 456 self.data = data
457
458 459 -class BadDataError(DbfError):
460 """ 461 bad data in table 462 """ 463
464 - def __init__(self, message, data=None):
465 DbfError.__init__(self, message) 466 self.data = data
467
468 469 -class FieldMissingError(KeyError, DbfError):
470 """ 471 Field does not exist in table 472 """ 473
474 - def __init__(self, fieldname):
475 KeyError.__init__(self, '%s: no such field in table' % fieldname) 476 DbfError.__init__(self, '%s: no such field in table' % fieldname) 477 self.data = fieldname
478
479 480 -class FieldSpecError(DbfError, ValueError):
481 """ 482 invalid field specification 483 """ 484
485 - def __init__(self, message):
486 ValueError.__init__(self, message) 487 DbfError.__init__(self, message)
488
489 490 -class NonUnicodeError(DbfError):
491 """ 492 Data for table not in unicode 493 """ 494
495 - def __init__(self, message=None):
497
498 499 -class NotFoundError(DbfError, ValueError, KeyError, IndexError):
500 """ 501 record criteria not met 502 """ 503
504 - def __init__(self, message=None, data=None):
505 ValueError.__init__(self, message) 506 KeyError.__init__(self, message) 507 IndexError.__init__(self, message) 508 DbfError.__init__(self, message) 509 self.data = data
510
511 512 -class DbfWarning(Exception):
513 """ 514 Normal operations elicit this response 515 """
516
517 518 -class Eof(DbfWarning, StopIteration):
519 """ 520 End of file reached 521 """ 522 523 message = 'End of file reached' 524
525 - def __init__(self):
526 StopIteration.__init__(self, self.message) 527 DbfWarning.__init__(self, self.message)
528
529 530 -class Bof(DbfWarning, StopIteration):
531 """ 532 Beginning of file reached 533 """ 534 535 message = 'Beginning of file reached' 536
537 - def __init__(self):
538 StopIteration.__init__(self, self.message) 539 DbfWarning.__init__(self, self.message)
540
541 542 -class DoNotIndex(DbfWarning):
543 """ 544 Returned by indexing functions to suppress a record from becoming part of the index 545 """ 546 547 message = 'Not indexing record' 548
549 - def __init__(self):
550 DbfWarning.__init__(self, self.message)
551 552 553 # wrappers around datetime and logical objects to allow null values 554 555 # gets replaced later by their final values 556 Unknown = Other = object()
557 558 -class NullType(object):
559 """ 560 Null object -- any interaction returns Null 561 """ 562
563 - def _null(self, *args, **kwargs):
564 return self
565 566 __eq__ = __ne__ = __ge__ = __gt__ = __le__ = __lt__ = _null 567 __add__ = __iadd__ = __radd__ = _null 568 __sub__ = __isub__ = __rsub__ = _null 569 __mul__ = __imul__ = __rmul__ = _null 570 __div__ = __idiv__ = __rdiv__ = _null 571 __mod__ = __imod__ = __rmod__ = _null 572 __pow__ = __ipow__ = __rpow__ = _null 573 __and__ = __iand__ = __rand__ = _null 574 __xor__ = __ixor__ = __rxor__ = _null 575 __or__ = __ior__ = __ror__ = _null 576 __truediv__ = __itruediv__ = __rtruediv__ = _null 577 __floordiv__ = __ifloordiv__ = __rfloordiv__ = _null 578 __lshift__ = __ilshift__ = __rlshift__ = _null 579 __rshift__ = __irshift__ = __rrshift__ = _null 580 __neg__ = __pos__ = __abs__ = __invert__ = _null 581 __call__ = __getattr__ = _null 582
583 - def __divmod__(self, other):
584 return self, self
585 __rdivmod__ = __divmod__ 586 587 if py_ver >= (2, 6): 588 __hash__ = None 589 else:
590 - def __hash__(self):
591 raise TypeError("unhashable type: 'Null'")
592
593 - def __new__(cls, *args):
594 return cls.null
595
596 - def __nonzero__(self):
597 return False
598
599 - def __repr__(self):
600 return '<null>'
601
602 - def __setattr__(self, name, value):
603 return None
604
605 - def __setitem___(self, index, value):
606 return None
607
608 - def __str__(self):
609 return ''
610 611 NullType.null = object.__new__(NullType) 612 Null = NullType()
613 614 615 -class Vapor(object):
616 """ 617 used in Vapor Records -- compares unequal with everything 618 """ 619
620 - def __eq__(self, other):
621 return False
622
623 - def __ne__(self, other):
624 return True
625 626 Vapor = Vapor()
627 628 629 -class Char(unicode):
630 """ 631 Strips trailing whitespace, and ignores trailing whitespace for comparisons 632 """ 633
634 - def __new__(cls, text=''):
635 if not isinstance(text, (basestring, cls)): 636 raise ValueError("Unable to automatically coerce %r to Char" % text) 637 result = unicode.__new__(cls, text.rstrip()) 638 return result
639 640 __hash__ = unicode.__hash__ 641
642 - def __eq__(self, other):
643 """ 644 ignores trailing whitespace 645 """ 646 if not isinstance(other, (self.__class__, basestring)): 647 return NotImplemented 648 return unicode(self) == other.rstrip()
649
650 - def __ge__(self, other):
651 """ 652 ignores trailing whitespace 653 """ 654 if not isinstance(other, (self.__class__, basestring)): 655 return NotImplemented 656 return unicode(self) >= other.rstrip()
657
658 - def __gt__(self, other):
659 """ 660 ignores trailing whitespace 661 """ 662 if not isinstance(other, (self.__class__, basestring)): 663 return NotImplemented 664 return unicode(self) > other.rstrip()
665
666 - def __le__(self, other):
667 """ 668 ignores trailing whitespace 669 """ 670 if not isinstance(other, (self.__class__, basestring)): 671 return NotImplemented 672 return unicode(self) <= other.rstrip()
673
674 - def __lt__(self, other):
675 """ 676 ignores trailing whitespace 677 """ 678 if not isinstance(other, (self.__class__, basestring)): 679 return NotImplemented 680 return unicode(self) < other.rstrip()
681
682 - def __ne__(self, other):
683 """ 684 ignores trailing whitespace 685 """ 686 if not isinstance(other, (self.__class__, basestring)): 687 return NotImplemented 688 return unicode(self) != other.rstrip()
689
690 - def __nonzero__(self):
691 """ 692 ignores trailing whitespace 693 """ 694 return bool(unicode(self))
695
696 - def __add__(self, other):
697 result = self.__class__(unicode(self) + other) 698 return result
699 700 basestring = str, unicode, Char 701 baseinteger = int, long
702 703 -class Date(object):
704 """ 705 adds null capable datetime.date constructs 706 """ 707 708 __slots__ = ['_date'] 709
710 - def __new__(cls, year=None, month=0, day=0):
711 """ 712 date should be either a datetime.date or date/month/day should 713 all be appropriate integers 714 """ 715 if year is None or year is Null: 716 return cls._null_date 717 nd = object.__new__(cls) 718 if isinstance(year, basestring): 719 return Date.strptime(year) 720 elif isinstance(year, (datetime.date)): 721 nd._date = year 722 elif isinstance(year, (Date)): 723 nd._date = year._date 724 else: 725 nd._date = datetime.date(year, month, day) 726 return nd
727
728 - def __add__(self, other):
729 if self and isinstance(other, (datetime.timedelta)): 730 return Date(self._date + other) 731 else: 732 return NotImplemented
733
734 - def __eq__(self, other):
735 if isinstance(other, self.__class__): 736 return self._date == other._date 737 if isinstance(other, datetime.date): 738 return self._date == other 739 if isinstance(other, type(None)): 740 return self._date is None 741 return NotImplemented
742
743 - def __format__(self, spec):
744 if self: 745 return self._date.__format__(spec) 746 return ''
747
748 - def __getattr__(self, name):
749 if name == '_date': 750 raise AttributeError('_date missing!') 751 elif self: 752 return getattr(self._date, name) 753 else: 754 raise AttributeError('NullDate object has no attribute %s' % name)
755
756 - def __ge__(self, other):
757 if isinstance(other, (datetime.date)): 758 return self._date >= other 759 elif isinstance(other, (Date)): 760 if other: 761 return self._date >= other._date 762 return False 763 return NotImplemented
764
765 - def __gt__(self, other):
766 if isinstance(other, (datetime.date)): 767 return self._date > other 768 elif isinstance(other, (Date)): 769 if other: 770 return self._date > other._date 771 return True 772 return NotImplemented
773
774 - def __hash__(self):
775 return hash(self._date)
776
777 - def __le__(self, other):
778 if self: 779 if isinstance(other, (datetime.date)): 780 return self._date <= other 781 elif isinstance(other, (Date)): 782 if other: 783 return self._date <= other._date 784 return False 785 else: 786 if isinstance(other, (datetime.date)): 787 return True 788 elif isinstance(other, (Date)): 789 if other: 790 return True 791 return True 792 return NotImplemented
793
794 - def __lt__(self, other):
795 if self: 796 if isinstance(other, (datetime.date)): 797 return self._date < other 798 elif isinstance(other, (Date)): 799 if other: 800 return self._date < other._date 801 return False 802 else: 803 if isinstance(other, (datetime.date)): 804 return True 805 elif isinstance(other, (Date)): 806 if other: 807 return True 808 return False 809 return NotImplemented
810
811 - def __ne__(self, other):
812 if self: 813 if isinstance(other, (datetime.date)): 814 return self._date != other 815 elif isinstance(other, (Date)): 816 if other: 817 return self._date != other._date 818 return True 819 else: 820 if isinstance(other, (datetime.date)): 821 return True 822 elif isinstance(other, (Date)): 823 if other: 824 return True 825 return False 826 return NotImplemented
827
828 - def __nonzero__(self):
829 return self._date is not None
830 831 __radd__ = __add__ 832
833 - def __rsub__(self, other):
834 if self and isinstance(other, (datetime.date)): 835 return other - self._date 836 elif self and isinstance(other, (Date)): 837 return other._date - self._date 838 elif self and isinstance(other, (datetime.timedelta)): 839 return Date(other - self._date) 840 else: 841 return NotImplemented
842
843 - def __repr__(self):
844 if self: 845 return "Date(%d, %d, %d)" % self.timetuple()[:3] 846 else: 847 return "Date()"
848
849 - def __str__(self):
850 if self: 851 return str(self._date) 852 return ""
853
854 - def __sub__(self, other):
855 if self and isinstance(other, (datetime.date)): 856 return self._date - other 857 elif self and isinstance(other, (Date)): 858 return self._date - other._date 859 elif self and isinstance(other, (datetime.timedelta)): 860 return Date(self._date - other) 861 else: 862 return NotImplemented
863
864 - def date(self):
865 if self: 866 return self._date 867 return None
868 869 @classmethod
870 - def fromordinal(cls, number):
871 if number: 872 return cls(datetime.date.fromordinal(number)) 873 return cls()
874 875 @classmethod
876 - def fromtimestamp(cls, timestamp):
877 return cls(datetime.date.fromtimestamp(timestamp))
878 879 @classmethod
880 - def fromymd(cls, yyyymmdd):
881 if yyyymmdd in ('', ' ', 'no date'): 882 return cls() 883 return cls(datetime.date(int(yyyymmdd[:4]), int(yyyymmdd[4:6]), int(yyyymmdd[6:])))
884
885 - def replace(self, year=None, month=None, day=None, delta_year=0, delta_month=0, delta_day=0):
886 if not self: 887 return self.__class__._null_date 888 old_year, old_month, old_day = self.timetuple()[:3] 889 if isinstance(month, RelativeMonth): 890 this_month = IsoMonth(old_month) 891 delta_month += month.months_from(this_month) 892 month = None 893 if isinstance(day, RelativeDay): 894 this_day = IsoDay(self.isoweekday()) 895 delta_day += day.days_from(this_day) 896 day = None 897 year = (year or old_year) + delta_year 898 month = (month or old_month) + delta_month 899 day = (day or old_day) + delta_day 900 days_in_month = (days_per_month, days_per_leap_month)[is_leapyear(year)] 901 while not(0 < month < 13) or not (0 < day <= days_in_month[month]): 902 while month < 1: 903 year -= 1 904 month = 12 + month 905 while month > 12: 906 year += 1 907 month = month - 12 908 days_in_month = (days_per_month, days_per_leap_month)[is_leapyear(year)] 909 while day < 1: 910 month -= 1 911 day = days_in_month[month] + day 912 if not 0 < month < 13: 913 break 914 while day > days_in_month[month]: 915 day = day - days_in_month[month] 916 month += 1 917 if not 0 < month < 13: 918 break 919 return Date(year, month, day)
920
921 - def strftime(self, format):
922 fmt_cls = type(format) 923 if self: 924 return fmt_cls(self._date.strftime(format)) 925 return fmt_cls('')
926 927 @classmethod
928 - def strptime(cls, date_string, format=None):
929 if format is not None: 930 return cls(*(time.strptime(date_string, format)[0:3])) 931 return cls(*(time.strptime(date_string, "%Y-%m-%d")[0:3]))
932 933 @classmethod
934 - def today(cls):
935 return cls(datetime.date.today())
936
937 - def ymd(self):
938 if self: 939 return "%04d%02d%02d" % self.timetuple()[:3] 940 else: 941 return ' '
942 943 Date.max = Date(datetime.date.max) 944 Date.min = Date(datetime.date.min) 945 Date._null_date = object.__new__(Date) 946 Date._null_date._date = None 947 NullDate = Date()
948 949 950 -class DateTime(object):
951 """ 952 adds null capable datetime.datetime constructs 953 """ 954 955 __slots__ = ['_datetime'] 956
957 - def __new__(cls, year=None, month=0, day=0, hour=0, minute=0, second=0, microsecond=0):
958 """year may be a datetime.datetime""" 959 if year is None or year is Null: 960 return cls._null_datetime 961 ndt = object.__new__(cls) 962 if isinstance(year, basestring): 963 return DateTime.strptime(year) 964 elif isinstance(year, (DateTime)): 965 ndt._datetime = year._datetime 966 elif isinstance(year, (datetime.datetime)): 967 microsecond = year.microsecond // 1000 * 1000 968 hour, minute, second = year.hour, year.minute, year.second 969 year, month, day = year.year, year.month, year.day 970 ndt._datetime = datetime.datetime(year, month, day, hour, minute, second, microsecond) 971 elif year is not None: 972 microsecond = microsecond // 1000 * 1000 973 ndt._datetime = datetime.datetime(year, month, day, hour, minute, second, microsecond) 974 return ndt
975
976 - def __add__(self, other):
977 if self and isinstance(other, (datetime.timedelta)): 978 return DateTime(self._datetime + other) 979 else: 980 return NotImplemented
981
982 - def __eq__(self, other):
983 if isinstance(other, self.__class__): 984 return self._datetime == other._datetime 985 if isinstance(other, datetime.date): 986 return self._datetime == other 987 if isinstance(other, type(None)): 988 return self._datetime is None 989 return NotImplemented
990
991 - def __format__(self, spec):
992 if self: 993 return self._datetime.__format__(spec) 994 return ''
995
996 - def __getattr__(self, name):
997 if name == '_datetime': 998 raise AttributeError('_datetime missing!') 999 elif self: 1000 return getattr(self._datetime, name) 1001 else: 1002 raise AttributeError('NullDateTime object has no attribute %s' % name)
1003
1004 - def __ge__(self, other):
1005 if self: 1006 if isinstance(other, (datetime.datetime)): 1007 return self._datetime >= other 1008 elif isinstance(other, (DateTime)): 1009 if other: 1010 return self._datetime >= other._datetime 1011 return False 1012 else: 1013 if isinstance(other, (datetime.datetime)): 1014 return False 1015 elif isinstance(other, (DateTime)): 1016 if other: 1017 return False 1018 return True 1019 return NotImplemented
1020
1021 - def __gt__(self, other):
1022 if self: 1023 if isinstance(other, (datetime.datetime)): 1024 return self._datetime > other 1025 elif isinstance(other, (DateTime)): 1026 if other: 1027 return self._datetime > other._datetime 1028 return True 1029 else: 1030 if isinstance(other, (datetime.datetime)): 1031 return False 1032 elif isinstance(other, (DateTime)): 1033 if other: 1034 return False 1035 return False 1036 return NotImplemented
1037
1038 - def __hash__(self):
1039 return self._datetime.__hash__()
1040
1041 - def __le__(self, other):
1042 if self: 1043 if isinstance(other, (datetime.datetime)): 1044 return self._datetime <= other 1045 elif isinstance(other, (DateTime)): 1046 if other: 1047 return self._datetime <= other._datetime 1048 return False 1049 else: 1050 if isinstance(other, (datetime.datetime)): 1051 return True 1052 elif isinstance(other, (DateTime)): 1053 if other: 1054 return True 1055 return True 1056 return NotImplemented
1057
1058 - def __lt__(self, other):
1059 if self: 1060 if isinstance(other, (datetime.datetime)): 1061 return self._datetime < other 1062 elif isinstance(other, (DateTime)): 1063 if other: 1064 return self._datetime < other._datetime 1065 return False 1066 else: 1067 if isinstance(other, (datetime.datetime)): 1068 return True 1069 elif isinstance(other, (DateTime)): 1070 if other: 1071 return True 1072 return False 1073 return NotImplemented
1074
1075 - def __ne__(self, other):
1076 if self: 1077 if isinstance(other, (datetime.datetime)): 1078 return self._datetime != other 1079 elif isinstance(other, (DateTime)): 1080 if other: 1081 return self._datetime != other._datetime 1082 return True 1083 else: 1084 if isinstance(other, (datetime.datetime)): 1085 return True 1086 elif isinstance(other, (DateTime)): 1087 if other: 1088 return True 1089 return False 1090 return NotImplemented
1091
1092 - def __nonzero__(self):
1093 return self._datetime is not None
1094 1095 __radd__ = __add__ 1096
1097 - def __rsub__(self, other):
1098 if self and isinstance(other, (datetime.datetime)): 1099 return other - self._datetime 1100 elif self and isinstance(other, (DateTime)): 1101 return other._datetime - self._datetime 1102 elif self and isinstance(other, (datetime.timedelta)): 1103 return DateTime(other - self._datetime) 1104 else: 1105 return NotImplemented
1106
1107 - def __repr__(self):
1108 if self: 1109 return "DateTime(%5d, %2d, %2d, %2d, %2d, %2d, %2d)" % ( 1110 self._datetime.timetuple()[:6] + (self._datetime.microsecond, ) 1111 ) 1112 else: 1113 return "DateTime()"
1114
1115 - def __str__(self):
1116 if self: 1117 return str(self._datetime) 1118 return ""
1119
1120 - def __sub__(self, other):
1121 if self and isinstance(other, (datetime.datetime)): 1122 return self._datetime - other 1123 elif self and isinstance(other, (DateTime)): 1124 return self._datetime - other._datetime 1125 elif self and isinstance(other, (datetime.timedelta)): 1126 return DateTime(self._datetime - other) 1127 else: 1128 return NotImplemented
1129 1130 @classmethod
1131 - def combine(cls, date, time):
1132 if Date(date) and Time(time): 1133 return cls(date.year, date.month, date.day, time.hour, time.minute, time.second, time.microsecond) 1134 return cls()
1135
1136 - def date(self):
1137 if self: 1138 return Date(self.year, self.month, self.day) 1139 return Date()
1140
1141 - def datetime(self):
1142 if self: 1143 return self._datetime 1144 return None
1145 1146 @classmethod
1147 - def fromordinal(cls, number):
1148 if number: 1149 return cls(datetime.datetime.fromordinal(number)) 1150 else: 1151 return cls()
1152 1153 @classmethod
1154 - def fromtimestamp(cls, timestamp):
1155 return DateTime(datetime.datetime.fromtimestamp(timestamp))
1156 1157 @classmethod
1158 - def now(cls):
1159 "only accurate to milliseconds" 1160 return cls(datetime.datetime.now())
1161
1162 - def replace(self, year=None, month=None, day=None, hour=None, minute=None, second=None, microsecond=None, 1163 delta_year=0, delta_month=0, delta_day=0, delta_hour=0, delta_minute=0, delta_second=0):
1164 if not self: 1165 return self.__class__._null_datetime 1166 old_year, old_month, old_day, old_hour, old_minute, old_second, old_micro = self.timetuple()[:7] 1167 if isinstance(month, RelativeMonth): 1168 this_month = IsoMonth(old_month) 1169 delta_month += month.months_from(this_month) 1170 month = None 1171 if isinstance(day, RelativeDay): 1172 this_day = IsoDay(self.isoweekday()) 1173 delta_day += day.days_from(this_day) 1174 day = None 1175 year = (year or old_year) + delta_year 1176 month = (month or old_month) + delta_month 1177 day = (day or old_day) + delta_day 1178 hour = (hour or old_hour) + delta_hour 1179 minute = (minute or old_minute) + delta_minute 1180 second = (second or old_second) + delta_second 1181 microsecond = microsecond or old_micro 1182 days_in_month = (days_per_month, days_per_leap_month)[is_leapyear(year)] 1183 while ( not (0 < month < 13) 1184 or not (0 < day <= days_in_month[month]) 1185 or not (0 <= hour < 24) 1186 or not (0 <= minute < 60) 1187 or not (0 <= second < 60) 1188 ): 1189 while month < 1: 1190 year -= 1 1191 month = 12 + month 1192 while month > 12: 1193 year += 1 1194 month = month - 12 1195 days_in_month = (days_per_month, days_per_leap_month)[is_leapyear(year)] 1196 while day < 1: 1197 month -= 1 1198 day = days_in_month[month] + day 1199 if not 0 < month < 13: 1200 break 1201 while day > days_in_month[month]: 1202 day = day - days_in_month[month] 1203 month += 1 1204 if not 0 < month < 13: 1205 break 1206 while hour < 1: 1207 day -= 1 1208 hour = 24 + hour 1209 while hour > 23: 1210 day += 1 1211 hour = hour - 24 1212 while minute < 0: 1213 hour -= 1 1214 minute = 60 + minute 1215 while minute > 59: 1216 hour += 1 1217 minute = minute - 60 1218 while second < 0: 1219 minute -= 1 1220 second = 60 + second 1221 while second > 59: 1222 minute += 1 1223 second = second - 60 1224 return DateTime(year, month, day, hour, minute, second, microsecond)
1225
1226 - def strftime(self, format):
1227 fmt_cls = type(format) 1228 if self: 1229 return fmt_cls(self._datetime.strftime(format)) 1230 return fmt_cls('')
1231 1232 @classmethod
1233 - def strptime(cls, datetime_string, format=None):
1234 if format is not None: 1235 return cls(datetime.datetime.strptime(datetime_string, format)) 1236 for format in ( 1237 "%Y-%m-%d %H:%M:%S.%f", 1238 "%Y-%m-%d %H:%M:%S", 1239 ): 1240 try: 1241 return cls(datetime.datetime.strptime(datetime_string, format)) 1242 except ValueError: 1243 pass 1244 raise ValueError("Unable to convert %r" % datetime_string)
1245
1246 - def time(self):
1247 if self: 1248 return Time(self.hour, self.minute, self.second, self.microsecond) 1249 return Time()
1250 1251 @classmethod
1252 - def utcnow(cls):
1253 return cls(datetime.datetime.utcnow())
1254 1255 @classmethod
1256 - def today(cls):
1257 return cls(datetime.datetime.today())
1258 1259 DateTime.max = DateTime(datetime.datetime.max) 1260 DateTime.min = DateTime(datetime.datetime.min) 1261 DateTime._null_datetime = object.__new__(DateTime) 1262 DateTime._null_datetime._datetime = None 1263 NullDateTime = DateTime()
1264 1265 1266 -class Time(object):
1267 """ 1268 adds null capable datetime.time constructs 1269 """ 1270 1271 __slots__ = ['_time'] 1272
1273 - def __new__(cls, hour=None, minute=0, second=0, microsecond=0):
1274 """ 1275 hour may be a datetime.time or a str(Time) 1276 """ 1277 if hour is None or hour is Null: 1278 return cls._null_time 1279 nt = object.__new__(cls) 1280 if isinstance(hour, basestring): 1281 hour = Time.strptime(hour) 1282 if isinstance(hour, (Time)): 1283 nt._time = hour._time 1284 elif isinstance(hour, (datetime.time)): 1285 microsecond = hour.microsecond // 1000 * 1000 1286 hour, minute, second = hour.hour, hour.minute, hour.second 1287 nt._time = datetime.time(hour, minute, second, microsecond) 1288 elif hour is not None: 1289 microsecond = microsecond // 1000 * 1000 1290 nt._time = datetime.time(hour, minute, second, microsecond) 1291 return nt
1292
1293 - def __add__(self, other):
1294 if self and isinstance(other, (datetime.timedelta)): 1295 t = self._time 1296 t = datetime.datetime(2012, 6, 27, t.hour, t.minute, t.second, t.microsecond) 1297 t += other 1298 return Time(t.hour, t.minute, t.second, t.microsecond) 1299 else: 1300 return NotImplemented
1301
1302 - def __eq__(self, other):
1303 if isinstance(other, self.__class__): 1304 return self._time == other._time 1305 if isinstance(other, datetime.time): 1306 return self._time == other 1307 if isinstance(other, type(None)): 1308 return self._time is None 1309 return NotImplemented
1310
1311 - def __format__(self, spec):
1312 if self: 1313 return self._time.__format__(spec) 1314 return ''
1315
1316 - def __getattr__(self, name):
1317 if name == '_time': 1318 raise AttributeError('_time missing!') 1319 elif self: 1320 return getattr(self._time, name) 1321 else: 1322 raise AttributeError('NullTime object has no attribute %s' % name)
1323
1324 - def __ge__(self, other):
1325 if self: 1326 if isinstance(other, (datetime.time)): 1327 return self._time >= other 1328 elif isinstance(other, (Time)): 1329 if other: 1330 return self._time >= other._time 1331 return False 1332 else: 1333 if isinstance(other, (datetime.time)): 1334 return False 1335 elif isinstance(other, (Time)): 1336 if other: 1337 return False 1338 return True 1339 return NotImplemented
1340
1341 - def __gt__(self, other):
1342 if self: 1343 if isinstance(other, (datetime.time)): 1344 return self._time > other 1345 elif isinstance(other, (DateTime)): 1346 if other: 1347 return self._time > other._time 1348 return True 1349 else: 1350 if isinstance(other, (datetime.time)): 1351 return False 1352 elif isinstance(other, (Time)): 1353 if other: 1354 return False 1355 return False 1356 return NotImplemented
1357
1358 - def __hash__(self):
1359 return self._datetime.__hash__()
1360
1361 - def __le__(self, other):
1362 if self: 1363 if isinstance(other, (datetime.time)): 1364 return self._time <= other 1365 elif isinstance(other, (Time)): 1366 if other: 1367 return self._time <= other._time 1368 return False 1369 else: 1370 if isinstance(other, (datetime.time)): 1371 return True 1372 elif isinstance(other, (Time)): 1373 if other: 1374 return True 1375 return True 1376 return NotImplemented
1377
1378 - def __lt__(self, other):
1379 if self: 1380 if isinstance(other, (datetime.time)): 1381 return self._time < other 1382 elif isinstance(other, (Time)): 1383 if other: 1384 return self._time < other._time 1385 return False 1386 else: 1387 if isinstance(other, (datetime.time)): 1388 return True 1389 elif isinstance(other, (Time)): 1390 if other: 1391 return True 1392 return False 1393 return NotImplemented
1394
1395 - def __ne__(self, other):
1396 if self: 1397 if isinstance(other, (datetime.time)): 1398 return self._time != other 1399 elif isinstance(other, (Time)): 1400 if other: 1401 return self._time != other._time 1402 return True 1403 else: 1404 if isinstance(other, (datetime.time)): 1405 return True 1406 elif isinstance(other, (Time)): 1407 if other: 1408 return True 1409 return False 1410 return NotImplemented
1411
1412 - def __nonzero__(self):
1413 return self._time is not None
1414 1415 __radd__ = __add__ 1416
1417 - def __rsub__(self, other):
1418 if self and isinstance(other, (Time, datetime.time)): 1419 t = self._time 1420 t = datetime.datetime(2012, 6, 27, t.hour, t.minute, t.second, t.microsecond) 1421 other = datetime.datetime(2012, 6, 27, other.hour, other.minute, other.second, other.microsecond) 1422 other -= t 1423 return other 1424 else: 1425 return NotImplemented
1426
1427 - def __repr__(self):
1428 if self: 1429 return "Time(%d, %d, %d, %d)" % (self.hour, self.minute, self.second, self.microsecond) 1430 else: 1431 return "Time()"
1432
1433 - def __str__(self):
1434 if self: 1435 return str(self._time) 1436 return ""
1437
1438 - def __sub__(self, other):
1439 if self and isinstance(other, (Time, datetime.time)): 1440 t = self._time 1441 t = datetime.datetime(2012, 6, 27, t.hour, t.minute, t.second, t.microsecond) 1442 o = datetime.datetime(2012, 6, 27, other.hour, other.minute, other.second, other.microsecond) 1443 return t - o 1444 elif self and isinstance(other, (datetime.timedelta)): 1445 t = self._time 1446 t = datetime.datetime(2012, 6, 27, t.hour, t.minute, t.second, t.microsecond) 1447 t -= other 1448 return Time(t.hour, t.minute, t.second, t.microsecond) 1449 else: 1450 return NotImplemented
1451 1452 @classmethod
1453 - def fromfloat(cls, num):
1454 "2.5 == 2 hours, 30 minutes, 0 seconds, 0 microseconds" 1455 if num < 0: 1456 raise ValueError("positive value required (got %r)" % num) 1457 if num == 0: 1458 return Time(0) 1459 hours = int(num) 1460 if hours: 1461 num = num % hours 1462 minutes = int(num * 60) 1463 if minutes: 1464 num = num * 60 % minutes 1465 else: 1466 num = num * 60 1467 seconds = int(num * 60) 1468 if seconds: 1469 num = num * 60 % seconds 1470 else: 1471 num = num * 60 1472 microseconds = int(num * 1000) 1473 return Time(hours, minutes, seconds, microseconds)
1474 1475 @staticmethod
1476 - def now():
1477 "only accurate to milliseconds" 1478 return DateTime.now().time()
1479
1480 - def replace(self, hour=None, minute=None, second=None, microsecond=None, delta_hour=0, delta_minute=0, delta_second=0):
1481 if not self: 1482 return self.__class__._null_time 1483 old_hour, old_minute, old_second, old_micro = self.hour, self.minute, self.second, self.microsecond 1484 hour = (hour or old_hour) + delta_hour 1485 minute = (minute or old_minute) + delta_minute 1486 second = (second or old_second) + delta_second 1487 microsecond = microsecond or old_micro 1488 while not (0 <= hour < 24) or not (0 <= minute < 60) or not (0 <= second < 60): 1489 while second < 0: 1490 minute -= 1 1491 second = 60 + second 1492 while second > 59: 1493 minute += 1 1494 second = second - 60 1495 while minute < 0: 1496 hour -= 1 1497 minute = 60 + minute 1498 while minute > 59: 1499 hour += 1 1500 minute = minute - 60 1501 while hour < 1: 1502 hour = 24 + hour 1503 while hour > 23: 1504 hour = hour - 24 1505 return Time(hour, minute, second, microsecond)
1506
1507 - def strftime(self, format):
1508 fmt_cls = type(format) 1509 if self: 1510 return fmt_cls(self._time.strftime(format)) 1511 return fmt_cls('')
1512 1513 @classmethod
1514 - def strptime(cls, time_string, format=None):
1515 if format is not None: 1516 return cls(datetime.time.strptime(time_string, format)) 1517 for format in ( 1518 "%H:%M:%S.%f", 1519 "%H:%M:%S", 1520 ): 1521 try: 1522 return cls(datetime.datetime.strptime(datetime_string, format)) 1523 except ValueError: 1524 pass 1525 raise ValueError("Unable to convert %r" % datetime_string)
1526
1527 - def time(self):
1528 if self: 1529 return self._time 1530 return None
1531
1532 - def tofloat(self):
1533 "returns Time as a float" 1534 hour = self.hour 1535 minute = self.minute * (1.0 / 60) 1536 second = self.second * (1.0 / 3600) 1537 microsecond = self.microsecond * (1.0 / 3600000) 1538 return hour + minute + second + microsecond
1539 1540 Time.max = Time(datetime.time.max) 1541 Time.min = Time(datetime.time.min) 1542 Time._null_time = object.__new__(Time) 1543 Time._null_time._time = None 1544 NullTime = Time()
1545 1546 1547 -class Period(object):
1548 "for matching various time ranges" 1549
1550 - def __init__(self, year=None, month=None, day=None, hour=None, minute=None, second=None, microsecond=None):
1551 params = vars() 1552 self._mask = {} 1553 for attr in ('year', 'month', 'day', 'hour', 'minute', 'second', 'microsecond'): 1554 value = params[attr] 1555 if value is not None: 1556 self._mask[attr] = value
1557
1558 - def __contains__(self, other):
1559 if not self._mask: 1560 return True 1561 for attr, value in self._mask.items(): 1562 other_value = getattr(other, attr, None) 1563 try: 1564 if other_value == value or other_value in value: 1565 continue 1566 except TypeError: 1567 pass 1568 return False 1569 return True
1570
1571 - def __repr__(self):
1572 items = [] 1573 for attr in ('year', 'month', 'day', 'hour', 'minute', 'second', 'microsecond'): 1574 if attr in self._mask: 1575 items.append('%s=%s' % (attr, self._mask[attr])) 1576 return "Period(%s)" % ', '.join(items)
1577
1578 1579 -class Logical(object):
1580 """ 1581 Logical field return type. 1582 1583 Accepts values of True, False, or None/Null 1584 """ 1585
1586 - def __new__(cls, value=None):
1587 if value is None or value is Null or value is Other or value is Unknown: 1588 return cls.unknown 1589 elif isinstance(value, basestring): 1590 if value.lower() in ('t', 'true', 'y', 'yes', 'on'): 1591 return cls.true 1592 elif value.lower() in ('f', 'false', 'n', 'no', 'off'): 1593 return cls.false 1594 elif value.lower() in ('?', 'unknown', 'null', 'none', ' ', ''): 1595 return cls.unknown 1596 else: 1597 raise ValueError('unknown value for Logical: %s' % value) 1598 else: 1599 return (cls.false, cls.true)[bool(value)]
1600
1601 - def __add__(x, y):
1602 if isinstance(y, type(None)) or y is Unknown or x is Unknown: 1603 return Unknown 1604 try: 1605 i = int(y) 1606 except Exception: 1607 return NotImplemented 1608 return int(x) + i
1609 1610 1611 __radd__ = __iadd__ = __add__ 1612
1613 - def __sub__(x, y):
1614 if isinstance(y, type(None)) or y is Unknown or x is Unknown: 1615 return Unknown 1616 try: 1617 i = int(y) 1618 except Exception: 1619 return NotImplemented 1620 return int(x) - i
1621 1622 __isub__ = __sub__ 1623
1624 - def __rsub__(y, x):
1625 if isinstance(x, type(None)) or x is Unknown or y is Unknown: 1626 return Unknown 1627 try: 1628 i = int(x) 1629 except Exception: 1630 return NotImplemented 1631 return i - int(y)
1632
1633 - def __mul__(x, y):
1634 if x == 0 or y == 0: 1635 return 0 1636 elif isinstance(y, type(None)) or y is Unknown or x is Unknown: 1637 return Unknown 1638 try: 1639 i = int(y) 1640 except Exception: 1641 return NotImplemented 1642 return int(x) * i
1643 1644 __rmul__ = __imul__ = __mul__ 1645
1646 - def __div__(x, y):
1647 if isinstance(y, type(None)) or y == 0 or y is Unknown or x is Unknown: 1648 return Unknown 1649 try: 1650 i = int(y) 1651 except Exception: 1652 return NotImplemented 1653 return int(x).__div__(i)
1654 1655 __idiv__ = __div__ 1656
1657 - def __rdiv__(y, x):
1658 if isinstance(x, type(None)) or y == 0 or x is Unknown or y is Unknown: 1659 return Unknown 1660 try: 1661 i = int(x) 1662 except Exception: 1663 return NotImplemented 1664 return i.__div__(int(y))
1665
1666 - def __truediv__(x, y):
1667 if isinstance(y, type(None)) or y == 0 or y is Unknown or x is Unknown: 1668 return Unknown 1669 try: 1670 i = int(y) 1671 except Exception: 1672 return NotImplemented 1673 return int(x).__truediv__(i)
1674 1675 __itruediv__ = __truediv__ 1676
1677 - def __rtruediv__(y, x):
1678 if isinstance(x, type(None)) or y == 0 or y is Unknown or x is Unknown: 1679 return Unknown 1680 try: 1681 i = int(x) 1682 except Exception: 1683 return NotImplemented 1684 return i.__truediv__(int(y))
1685
1686 - def __floordiv__(x, y):
1687 if isinstance(y, type(None)) or y == 0 or y is Unknown or x is Unknown: 1688 return Unknown 1689 try: 1690 i = int(y) 1691 except Exception: 1692 return NotImplemented 1693 return int(x).__floordiv__(i)
1694 1695 __ifloordiv__ = __floordiv__ 1696
1697 - def __rfloordiv__(y, x):
1698 if isinstance(x, type(None)) or y == 0 or y is Unknown or x is Unknown: 1699 return Unknown 1700 try: 1701 i = int(x) 1702 except Exception: 1703 return NotImplemented 1704 return i.__floordiv__(int(y))
1705
1706 - def __divmod__(x, y):
1707 if isinstance(y, type(None)) or y == 0 or y is Unknown or x is Unknown: 1708 return (Unknown, Unknown) 1709 try: 1710 i = int(y) 1711 except Exception: 1712 return NotImplemented 1713 return divmod(int(x), i)
1714
1715 - def __rdivmod__(y, x):
1716 if isinstance(x, type(None)) or y == 0 or y is Unknown or x is Unknown: 1717 return (Unknown, Unknown) 1718 try: 1719 i = int(x) 1720 except Exception: 1721 return NotImplemented 1722 return divmod(i, int(y))
1723
1724 - def __mod__(x, y):
1725 if isinstance(y, type(None)) or y == 0 or y is Unknown or x is Unknown: 1726 return Unknown 1727 try: 1728 i = int(y) 1729 except Exception: 1730 return NotImplemented 1731 return int(x) % i
1732 1733 __imod__ = __mod__ 1734
1735 - def __rmod__(y, x):
1736 if isinstance(x, type(None)) or y == 0 or x is Unknown or y is Unknown: 1737 return Unknown 1738 try: 1739 i = int(x) 1740 except Exception: 1741 return NotImplemented 1742 return i % int(y)
1743
1744 - def __pow__(x, y):
1745 if not isinstance(y, (x.__class__, bool, type(None), int)): 1746 return NotImplemented 1747 if isinstance(y, type(None)) or y is Unknown: 1748 return Unknown 1749 i = int(y) 1750 if i == 0: 1751 return 1 1752 if x is Unknown: 1753 return Unknown 1754 return int(x) ** i
1755 1756 __ipow__ = __pow__ 1757
1758 - def __rpow__(y, x):
1759 if not isinstance(x, (y.__class__, bool, type(None), int)): 1760 return NotImplemented 1761 if y is Unknown: 1762 return Unknown 1763 i = int(y) 1764 if i == 0: 1765 return 1 1766 if x is Unknown or isinstance(x, type(None)): 1767 return Unknown 1768 return int(x) ** i
1769
1770 - def __lshift__(x, y):
1771 if isinstance(y, type(None)) or x is Unknown or y is Unknown: 1772 return Unknown 1773 return int(x.value) << int(y)
1774 1775 __ilshift__ = __lshift__ 1776
1777 - def __rlshift__(y, x):
1778 if isinstance(x, type(None)) or x is Unknown or y is Unknown: 1779 return Unknown 1780 return int(x) << int(y)
1781
1782 - def __rshift__(x, y):
1783 if isinstance(y, type(None)) or x is Unknown or y is Unknown: 1784 return Unknown 1785 return int(x.value) >> int(y)
1786 1787 __irshift__ = __rshift__ 1788
1789 - def __rrshift__(y, x):
1790 if isinstance(x, type(None)) or x is Unknown or y is Unknown: 1791 return Unknown 1792 return int(x) >> int(y)
1793
1794 - def __neg__(x):
1795 "NEG (negation)" 1796 if x in (Truth, Falsth): 1797 return -x.value 1798 return Unknown
1799
1800 - def __pos__(x):
1801 "POS (posation)" 1802 if x in (Truth, Falsth): 1803 return +x.value 1804 return Unknown
1805
1806 - def __abs__(x):
1807 if x in (Truth, Falsth): 1808 return abs(x.value) 1809 return Unknown
1810
1811 - def __invert__(x):
1812 if x in (Truth, Falsth): 1813 return ~x.value 1814 return Unknown
1815
1816 - def __complex__(x):
1817 if x.value is None: 1818 raise ValueError("unable to return complex() of %r" % x) 1819 return complex(x.value)
1820
1821 - def __int__(x):
1822 if x.value is None: 1823 raise ValueError("unable to return int() of %r" % x) 1824 return int(x.value)
1825
1826 - def __long__(x):
1827 if x.value is None: 1828 raise ValueError("unable to return long() of %r" % x) 1829 return long(x.value)
1830
1831 - def __float__(x):
1832 if x.value is None: 1833 raise ValueError("unable to return float() of %r" % x) 1834 return float(x.value)
1835
1836 - def __oct__(x):
1837 if x.value is None: 1838 raise ValueError("unable to return oct() of %r" % x) 1839 return oct(x.value)
1840
1841 - def __hex__(x):
1842 if x.value is None: 1843 raise ValueError("unable to return hex() of %r" % x) 1844 return hex(x.value)
1845
1846 - def __and__(x, y):
1847 """ 1848 AND (conjunction) x & y: 1849 True iff both x, y are True 1850 False iff at least one of x, y is False 1851 Unknown otherwise 1852 """ 1853 if (isinstance(x, int) and not isinstance(x, bool)) or (isinstance(y, int) and not isinstance(y, bool)): 1854 if x == 0 or y == 0: 1855 return 0 1856 elif x is Unknown or y is Unknown: 1857 return Unknown 1858 return int(x) & int(y) 1859 elif x in (False, Falsth) or y in (False, Falsth): 1860 return Falsth 1861 elif x in (True, Truth) and y in (True, Truth): 1862 return Truth 1863 elif isinstance(x, type(None)) or isinstance(y, type(None)) or y is Unknown or x is Unknown: 1864 return Unknown 1865 return NotImplemented
1866 1867 __rand__ = __and__ 1868
1869 - def __or__(x, y):
1870 "OR (disjunction): x | y => True iff at least one of x, y is True" 1871 if (isinstance(x, int) and not isinstance(x, bool)) or (isinstance(y, int) and not isinstance(y, bool)): 1872 if x is Unknown or y is Unknown: 1873 return Unknown 1874 return int(x) | int(y) 1875 elif x in (True, Truth) or y in (True, Truth): 1876 return Truth 1877 elif x in (False, Falsth) and y in (False, Falsth): 1878 return Falsth 1879 elif isinstance(x, type(None)) or isinstance(y, type(None)) or y is Unknown or x is Unknown: 1880 return Unknown 1881 return NotImplemented
1882 1883 __ror__ = __or__ 1884
1885 - def __xor__(x, y):
1886 "XOR (parity) x ^ y: True iff only one of x,y is True" 1887 if (isinstance(x, int) and not isinstance(x, bool)) or (isinstance(y, int) and not isinstance(y, bool)): 1888 if x is Unknown or y is Unknown: 1889 return Unknown 1890 return int(x) ^ int(y) 1891 elif x in (True, Truth, False, Falsth) and y in (True, Truth, False, Falsth): 1892 return { 1893 (True, True) : Falsth, 1894 (True, False) : Truth, 1895 (False, True) : Truth, 1896 (False, False): Falsth, 1897 }[(x, y)] 1898 elif isinstance(x, type(None)) or isinstance(y, type(None)) or y is Unknown or x is Unknown: 1899 return Unknown 1900 return NotImplemented
1901 1902 __rxor__ = __xor__ 1903
1904 - def __nonzero__(x):
1905 if x is Unknown: 1906 raise TypeError('True/False value of %r is unknown' % x) 1907 return x.value is True
1908
1909 - def __eq__(x, y):
1910 if isinstance(y, x.__class__): 1911 return x.value == y.value 1912 elif isinstance(y, (bool, type(None), int)): 1913 return x.value == y 1914 return NotImplemented
1915
1916 - def __ge__(x, y):
1917 if isinstance(y, type(None)) or x is Unknown or y is Unknown: 1918 return x.value == None 1919 elif isinstance(y, x.__class__): 1920 return x.value >= y.value 1921 elif isinstance(y, (bool, int)): 1922 return x.value >= y 1923 return NotImplemented
1924
1925 - def __gt__(x, y):
1926 if isinstance(y, type(None)) or x is Unknown or y is Unknown: 1927 return False 1928 elif isinstance(y, x.__class__): 1929 return x.value > y.value 1930 elif isinstance(y, (bool, int)): 1931 return x.value > y 1932 return NotImplemented
1933
1934 - def __le__(x, y):
1935 if isinstance(y, type(None)) or x is Unknown or y is Unknown: 1936 return x.value == None 1937 elif isinstance(y, x.__class__): 1938 return x.value <= y.value 1939 elif isinstance(y, (bool, int)): 1940 return x.value <= y 1941 return NotImplemented
1942
1943 - def __lt__(x, y):
1944 if isinstance(y, type(None)) or x is Unknown or y is Unknown: 1945 return False 1946 elif isinstance(y, x.__class__): 1947 return x.value < y.value 1948 elif isinstance(y, (bool, int)): 1949 return x.value < y 1950 return NotImplemented
1951
1952 - def __ne__(x, y):
1953 if isinstance(y, x.__class__): 1954 return x.value != y.value 1955 elif isinstance(y, (bool, type(None), int)): 1956 return x.value != y 1957 return NotImplemented
1958
1959 - def __hash__(x):
1960 return hash(x.value)
1961
1962 - def __index__(x):
1963 if x.value is None: 1964 raise ValueError("unable to return index of %r" % x) 1965 return x.value
1966
1967 - def __repr__(x):
1968 return "Logical(%r)" % x.string
1969
1970 - def __str__(x):
1971 return x.string
1972 1973 Logical.true = object.__new__(Logical) 1974 Logical.true.value = True 1975 Logical.true.string = 'T' 1976 Logical.false = object.__new__(Logical) 1977 Logical.false.value = False 1978 Logical.false.string = 'F' 1979 Logical.unknown = object.__new__(Logical) 1980 Logical.unknown.value = None 1981 Logical.unknown.string = '?' 1982 Truth = Logical(True) 1983 Falsth = Logical(False) 1984 Unknown = Logical()
1985 1986 1987 -class Quantum(object):
1988 """ 1989 Logical field return type that implements boolean algebra 1990 1991 Accepts values of True/On, False/Off, or None/Null/Unknown/Other 1992 """ 1993
1994 - def __new__(cls, value=None):
1995 if value is None or value is Null or value is Other or value is Unknown: 1996 return cls.unknown 1997 elif isinstance(value, basestring): 1998 if value.lower() in ('t', 'true', 'y', 'yes', 'on'): 1999 return cls.true 2000 elif value.lower() in ('f', 'false', 'n', 'no', 'off'): 2001 return cls.false 2002 elif value.lower() in ('?', 'unknown', 'null', 'none', ' ', ''): 2003 return cls.unknown 2004 else: 2005 raise ValueError('unknown value for Quantum: %s' % value) 2006 else: 2007 return (cls.false, cls.true)[bool(value)]
2008
2009 - def A(x, y):
2010 "OR (disjunction): x | y => True iff at least one of x, y is True" 2011 if not isinstance(y, (x.__class__, bool, NullType, type(None))): 2012 return NotImplemented 2013 if x.value is True or y is not Other and y == True: 2014 return x.true 2015 elif x.value is False and y is not Other and y == False: 2016 return x.false 2017 return Other
2018
2019 - def _C_material(x, y):
2020 "IMP (material implication) x >> y => False iff x == True and y == False" 2021 if not isinstance(y, (x.__class__, bool, NullType, type(None))): 2022 return NotImplemented 2023 if (x.value is False 2024 or (x.value is True and y is not Other and y == True)): 2025 return x.true 2026 elif x.value is True and y is not Other and y == False: 2027 return False 2028 return Other
2029
2030 - def _C_material_reversed(y, x):
2031 "IMP (material implication) x >> y => False iff x = True and y = False" 2032 if not isinstance(x, (y.__class__, bool, NullType, type(None))): 2033 return NotImplemented 2034 if (x is not Other and x == False 2035 or (x is not Other and x == True and y.value is True)): 2036 return y.true 2037 elif x is not Other and x == True and y.value is False: 2038 return y.false 2039 return Other
2040
2041 - def _C_relevant(x, y):
2042 "IMP (relevant implication) x >> y => True iff both x, y are True, False iff x == True and y == False, Other if x is False" 2043 if not isinstance(y, (x.__class__, bool, NullType, type(None))): 2044 return NotImplemented 2045 if x.value is True and y is not Other and y == True: 2046 return x.true 2047 if x.value is True and y is not Other and y == False: 2048 return x.false 2049 return Other
2050
2051 - def _C_relevant_reversed(y, x):
2052 "IMP (relevant implication) x >> y => True iff both x, y are True, False iff x == True and y == False, Other if y is False" 2053 if not isinstance(x, (y.__class__, bool, NullType, type(None))): 2054 return NotImplemented 2055 if x is not Other and x == True and y.value is True: 2056 return y.true 2057 if x is not Other and x == True and y.value is False: 2058 return y.false 2059 return Other
2060
2061 - def D(x, y):
2062 "NAND (negative AND) x.D(y): False iff x and y are both True" 2063 if not isinstance(y, (x.__class__, bool, NullType, type(None))): 2064 return NotImplemented 2065 if x.value is False or y is not Other and y == False: 2066 return x.true 2067 elif x.value is True and y is not Other and y == True: 2068 return x.false 2069 return Other
2070
2071 - def E(x, y):
2072 "EQV (equivalence) x.E(y): True iff x and y are the same" 2073 if not isinstance(y, (x.__class__, bool, NullType, type(None))): 2074 return NotImplemented 2075 elif ( 2076 (x.value is True and y is not Other and y == True) 2077 or 2078 (x.value is False and y is not Other and y == False) 2079 ): 2080 return x.true 2081 elif ( 2082 (x.value is True and y is not Other and y == False) 2083 or 2084 (x.value is False and y is not Other and y == True) 2085 ): 2086 return x.false 2087 return Other
2088
2089 - def J(x, y):
2090 "XOR (parity) x ^ y: True iff only one of x,y is True" 2091 if not isinstance(y, (x.__class__, bool, NullType, type(None))): 2092 return NotImplemented 2093 if ( 2094 (x.value is True and y is not Other and y == False) 2095 or 2096 (x.value is False and y is not Other and y == True) 2097 ): 2098 return x.true 2099 if ( 2100 (x.value is False and y is not Other and y == False) 2101 or 2102 (x.value is True and y is not Other and y == True) 2103 ): 2104 return x.false 2105 return Other
2106
2107 - def K(x, y):
2108 "AND (conjunction) x & y: True iff both x, y are True" 2109 if not isinstance(y, (x.__class__, bool, NullType, type(None))): 2110 return NotImplemented 2111 if x.value is True and y is not Other and y == True: 2112 return x.true 2113 elif x.value is False or y is not Other and y == False: 2114 return x.false 2115 return Other
2116
2117 - def N(x):
2118 "NEG (negation) -x: True iff x = False" 2119 if x is x.true: 2120 return x.false 2121 elif x is x.false: 2122 return x.true 2123 return Other
2124 2125 @classmethod
2126 - def set_implication(cls, method):
2127 "sets IMP to material or relevant" 2128 if not isinstance(method, basestring) or method.lower() not in ('material', 'relevant'): 2129 raise ValueError("method should be 'material' (for strict boolean) or 'relevant', not %r'" % method) 2130 if method.lower() == 'material': 2131 cls.C = cls._C_material 2132 cls.__rshift__ = cls._C_material 2133 cls.__rrshift__ = cls._C_material_reversed 2134 elif method.lower() == 'relevant': 2135 cls.C = cls._C_relevant 2136 cls.__rshift__ = cls._C_relevant 2137 cls.__rrshift__ = cls._C_relevant_reversed
2138
2139 - def __eq__(x, y):
2140 if not isinstance(y, (x.__class__, bool, NullType, type(None))): 2141 return NotImplemented 2142 if ( 2143 (x.value is True and y is not Other and y == True) 2144 or 2145 (x.value is False and y is not Other and y == False) 2146 ): 2147 return x.true 2148 elif ( 2149 (x.value is True and y is not Other and y == False) 2150 or 2151 (x.value is False and y is not Other and y == True) 2152 ): 2153 return x.false 2154 return Other
2155
2156 - def __hash__(x):
2157 return hash(x.value)
2158
2159 - def __ne__(x, y):
2160 if not isinstance(y, (x.__class__, bool, NullType, type(None))): 2161 return NotImplemented 2162 if ( 2163 (x.value is True and y is not Other and y == False) 2164 or 2165 (x.value is False and y is not Other and y == True) 2166 ): 2167 return x.true 2168 elif ( 2169 (x.value is True and y is not Other and y == True) 2170 or 2171 (x.value is False and y is not Other and y == False) 2172 ): 2173 return x.false 2174 return Other
2175
2176 - def __nonzero__(x):
2177 if x is Other: 2178 raise TypeError('True/False value of %r is unknown' % x) 2179 return x.value is True
2180
2181 - def __repr__(x):
2182 return "Quantum(%r)" % x.string
2183
2184 - def __str__(x):
2185 return x.string
2186 2187 __add__ = A 2188 __and__ = K 2189 __mul__ = K 2190 __neg__ = N 2191 __or__ = A 2192 __radd__ = A 2193 __rand__ = K 2194 __rshift__ = None 2195 __rmul__ = K 2196 __ror__ = A 2197 __rrshift__ = None 2198 __rxor__ = J 2199 __xor__ = J
2200 2201 Quantum.true = object.__new__(Quantum) 2202 Quantum.true.value = True 2203 Quantum.true.string = 'Y' 2204 Quantum.false = object.__new__(Quantum) 2205 Quantum.false.value = False 2206 Quantum.false.string = 'N' 2207 Quantum.unknown = object.__new__(Quantum) 2208 Quantum.unknown.value = None 2209 Quantum.unknown.string = '?' 2210 Quantum.set_implication('material') 2211 On = Quantum(True) 2212 Off = Quantum(False) 2213 Other = Quantum() 2214 2215 2216 # add xmlrpc support 2217 from xmlrpclib import Marshaller 2218 Marshaller.dispatch[Char] = Marshaller.dump_unicode 2219 Marshaller.dispatch[Logical] = Marshaller.dump_bool 2220 Marshaller.dispatch[DateTime] = Marshaller.dump_datetime 2221 del Marshaller
2222 2223 2224 # Internal classes 2225 2226 -class _Navigation(object):
2227 """ 2228 Navigation base class that provides VPFish movement methods 2229 """ 2230 2231 _index = -1 2232
2233 - def _nav_check(self):
2234 """ 2235 implemented by subclass; must return True if underlying structure meets need 2236 """ 2237 raise NotImplementedError()
2238
2239 - def _get_index(self, direction, n=1, start=None):
2240 """ 2241 returns index of next available record towards direction 2242 """ 2243 if start is not None: 2244 index = start 2245 else: 2246 index = self._index 2247 if direction == 'reverse': 2248 move = -1 * n 2249 limit = 0 2250 index += move 2251 if index < limit: 2252 return -1 2253 else: 2254 return index 2255 elif direction == 'forward': 2256 move = +1 * n 2257 limit = len(self) - 1 2258 index += move 2259 if index > limit: 2260 return len(self) 2261 else: 2262 return index 2263 else: 2264 raise ValueError("direction should be 'forward' or 'reverse', not %r" % direction)
2265 2266 @property
2267 - def bof(self):
2268 """ 2269 returns True if no more usable records towards the beginning of the table 2270 """ 2271 self._nav_check() 2272 index = self._get_index('reverse') 2273 return index == -1
2274
2275 - def bottom(self):
2276 """ 2277 sets record index to bottom of table (end of table) 2278 """ 2279 self._nav_check() 2280 self._index = len(self) 2281 return self._index
2282 2283 @property
2284 - def current_record(self):
2285 """ 2286 returns current record (deleted or not) 2287 """ 2288 self._nav_check() 2289 index = self._index 2290 if index < 0: 2291 return RecordVaporWare('bof', self) 2292 elif index >= len(self): 2293 return RecordVaporWare('eof', self) 2294 return self[index]
2295 2296 @property
2297 - def current(self):
2298 """ 2299 returns current index 2300 """ 2301 self._nav_check() 2302 return self._index
2303 2304 @property
2305 - def eof(self):
2306 """ 2307 returns True if no more usable records towards the end of the table 2308 """ 2309 self._nav_check() 2310 index = self._get_index('forward') 2311 return index == len(self)
2312 2313 @property
2314 - def first_record(self):
2315 """ 2316 returns first available record (does not move index) 2317 """ 2318 self._nav_check() 2319 index = self._get_index('forward', start=-1) 2320 if -1 < index < len(self): 2321 return self[index] 2322 else: 2323 return RecordVaporWare('bof', self)
2324
2325 - def goto(self, where):
2326 """ 2327 changes the record pointer to the first matching (deleted) record 2328 where should be either an integer, or 'top' or 'bottom'. 2329 top -> before first record 2330 bottom -> after last record 2331 """ 2332 self._nav_check() 2333 max = len(self) 2334 if isinstance(where, baseinteger): 2335 if not -max <= where < max: 2336 raise IndexError("Record %d does not exist" % where) 2337 if where < 0: 2338 where += max 2339 self._index = where 2340 return self._index 2341 move = getattr(self, where, None) 2342 if move is None: 2343 raise DbfError("unable to go to %r" % where) 2344 return move()
2345 2346 @property
2347 - def last_record(self):
2348 """ 2349 returns last available record (does not move index) 2350 """ 2351 self._nav_check() 2352 index = self._get_index('reverse', start=len(self)) 2353 if -1 < index < len(self): 2354 return self[index] 2355 else: 2356 return RecordVaporWare('bof', self)
2357 2358 @property
2359 - def next_record(self):
2360 """ 2361 returns next available record (does not move index) 2362 """ 2363 self._nav_check() 2364 index = self._get_index('forward') 2365 if -1 < index < len(self): 2366 return self[index] 2367 else: 2368 return RecordVaporWare('eof', self)
2369 2370 @property
2371 - def prev_record(self):
2372 """ 2373 returns previous available record (does not move index) 2374 """ 2375 self._nav_check() 2376 index = self._get_index('reverse') 2377 if -1 < index < len(self): 2378 return self[index] 2379 else: 2380 return RecordVaporWare('bof', self)
2381
2382 - def skip(self, n=1):
2383 """ 2384 move index to the next nth available record 2385 """ 2386 self._nav_check() 2387 if n < 0: 2388 n *= -1 2389 direction = 'reverse' 2390 else: 2391 direction = 'forward' 2392 self._index = index = self._get_index(direction, n) 2393 if index < 0: 2394 raise Bof() 2395 elif index >= len(self): 2396 raise Eof() 2397 else: 2398 return index
2399
2400 - def top(self):
2401 """ 2402 sets record index to top of table (beginning of table) 2403 """ 2404 self._nav_check() 2405 self._index = -1 2406 return self._index
2407
2408 2409 -class Record(object):
2410 """ 2411 Provides routines to extract and save data within the fields of a 2412 dbf record. 2413 """ 2414 2415 __slots__ = ('_recnum', '_meta', '_data', '_old_data', '_dirty', 2416 '_memos', '_write_to_disk', '__weakref__') 2417
2418 - def __new__(cls, recnum, layout, kamikaze='', _fromdisk=False):
2419 """ 2420 record = ascii array of entire record; 2421 layout=record specification; 2422 memo = memo object for table 2423 """ 2424 record = object.__new__(cls) 2425 record._dirty = False 2426 record._recnum = recnum 2427 record._meta = layout 2428 record._memos = {} 2429 record._write_to_disk = True 2430 record._old_data = None 2431 header = layout.header 2432 record._data = layout.blankrecord[:] 2433 if kamikaze and len(record._data) != len(kamikaze): 2434 raise BadDataError("record data is not the correct length (should be %r, not %r)" % 2435 (len(record._data), len(kamikaze)), data=kamikaze[:]) 2436 if recnum == -1: # not a disk-backed record 2437 return record 2438 elif type(kamikaze) == array: 2439 record._data = kamikaze[:] 2440 elif type(kamikaze) == str: 2441 if kamikaze: 2442 record._data = array('c', kamikaze) 2443 else: 2444 raise BadDataError("%r recieved for record data" % kamikaze) 2445 if record._data[0] == '\x00': 2446 record._data[0] = ' ' 2447 if record._data[0] not in (' ', '*', '\x00'): # the null byte strikes again! 2448 raise DbfError("record data not correct -- first character should be a ' ' or a '*'.") 2449 if not _fromdisk and layout.location == ON_DISK: 2450 record._update_disk() 2451 return record
2452
2453 - def __contains__(self, value):
2454 for field in self._meta.user_fields: 2455 if self[field] == value: 2456 return True 2457 return False
2458
2459 - def __enter__(self):
2460 if not self._write_to_disk: 2461 raise DbfError("`with record` is not reentrant") 2462 self._start_flux() 2463 return self
2464
2465 - def __eq__(self, other):
2466 if not isinstance(other, (Record, RecordTemplate, dict, tuple)): 2467 return NotImplemented 2468 if isinstance(other, (Record, RecordTemplate)): 2469 if field_names(self) != field_names(other): 2470 return False 2471 for field in self._meta.user_fields: 2472 s_value, o_value = self[field], other[field] 2473 if s_value is not o_value and s_value != o_value: 2474 return False 2475 elif isinstance(other, dict): 2476 if sorted(field_names(self)) != sorted(other.keys()): 2477 return False 2478 for field in self._meta.user_fields: 2479 s_value, o_value = self[field], other[field] 2480 if s_value is not o_value and s_value != o_value: 2481 return False 2482 else: # tuple 2483 if len(self) != len(other): 2484 return False 2485 for s_value, o_value in zip(self, other): 2486 if s_value is not o_value and s_value != o_value: 2487 return False 2488 return True
2489
2490 - def __exit__(self, *args):
2491 if args == (None, None, None): 2492 self._commit_flux() 2493 else: 2494 self._rollback_flux()
2495
2496 - def __iter__(self):
2497 return (self[field] for field in self._meta.user_fields)
2498
2499 - def __getattr__(self, name):
2500 if name[0:2] == '__' and name[-2:] == '__': 2501 raise AttributeError('Method %s is not implemented.' % name) 2502 if not name in self._meta.fields: 2503 raise FieldMissingError(name) 2504 if name in self._memos: 2505 return self._memos[name] 2506 try: 2507 index = self._meta.fields.index(name) 2508 value = self._retrieve_field_value(index, name) 2509 return value 2510 except DbfError: 2511 error = sys.exc_info()[1] 2512 error.message = "field --%s-- is %s -> %s" % (name, self._meta.fieldtypes[fielddef['type']]['Type'], error.message) 2513 raise
2514
2515 - def __getitem__(self, item):
2516 if isinstance(item, baseinteger): 2517 fields = self._meta.user_fields 2518 field_count = len(fields) 2519 if not -field_count <= item < field_count: 2520 raise NotFoundError("Field offset %d is not in record" % item) 2521 field = fields[item] 2522 if field in self._memos: 2523 return self._memos[field] 2524 return self[field] 2525 elif isinstance(item, slice): 2526 sequence = [] 2527 if isinstance(item.start, basestring) or isinstance(item.stop, basestring): 2528 field_names = dbf.field_names(self) 2529 start, stop, step = item.start, item.stop, item.step 2530 if start not in field_names or stop not in field_names: 2531 raise MissingFieldError("Either %r or %r (or both) are not valid field names" % (start, stop)) 2532 if step is not None and not isinstance(step, baseinteger): 2533 raise DbfError("step value must be an int or long, not %r" % type(step)) 2534 start = field_names.index(start) 2535 stop = field_names.index(stop) + 1 2536 item = slice(start, stop, step) 2537 for index in self._meta.fields[item]: 2538 sequence.append(self[index]) 2539 return sequence 2540 elif isinstance(item, basestring): 2541 return self.__getattr__(item) 2542 else: 2543 raise TypeError("%r is not a field name" % item)
2544
2545 - def __len__(self):
2546 return self._meta.user_field_count
2547
2548 - def __ne__(self, other):
2549 if not isinstance(other, (Record, RecordTemplate, dict, tuple)): 2550 return NotImplemented 2551 return not self == other
2552
2553 - def __setattr__(self, name, value):
2554 if name in self.__slots__: 2555 object.__setattr__(self, name, value) 2556 return 2557 if self._meta.status != READ_WRITE: 2558 raise DbfError("%s not in read/write mode" % self._meta.filename) 2559 elif self._write_to_disk: 2560 raise DbfError("unable to modify fields individually except in `with` or `Process()`") 2561 elif not name in self._meta.fields: 2562 raise FieldMissingError(name) 2563 if name in self._meta.memofields: 2564 self._memos[name] = value 2565 self._dirty = True 2566 return 2567 index = self._meta.fields.index(name) 2568 try: 2569 self._update_field_value(index, name, value) 2570 except DbfError: 2571 error = sys.exc_info()[1] 2572 fielddef = self._meta[name] 2573 message = "%s (%s) = %r --> %s" % (name, self._meta.fieldtypes[fielddef[TYPE]]['Type'], value, error.args) 2574 data = name 2575 err_cls = error.__class__ 2576 raise err_cls(message, data)
2577
2578 - def __setitem__(self, name, value):
2579 if self._meta.status != READ_WRITE: 2580 raise DbfError("%s not in read/write mode" % self._meta.filename) 2581 if self._write_to_disk: 2582 raise DbfError("unable to modify fields individually except in `with` or `Process()`") 2583 if isinstance(name, basestring): 2584 self.__setattr__(name, value) 2585 elif isinstance(name, baseinteger): 2586 self.__setattr__(self._meta.fields[name], value) 2587 elif isinstance(name, slice): 2588 sequence = [] 2589 field_names = dbf.field_names(self) 2590 if isinstance(name.start, basestring) or isinstance(name.stop, basestring): 2591 start, stop, step = name.start, name.stop, name.step 2592 if start not in field_names or stop not in field_names: 2593 raise MissingFieldError("Either %r or %r (or both) are not valid field names" % (start, stop)) 2594 if step is not None and not isinstance(step, baseinteger): 2595 raise DbfError("step value must be an int or long, not %r" % type(step)) 2596 start = field_names.index(start) 2597 stop = field_names.index(stop) + 1 2598 name = slice(start, stop, step) 2599 for field in self._meta.fields[name]: 2600 sequence.append(field) 2601 if len(sequence) != len(value): 2602 raise DbfError("length of slices not equal") 2603 for field, val in zip(sequence, value): 2604 self[field] = val 2605 else: 2606 raise TypeError("%s is not a field name" % name)
2607
2608 - def __str__(self):
2609 result = [] 2610 for seq, field in enumerate(field_names(self)): 2611 result.append("%3d - %-10s: %r" % (seq, field, self[field])) 2612 return '\n'.join(result)
2613
2614 - def __repr__(self):
2615 return self._data.tostring()
2616
2617 - def _commit_flux(self):
2618 """ 2619 stores field updates to disk; if any errors restores previous contents and propogates exception 2620 """ 2621 if self._write_to_disk: 2622 raise DbfError("record not in flux") 2623 try: 2624 self._write() 2625 except Exception: 2626 exc = sys.exc_info()[1] 2627 self._data[:] = self._old_data 2628 self._update_disk(data=self._old_data) 2629 raise DbfError("unable to write updates to disk, original data restored: %r" % (exc,)) 2630 self._memos.clear() 2631 self._old_data = None 2632 self._write_to_disk = True 2633 self._reindex_record()
2634 2635 @classmethod
2636 - def _create_blank_data(cls, layout):
2637 """ 2638 creates a blank record data chunk 2639 """ 2640 record = object.__new__(cls) 2641 record._dirty = False 2642 record._recnum = -1 2643 record._meta = layout 2644 record._data = array('c', ' ' * layout.header.record_length) 2645 layout.memofields = [] 2646 signature = [layout.table().codepage.name] 2647 for index, name in enumerate(layout.fields): 2648 if name == '_nullflags': 2649 record._data[layout['_nullflags'][START]:layout['_nullflags'][END]] = array('c', chr(0) * layout['_nullflags'][LENGTH]) 2650 for index, name in enumerate(layout.fields): 2651 signature.append(name) 2652 if name != '_nullflags': 2653 type = layout[name][TYPE] 2654 start = layout[name][START] 2655 size = layout[name][LENGTH] 2656 end = layout[name][END] 2657 record._data[start:end] = array('c', layout.fieldtypes[type]['Blank'](size)) 2658 if layout[name][TYPE] in layout.memo_types: 2659 layout.memofields.append(name) 2660 decimals = layout[name][DECIMALS] 2661 signature[-1] = '_'.join([str(x) for x in (signature[-1], type, size, decimals)]) 2662 layout.blankrecord = record._data[:] 2663 data_types = [] 2664 for fieldtype, defs in sorted(layout.fieldtypes.items()): 2665 if fieldtype != '0': # ignore the nullflags field 2666 data_types.append("%s_%s_%s" % (fieldtype, defs['Empty'], defs['Class'])) 2667 layout.record_sig = ('___'.join(signature), '___'.join(data_types))
2668
2669 - def _reindex_record(self):
2670 """ 2671 rerun all indices with this record 2672 """ 2673 if self._meta.status == CLOSED: 2674 raise DbfError("%s is closed; cannot alter indices" % self._meta.filename) 2675 elif not self._write_to_disk: 2676 raise DbfError("unable to reindex record until it is written to disk") 2677 for dbfindex in self._meta.table()._indexen: 2678 dbfindex(self)
2679
2680 - def _retrieve_field_value(self, index, name):
2681 """ 2682 calls appropriate routine to convert value stored in field from array 2683 """ 2684 fielddef = self._meta[name] 2685 flags = fielddef[FLAGS] 2686 nullable = flags & NULLABLE and '_nullflags' in self._meta 2687 binary = flags & BINARY 2688 if nullable: 2689 byte, bit = divmod(index, 8) 2690 null_def = self._meta['_nullflags'] 2691 null_data = self._data[null_def[START]:null_def[END]] 2692 try: 2693 if ord(null_data[byte]) >> bit & 1: 2694 return Null 2695 except IndexError: 2696 print(null_data) 2697 print(index) 2698 print(byte, bit) 2699 print(len(self._data), self._data) 2700 print(null_def) 2701 print(null_data) 2702 raise 2703 2704 record_data = self._data[fielddef[START]:fielddef[END]] 2705 field_type = fielddef[TYPE] 2706 retrieve = self._meta.fieldtypes[field_type]['Retrieve'] 2707 datum = retrieve(record_data, fielddef, self._meta.memo, self._meta.decoder) 2708 return datum
2709
2710 - def _rollback_flux(self):
2711 """ 2712 discards all changes since ._start_flux() 2713 """ 2714 if self._write_to_disk: 2715 raise DbfError("record not in flux") 2716 self._data = self._old_data 2717 self._old_data = None 2718 self._memos.clear() 2719 self._write_to_disk = True 2720 self._write()
2721
2722 - def _start_flux(self):
2723 """ 2724 Allows record.field_name = ... and record[...] = ...; must use ._commit_flux() to commit changes 2725 """ 2726 if self._meta.status == CLOSED: 2727 raise DbfError("%s is closed; cannot modify record" % self._meta.filename) 2728 elif self._recnum < 0: 2729 raise DbfError("record has been packed; unable to update") 2730 elif not self._write_to_disk: 2731 raise DbfError("record already in a state of flux") 2732 self._old_data = self._data[:] 2733 self._write_to_disk = False
2734
2735 - def _update_field_value(self, index, name, value):
2736 """ 2737 calls appropriate routine to convert value to ascii bytes, and save it in record 2738 """ 2739 fielddef = self._meta[name] 2740 field_type = fielddef[TYPE] 2741 flags = fielddef[FLAGS] 2742 binary = flags & BINARY 2743 nullable = flags & NULLABLE and '_nullflags' in self._meta 2744 update = self._meta.fieldtypes[field_type]['Update'] 2745 if nullable: 2746 byte, bit = divmod(index, 8) 2747 null_def = self._meta['_nullflags'] 2748 null_data = self._data[null_def[START]:null_def[END]].tostring() 2749 null_data = [ord(c) for c in null_data] 2750 if value is Null: 2751 null_data[byte] |= 1 << bit 2752 value = None 2753 else: 2754 null_data[byte] &= 0xff ^ 1 << bit 2755 null_data = array('c', [chr(n) for n in null_data]) 2756 self._data[null_def[START]:null_def[END]] = null_data 2757 if value is not Null: 2758 bytes = array('c', update(value, fielddef, self._meta.memo, self._meta.input_decoder, self._meta.encoder)) 2759 size = fielddef[LENGTH] 2760 if len(bytes) > size: 2761 raise DataOverflowError("tried to store %d bytes in %d byte field" % (len(bytes), size)) 2762 blank = array('c', ' ' * size) 2763 start = fielddef[START] 2764 end = start + size 2765 blank[:len(bytes)] = bytes[:] 2766 self._data[start:end] = blank[:] 2767 self._dirty = True
2768
2769 - def _update_disk(self, location='', data=None):
2770 layout = self._meta 2771 if self._recnum < 0: 2772 raise DbfError("cannot update a packed record") 2773 if layout.location == ON_DISK: 2774 header = layout.header 2775 if location == '': 2776 location = self._recnum * header.record_length + header.start 2777 if data is None: 2778 data = self._data 2779 layout.dfd.seek(location) 2780 layout.dfd.write(data) 2781 self._dirty = False 2782 table = layout.table() 2783 if table is not None: # is None when table is being destroyed 2784 for index in table._indexen: 2785 index(self)
2786
2787 - def _write(self):
2788 for field, value in self._memos.items(): 2789 index = self._meta.fields.index(field) 2790 self._update_field_value(index, field, value) 2791 self._update_disk()
2792
2793 2794 -class RecordTemplate(object):
2795 """ 2796 Provides routines to mimic a dbf record. 2797 """ 2798 2799 __slots__ = ('_meta', '_data', '_old_data', '_memos', '_write_to_disk', '__weakref__') 2800
2801 - def _commit_flux(self):
2802 """ 2803 Flushes field updates to disk 2804 If any errors restores previous contents and raises `DbfError` 2805 """ 2806 if self._write_to_disk: 2807 raise DbfError("record not in flux") 2808 self._memos.clear() 2809 self._old_data = None 2810 self._write_to_disk = True
2811
2812 - def _retrieve_field_value(self, index, name):
2813 """ 2814 Calls appropriate routine to convert value stored in field from 2815 array 2816 """ 2817 fielddef = self._meta[name] 2818 flags = fielddef[FLAGS] 2819 nullable = flags & NULLABLE and '_nullflags' in self._meta 2820 binary = flags & BINARY 2821 if nullable: 2822 byte, bit = divmod(index, 8) 2823 null_def = self._meta['_nullflags'] 2824 null_data = self._data[null_def[START]:null_def[END]] 2825 if ord(null_data[byte]) >> bit & 1: 2826 return Null 2827 record_data = self._data[fielddef[START]:fielddef[END]] 2828 field_type = fielddef[TYPE] 2829 retrieve = self._meta.fieldtypes[field_type]['Retrieve'] 2830 datum = retrieve(record_data, fielddef, self._meta.memo, self._meta.decoder) 2831 return datum
2832
2833 - def _rollback_flux(self):
2834 """ 2835 discards all changes since ._start_flux() 2836 """ 2837 if self._write_to_disk: 2838 raise DbfError("template not in flux") 2839 self._data = self._old_data 2840 self._old_data = None 2841 self._memos.clear() 2842 self._write_to_disk = True
2843
2844 - def _start_flux(self):
2845 """ 2846 Allows record.field_name = ... and record[...] = ...; must use ._commit_flux() to commit changes 2847 """ 2848 if not self._write_to_disk: 2849 raise DbfError("template already in a state of flux") 2850 self._old_data = self._data[:] 2851 self._write_to_disk = False
2852
2853 - def _update_field_value(self, index, name, value):
2854 """ 2855 calls appropriate routine to convert value to ascii bytes, and save it in record 2856 """ 2857 fielddef = self._meta[name] 2858 field_type = fielddef[TYPE] 2859 flags = fielddef[FLAGS] 2860 binary = flags & BINARY 2861 nullable = flags & NULLABLE and '_nullflags' in self._meta 2862 update = self._meta.fieldtypes[field_type]['Update'] 2863 if nullable: 2864 byte, bit = divmod(index, 8) 2865 null_def = self._meta['_nullflags'] 2866 null_data = self._data[null_def[START]:null_def[END]].tostring() 2867 null_data = [ord(c) for c in null_data] 2868 if value is Null: 2869 null_data[byte] |= 1 << bit 2870 value = None 2871 else: 2872 null_data[byte] &= 0xff ^ 1 << bit 2873 null_data = array('c', [chr(n) for n in null_data]) 2874 self._data[null_def[START]:null_def[END]] = null_data 2875 if value is not Null: 2876 bytes = array('c', update(value, fielddef, self._meta.memo, self._meta.input_decoder, self._meta.encoder)) 2877 size = fielddef[LENGTH] 2878 if len(bytes) > size: 2879 raise DataOverflowError("tried to store %d bytes in %d byte field" % (len(bytes), size)) 2880 blank = array('c', ' ' * size) 2881 start = fielddef[START] 2882 end = start + size 2883 blank[:len(bytes)] = bytes[:] 2884 self._data[start:end] = blank[:]
2885
2886 - def __new__(cls, layout, original_record=None, defaults=None):
2887 """ 2888 record = ascii array of entire record; layout=record specification 2889 """ 2890 sig = layout.record_sig 2891 if sig not in _Template_Records: 2892 table = layout.table() 2893 _Template_Records[sig] = table.new( 2894 ':%s:' % layout.filename, 2895 default_data_types=table._meta._default_data_types, 2896 field_data_types=table._meta._field_data_types, on_disk=False 2897 )._meta 2898 layout = _Template_Records[sig] 2899 record = object.__new__(cls) 2900 record._write_to_disk = True 2901 record._meta = layout 2902 record._memos = {} 2903 for name in layout.memofields: 2904 field_type = layout[name][TYPE] 2905 record._memos[name] = layout.fieldtypes[field_type]['Empty']() 2906 header = layout.header 2907 if original_record is None: 2908 record._data = layout.blankrecord[:] 2909 else: 2910 record._data = original_record._data[:] 2911 for name in layout.memofields: 2912 record._memos[name] = original_record[name] 2913 for field in field_names(defaults or {}): 2914 record[field] = defaults[field] 2915 record._old_data = record._data[:] 2916 return record
2917
2918 - def __contains__(self, key):
2919 return key in self._meta.user_fields
2920
2921 - def __eq__(self, other):
2922 if not isinstance(other, (Record, RecordTemplate, dict, tuple)): 2923 return NotImplemented 2924 if isinstance(other, (Record, RecordTemplate)): 2925 if field_names(self) != field_names(other): 2926 return False 2927 for field in self._meta.user_fields: 2928 s_value, o_value = self[field], other[field] 2929 if s_value is not o_value and s_value != o_value: 2930 return False 2931 elif isinstance(other, dict): 2932 if sorted(field_names(self)) != sorted(other.keys()): 2933 return False 2934 for field in self._meta.user_fields: 2935 s_value, o_value = self[field], other[field] 2936 if s_value is not o_value and s_value != o_value: 2937 return False 2938 else: # tuple 2939 if len(self) != len(other): 2940 return False 2941 for s_value, o_value in zip(self, other): 2942 if s_value is not o_value and s_value != o_value: 2943 return False 2944 return True
2945
2946 - def __iter__(self):
2947 return (self[field] for field in self._meta.user_fields)
2948
2949 - def __getattr__(self, name):
2950 if name[0:2] == '__' and name[-2:] == '__': 2951 raise AttributeError('Method %s is not implemented.' % name) 2952 if not name in self._meta.fields: 2953 raise FieldMissingError(name) 2954 if name in self._memos: 2955 return self._memos[name] 2956 try: 2957 index = self._meta.fields.index(name) 2958 value = self._retrieve_field_value(index, name) 2959 return value 2960 except DbfError: 2961 error = sys.exc_info()[1] 2962 error.message = "field --%s-- is %s -> %s" % (name, self._meta.fieldtypes[fielddef['type']]['Type'], error.message) 2963 raise
2964
2965 - def __getitem__(self, item):
2966 fields = self._meta.user_fields 2967 if isinstance(item, baseinteger): 2968 field_count = len(fields) 2969 if not -field_count <= item < field_count: 2970 raise NotFoundError("Field offset %d is not in record" % item) 2971 field = fields[item] 2972 if field in self._memos: 2973 return self._memos[field] 2974 return self[field] 2975 elif isinstance(item, slice): 2976 sequence = [] 2977 if isinstance(item.start, basestring) or isinstance(item.stop, basestring): 2978 start, stop, step = item.start, item.stop, item.step 2979 if start not in fields or stop not in fields: 2980 raise MissingFieldError("Either %r or %r (or both) are not valid field names" % (start, stop)) 2981 if step is not None and not isinstance(step, baseinteger): 2982 raise DbfError("step value must be an int or long, not %r" % type(step)) 2983 start = fields.index(start) 2984 stop = fields.index(stop) + 1 2985 item = slice(start, stop, step) 2986 for index in self._meta.fields[item]: 2987 sequence.append(self[index]) 2988 return sequence 2989 elif isinstance(item, basestring): 2990 return self.__getattr__(item) 2991 else: 2992 raise TypeError("%r is not a field name" % item)
2993
2994 - def __len__(self):
2995 return self._meta.user_field_count
2996
2997 - def __ne__(self, other):
2998 if not isinstance(other, (Record, RecordTemplate, dict, tuple)): 2999 return NotImplemented 3000 return not self == other
3001
3002 - def __setattr__(self, name, value):
3003 if name in self.__slots__: 3004 object.__setattr__(self, name, value) 3005 return 3006 if not name in self._meta.fields: 3007 raise FieldMissingError(name) 3008 if name in self._meta.memofields: 3009 self._memos[name] = value 3010 self._dirty = True 3011 return 3012 index = self._meta.fields.index(name) 3013 try: 3014 self._update_field_value(index, name, value) 3015 except DbfError: 3016 error = sys.exc_info()[1] 3017 fielddef = self._meta[name] 3018 message = "%s (%s) = %r --> %s" % (name, self._meta.fieldtypes[fielddef[TYPE]]['Type'], value, error.message) 3019 data = name 3020 err_cls = error.__class__ 3021 raise err_cls(message, data)
3022
3023 - def __setitem__(self, name, value):
3024 if isinstance(name, basestring): 3025 self.__setattr__(name, value) 3026 elif isinstance(name, baseinteger): 3027 self.__setattr__(self._meta.fields[name], value) 3028 elif isinstance(name, slice): 3029 sequence = [] 3030 field_names = dbf.field_names(self) 3031 if isinstance(name.start, basestring) or isinstance(name.stop, basestring): 3032 start, stop, step = name.start, name.stop, name.step 3033 if start not in field_names or stop not in field_names: 3034 raise MissingFieldError("Either %r or %r (or both) are not valid field names" % (start, stop)) 3035 if step is not None and not isinstance(step, baseinteger): 3036 raise DbfError("step value must be an int or long, not %r" % type(step)) 3037 start = field_names.index(start) 3038 stop = field_names.index(stop) + 1 3039 name = slice(start, stop, step) 3040 for field in self._meta.fields[name]: 3041 sequence.append(field) 3042 if len(sequence) != len(value): 3043 raise DbfError("length of slices not equal") 3044 for field, val in zip(sequence, value): 3045 self[field] = val 3046 else: 3047 raise TypeError("%s is not a field name" % name)
3048 3049
3050 - def __repr__(self):
3051 return self._data.tostring()
3052
3053 - def __str__(self):
3054 result = [] 3055 for seq, field in enumerate(field_names(self)): 3056 result.append("%3d - %-10s: %r" % (seq, field, self[field])) 3057 return '\n'.join(result)
3058
3059 3060 -class RecordVaporWare(object):
3061 """ 3062 Provides routines to mimic a dbf record, but all values are non-existent. 3063 """ 3064 3065 __slots__ = ('_recno', '_sequence') 3066
3067 - def __new__(cls, position, sequence):
3068 """ 3069 record = ascii array of entire record 3070 layout=record specification 3071 memo = memo object for table 3072 """ 3073 if position not in ('bof', 'eof'): 3074 raise ValueError("position should be 'bof' or 'eof', not %r" % position) 3075 vapor = object.__new__(cls) 3076 vapor._recno = (-1, None)[position == 'eof'] 3077 vapor._sequence = sequence 3078 return vapor
3079
3080 - def __contains__(self, key):
3081 return False
3082
3083 - def __eq__(self, other):
3084 if not isinstance(other, (Record, RecordTemplate, RecordVaporWare, dict, tuple)): 3085 return NotImplemented 3086 return False
3087 3088
3089 - def __getattr__(self, name):
3090 if name[0:2] == '__' and name[-2:] == '__': 3091 raise AttributeError('Method %s is not implemented.' % name) 3092 else: 3093 return Vapor
3094
3095 - def __getitem__(self, item):
3096 if isinstance(item, baseinteger): 3097 return Vapor 3098 elif isinstance(item, slice): 3099 raise TypeError('slice notation not allowed on Vapor records') 3100 elif isinstance(item, basestring): 3101 return self.__getattr__(item) 3102 else: 3103 raise TypeError("%r is not a field name" % item)
3104
3105 - def __len__(self):
3106 raise TypeError("Vapor records have no length")
3107
3108 - def __ne__(self, other):
3109 if not isinstance(other, (Record, RecordTemplate, RecordVaporWare, dict, tuple)): 3110 return NotImplemented 3111 return True
3112
3113 - def __nonzero__(self):
3114 """ 3115 Vapor records are always False 3116 """ 3117 return False
3118
3119 - def __setattr__(self, name, value):
3120 if name in self.__slots__: 3121 object.__setattr__(self, name, value) 3122 return 3123 raise TypeError("cannot change Vapor record")
3124
3125 - def __setitem__(self, name, value):
3126 if isinstance(name, (basestring, baseinteger)): 3127 raise TypeError("cannot change Vapor record") 3128 elif isinstance(name, slice): 3129 raise TypeError("slice notation not allowed on Vapor records") 3130 else: 3131 raise TypeError("%s is not a field name" % name)
3132
3133 - def __repr__(self):
3134 return "RecordVaporWare(position=%r, sequence=%r)" % (('bof', 'eof')[recno(self) is None], self._sequence)
3135
3136 - def __str__(self):
3137 return 'VaporRecord(%r)' % recno(self)
3138 3139 @property
3140 - def _recnum(self):
3141 if self._recno is None: 3142 return len(self._sequence) 3143 else: 3144 return self._recno
3145
3146 3147 -class _DbfMemo(object):
3148 """ 3149 Provides access to memo fields as dictionaries 3150 Must override _init, _get_memo, and _put_memo to 3151 store memo contents to disk 3152 """ 3153
3154 - def _init(self):
3155 """ 3156 Initialize disk file usage 3157 """
3158
3159 - def _get_memo(self, block):
3160 """ 3161 Retrieve memo contents from disk 3162 """
3163
3164 - def _put_memo(self, data):
3165 """ 3166 Store memo contents to disk 3167 """
3168
3169 - def _zap(self):
3170 """ 3171 Resets memo structure back to zero memos 3172 """ 3173 self.memory.clear() 3174 self.nextmemo = 1
3175
3176 - def __init__(self, meta):
3177 self.meta = meta 3178 self.memory = {} 3179 self.nextmemo = 1 3180 self._init() 3181 self.meta.newmemofile = False
3182
3183 - def get_memo(self, block):
3184 """ 3185 Gets the memo in block 3186 """ 3187 if self.meta.ignorememos or not block: 3188 return '' 3189 if self.meta.location == ON_DISK: 3190 return self._get_memo(block) 3191 else: 3192 return self.memory[block]
3193
3194 - def put_memo(self, data):
3195 """ 3196 Stores data in memo file, returns block number 3197 """ 3198 if self.meta.ignorememos or data == '': 3199 return 0 3200 if self.meta.location == IN_MEMORY: 3201 thismemo = self.nextmemo 3202 self.nextmemo += 1 3203 self.memory[thismemo] = data 3204 else: 3205 thismemo = self._put_memo(data) 3206 return thismemo
3207
3208 3209 -class _Db3Memo(_DbfMemo):
3210 """ 3211 dBase III specific 3212 """ 3213
3214 - def _init(self):
3215 self.meta.memo_size= 512 3216 self.record_header_length = 2 3217 if self.meta.location == ON_DISK and not self.meta.ignorememos: 3218 if self.meta.newmemofile: 3219 self.meta.mfd = open(self.meta.memoname, 'w+b') 3220 self.meta.mfd.write(pack_long_int(1) + '\x00' * 508) 3221 else: 3222 try: 3223 self.meta.mfd = open(self.meta.memoname, 'r+b') 3224 self.meta.mfd.seek(0) 3225 next = self.meta.mfd.read(4) 3226 self.nextmemo = unpack_long_int(next) 3227 except Exception: 3228 exc = sys.exc_info()[1] 3229 raise DbfError("memo file appears to be corrupt: %r" % exc.args)
3230
3231 - def _get_memo(self, block):
3232 block = int(block) 3233 self.meta.mfd.seek(block * self.meta.memo_size) 3234 eom = -1 3235 data = '' 3236 while eom == -1: 3237 newdata = self.meta.mfd.read(self.meta.memo_size) 3238 if not newdata: 3239 return data 3240 data += newdata 3241 eom = data.find('\x1a\x1a') 3242 return data[:eom]
3243
3244 - def _put_memo(self, data):
3245 data = data 3246 length = len(data) + self.record_header_length # room for two ^Z at end of memo 3247 blocks = length // self.meta.memo_size 3248 if length % self.meta.memo_size: 3249 blocks += 1 3250 thismemo = self.nextmemo 3251 self.nextmemo = thismemo + blocks 3252 self.meta.mfd.seek(0) 3253 self.meta.mfd.write(pack_long_int(self.nextmemo)) 3254 self.meta.mfd.seek(thismemo * self.meta.memo_size) 3255 self.meta.mfd.write(data) 3256 self.meta.mfd.write('\x1a\x1a') 3257 double_check = self._get_memo(thismemo) 3258 if len(double_check) != len(data): 3259 uhoh = open('dbf_memo_dump.err', 'wb') 3260 uhoh.write('thismemo: %d' % thismemo) 3261 uhoh.write('nextmemo: %d' % self.nextmemo) 3262 uhoh.write('saved: %d bytes' % len(data)) 3263 uhoh.write(data) 3264 uhoh.write('retrieved: %d bytes' % len(double_check)) 3265 uhoh.write(double_check) 3266 uhoh.close() 3267 raise DbfError("unknown error: memo not saved") 3268 return thismemo
3269
3270 - def _zap(self):
3271 if self.meta.location == ON_DISK and not self.meta.ignorememos: 3272 mfd = self.meta.mfd 3273 mfd.seek(0) 3274 mfd.truncate(0) 3275 mfd.write(pack_long_int(1) + '\x00' * 508) 3276 mfd.flush()
3277
3278 -class _VfpMemo(_DbfMemo):
3279 """ 3280 Visual Foxpro 6 specific 3281 """ 3282
3283 - def _init(self):
3284 if self.meta.location == ON_DISK and not self.meta.ignorememos: 3285 self.record_header_length = 8 3286 if self.meta.newmemofile: 3287 if self.meta.memo_size == 0: 3288 self.meta.memo_size = 1 3289 elif 1 < self.meta.memo_size < 33: 3290 self.meta.memo_size *= 512 3291 self.meta.mfd = open(self.meta.memoname, 'w+b') 3292 nextmemo = 512 // self.meta.memo_size 3293 if nextmemo * self.meta.memo_size < 512: 3294 nextmemo += 1 3295 self.nextmemo = nextmemo 3296 self.meta.mfd.write(pack_long_int(nextmemo, bigendian=True) + '\x00\x00' + \ 3297 pack_short_int(self.meta.memo_size, bigendian=True) + '\x00' * 504) 3298 else: 3299 try: 3300 self.meta.mfd = open(self.meta.memoname, 'r+b') 3301 self.meta.mfd.seek(0) 3302 header = self.meta.mfd.read(512) 3303 self.nextmemo = unpack_long_int(header[:4], bigendian=True) 3304 self.meta.memo_size = unpack_short_int(header[6:8], bigendian=True) 3305 except Exception: 3306 exc = sys.exc_info()[1] 3307 raise DbfError("memo file appears to be corrupt: %r" % exc.args)
3308
3309 - def _get_memo(self, block):
3310 self.meta.mfd.seek(block * self.meta.memo_size) 3311 header = self.meta.mfd.read(8) 3312 length = unpack_long_int(header[4:], bigendian=True) 3313 return self.meta.mfd.read(length)
3314
3315 - def _put_memo(self, data):
3316 data = data 3317 self.meta.mfd.seek(0) 3318 thismemo = unpack_long_int(self.meta.mfd.read(4), bigendian=True) 3319 self.meta.mfd.seek(0) 3320 length = len(data) + self.record_header_length 3321 blocks = length // self.meta.memo_size 3322 if length % self.meta.memo_size: 3323 blocks += 1 3324 self.meta.mfd.write(pack_long_int(thismemo + blocks, bigendian=True)) 3325 self.meta.mfd.seek(thismemo * self.meta.memo_size) 3326 self.meta.mfd.write('\x00\x00\x00\x01' + pack_long_int(len(data), bigendian=True) + data) 3327 return thismemo
3328
3329 - def _zap(self):
3330 if self.meta.location == ON_DISK and not self.meta.ignorememos: 3331 mfd = self.meta.mfd 3332 mfd.seek(0) 3333 mfd.truncate(0) 3334 nextmemo = 512 // self.meta.memo_size 3335 if nextmemo * self.meta.memo_size < 512: 3336 nextmemo += 1 3337 self.nextmemo = nextmemo 3338 mfd.write(pack_long_int(nextmemo, bigendian=True) + '\x00\x00' + \ 3339 pack_short_int(self.meta.memo_size, bigendian=True) + '\x00' * 504) 3340 mfd.flush()
3341
3342 3343 -class DbfCsv(csv.Dialect):
3344 """ 3345 csv format for exporting tables 3346 """ 3347 delimiter = ',' 3348 doublequote = True 3349 escapechar = None 3350 lineterminator = '\n' 3351 quotechar = '"' 3352 skipinitialspace = True 3353 quoting = csv.QUOTE_NONNUMERIC
3354 csv.register_dialect('dbf', DbfCsv)
3355 3356 3357 -class _DeadObject(object):
3358 """ 3359 used because you cannot weakref None 3360 """ 3361
3362 - def __nonzero__(self):
3363 return False
3364 3365 _DeadObject = _DeadObject() 3366 3367 3368 # Routines for saving, retrieving, and creating fields 3369 3370 VFPTIME = 1721425
3371 3372 -def pack_short_int(value, bigendian=False):
3373 """ 3374 Returns a two-bye integer from the value, or raises DbfError 3375 """ 3376 # 256 / 65,536 3377 if value > 65535: 3378 raise DataOverflowError("Maximum Integer size exceeded. Possible: 65535. Attempted: %d" % value) 3379 if bigendian: 3380 return struct.pack('>H', value) 3381 else: 3382 return struct.pack('<H', value)
3383
3384 -def pack_long_int(value, bigendian=False):
3385 """ 3386 Returns a four-bye integer from the value, or raises DbfError 3387 """ 3388 # 256 / 65,536 / 16,777,216 3389 if value > 4294967295: 3390 raise DataOverflowError("Maximum Integer size exceeded. Possible: 4294967295. Attempted: %d" % value) 3391 if bigendian: 3392 return struct.pack('>L', value) 3393 else: 3394 return struct.pack('<L', value)
3395
3396 -def pack_str(string):
3397 """ 3398 Returns an 11 byte, upper-cased, null padded string suitable for field names; 3399 raises DbfError if the string is bigger than 10 bytes 3400 """ 3401 if len(string) > 10: 3402 raise DbfError("Maximum string size is ten characters -- %s has %d characters" % (string, len(string))) 3403 return struct.pack('11s', string.upper())
3404
3405 -def unpack_short_int(bytes, bigendian=False):
3406 """ 3407 Returns the value in the two-byte integer passed in 3408 """ 3409 if bigendian: 3410 return struct.unpack('>H', bytes)[0] 3411 else: 3412 return struct.unpack('<H', bytes)[0]
3413
3414 -def unpack_long_int(bytes, bigendian=False):
3415 """ 3416 Returns the value in the four-byte integer passed in 3417 """ 3418 if bigendian: 3419 return int(struct.unpack('>L', bytes)[0]) 3420 else: 3421 return int(struct.unpack('<L', bytes)[0])
3422
3423 -def unpack_str(chars):
3424 """ 3425 Returns a normal, lower-cased string from a null-padded byte string 3426 """ 3427 field = struct.unpack('%ds' % len(chars), chars)[0] 3428 name = [] 3429 for ch in field: 3430 if ch == '\x00': 3431 break 3432 name.append(ch.lower()) 3433 return ''.join(name)
3434
3435 -def scinot(value, decimals):
3436 """ 3437 return scientific notation with not more than decimals-1 decimal places 3438 """ 3439 value = str(value) 3440 sign = '' 3441 if value[0] in ('+-'): 3442 sign = value[0] 3443 if sign == '+': 3444 sign = '' 3445 value = value[1:] 3446 if 'e' in value: #7.8e-05 3447 e = value.find('e') 3448 if e - 1 <= decimals: 3449 return sign + value 3450 integer, mantissa, power = value[0], value[1:e], value[e+1:] 3451 mantissa = mantissa[:decimals] 3452 value = sign + integer + mantissa + 'e' + power 3453 return value 3454 integer, mantissa = value[0], value[1:] 3455 if integer == '0': 3456 for e, integer in enumerate(mantissa): 3457 if integer not in ('.0'): 3458 break 3459 mantissa = '.' + mantissa[e+1:] 3460 mantissa = mantissa[:decimals] 3461 value = sign + integer + mantissa + 'e-%03d' % e 3462 return value 3463 e = mantissa.find('.') 3464 mantissa = '.' + mantissa.replace('.','') 3465 mantissa = mantissa[:decimals] 3466 value = sign + integer + mantissa + 'e+%03d' % e 3467 return value
3468
3469 -def unsupported_type(something, *ignore):
3470 """ 3471 called if a data type is not supported for that style of table 3472 """ 3473 return something
3474
3475 -def retrieve_character(bytes, fielddef, memo, decoder):
3476 """ 3477 Returns the string in bytes as fielddef[CLASS] or fielddef[EMPTY] 3478 """ 3479 data = bytes.tostring() 3480 if not data.strip(): 3481 cls = fielddef[EMPTY] 3482 if cls is NoneType: 3483 return None 3484 return cls(data) 3485 if fielddef[FLAGS] & BINARY: 3486 return data 3487 return fielddef[CLASS](decoder(data)[0])
3488
3489 -def update_character(string, fielddef, memo, decoder, encoder):
3490 """ 3491 returns the string as bytes (not unicode) as fielddef[CLASS] or fielddef[EMPTY] 3492 """ 3493 length = fielddef[LENGTH] 3494 if string == None: 3495 return length * ' ' 3496 if fielddef[FLAGS] & BINARY: 3497 if not isinstance(string, str): 3498 raise ValueError('binary field: %r not in bytes format' % string) 3499 string = str(string) 3500 return string 3501 else: 3502 if not isinstance(string, unicode): 3503 if not isinstance(string, str): 3504 raise ValueError("unable to coerce %r(%r) to string" % (type(string), string)) 3505 string = decoder(string)[0] 3506 string = encoder(string)[0] 3507 if not string[length:].strip(): # trims trailing white space to fit in field 3508 string = string[:length] 3509 return string
3510
3511 -def retrieve_currency(bytes, fielddef, *ignore):
3512 """ 3513 Returns the currency value in bytes 3514 """ 3515 value = struct.unpack('<q', bytes)[0] 3516 return fielddef[CLASS](("%de-4" % value).strip())
3517
3518 -def update_currency(value, *ignore):
3519 """ 3520 Returns the value to be stored in the record's disk data 3521 """ 3522 if value == None: 3523 value = 0 3524 currency = int(value * 10000) 3525 if not -9223372036854775808 < currency < 9223372036854775808: 3526 raise DataOverflowError("value %s is out of bounds" % value) 3527 return struct.pack('<q', currency)
3528
3529 -def retrieve_date(bytes, fielddef, *ignore):
3530 """ 3531 Returns the ascii coded date as fielddef[CLASS] or fielddef[EMPTY] 3532 """ 3533 text = bytes.tostring() 3534 if text == ' ': 3535 cls = fielddef[EMPTY] 3536 if cls is NoneType: 3537 return None 3538 return cls() 3539 year = int(text[0:4]) 3540 month = int(text[4:6]) 3541 day = int(text[6:8]) 3542 return fielddef[CLASS](year, month, day)
3543
3544 -def update_date(moment, *ignore):
3545 """ 3546 Returns the Date or datetime.date object ascii-encoded (yyyymmdd) 3547 """ 3548 if moment == None: 3549 return ' ' 3550 return "%04d%02d%02d" % moment.timetuple()[:3]
3551
3552 -def retrieve_double(bytes, fielddef, *ignore):
3553 """ 3554 Returns the double in bytes as fielddef[CLASS] ('default' == float) 3555 """ 3556 typ = fielddef[CLASS] 3557 if typ == 'default': 3558 typ = float 3559 return typ(struct.unpack('<d', bytes)[0])
3560
3561 -def update_double(value, *ignore):
3562 """ 3563 returns the value to be stored in the record's disk data 3564 """ 3565 if value == None: 3566 value = 0 3567 return struct.pack('<d', float(value))
3568
3569 -def retrieve_integer(bytes, fielddef, *ignore):
3570 """ 3571 Returns the binary number stored in bytes in little-endian 3572 format as fielddef[CLASS] 3573 """ 3574 typ = fielddef[CLASS] 3575 if typ == 'default': 3576 typ = int 3577 return typ(struct.unpack('<i', bytes)[0])
3578
3579 -def update_integer(value, *ignore):
3580 """ 3581 Returns value in little-endian binary format 3582 """ 3583 if value == None: 3584 value = 0 3585 try: 3586 value = int(value) 3587 except Exception: 3588 raise DbfError("incompatible type: %s(%s)" % (type(value), value)) 3589 if not -2147483648 < value < 2147483647: 3590 raise DataOverflowError("Integer size exceeded. Possible: -2,147,483,648..+2,147,483,647. Attempted: %d" % value) 3591 return struct.pack('<i', int(value))
3592
3593 -def retrieve_logical(bytes, fielddef, *ignore):
3594 """ 3595 Returns True if bytes is 't', 'T', 'y', or 'Y' 3596 None if '?' 3597 False otherwise 3598 """ 3599 cls = fielddef[CLASS] 3600 empty = fielddef[EMPTY] 3601 bytes = bytes.tostring() 3602 if bytes in 'tTyY': 3603 return cls(True) 3604 elif bytes in 'fFnN': 3605 return cls(False) 3606 elif bytes in '? ': 3607 if empty is NoneType: 3608 return None 3609 return empty() 3610 elif LOGICAL_BAD_IS_NONE: 3611 return None 3612 else: 3613 raise BadDataError('Logical field contained %r' % bytes) 3614 return typ(bytes)
3615
3616 -def update_logical(data, *ignore):
3617 """ 3618 Returns 'T' if logical is True, 'F' if False, '?' otherwise 3619 """ 3620 if data is Unknown or data is None or data is Null or data is Other: 3621 return '?' 3622 if data == True: 3623 return 'T' 3624 if data == False: 3625 return 'F' 3626 raise ValueError("unable to automatically coerce %r to Logical" % data)
3627
3628 -def retrieve_memo(bytes, fielddef, memo, decoder):
3629 """ 3630 Returns the block of data from a memo file 3631 """ 3632 stringval = bytes.tostring().strip() 3633 if not stringval or memo is None: 3634 cls = fielddef[EMPTY] 3635 if cls is NoneType: 3636 return None 3637 return cls() 3638 block = int(stringval) 3639 data = memo.get_memo(block) 3640 if fielddef[FLAGS] & BINARY: 3641 return data 3642 return fielddef[CLASS](decoder(data)[0])
3643
3644 -def update_memo(string, fielddef, memo, decoder, encoder):
3645 """ 3646 Writes string as a memo, returns the block number it was saved into 3647 """ 3648 if memo is None: 3649 raise DbfError('Memos are being ignored, unable to update') 3650 if string == None: 3651 string = '' 3652 if fielddef[FLAGS] & BINARY: 3653 if not isinstance(string, str): 3654 raise ValueError('binary field: %r not in bytes format' % string) 3655 string = str(string) 3656 else: 3657 if not isinstance(string, unicode): 3658 if not isinstance(string, str): 3659 raise ValueError("unable to coerce %r(%r) to string" % (type(string), string)) 3660 string = decoder(string)[0] 3661 string = encoder(string)[0] 3662 block = memo.put_memo(string) 3663 if block == 0: 3664 block = '' 3665 return "%*s" % (fielddef[LENGTH], block)
3666
3667 -def retrieve_numeric(bytes, fielddef, *ignore):
3668 """ 3669 Returns the number stored in bytes as integer if field spec for 3670 decimals is 0, float otherwise 3671 """ 3672 string = bytes.tostring().replace('\x00', '').strip() 3673 cls = fielddef[CLASS] 3674 if not string or string[0:1] == '*': # value too big to store (Visual FoxPro idiocy) 3675 cls = fielddef[EMPTY] 3676 if cls is NoneType: 3677 return None 3678 return cls() 3679 if cls == 'default': 3680 if fielddef[DECIMALS] == 0: 3681 return int(string) 3682 else: 3683 return float(string) 3684 else: 3685 return cls(string.strip())
3686
3687 -def update_numeric(value, fielddef, *ignore):
3688 """ 3689 returns value as ascii representation, rounding decimal 3690 portion as necessary 3691 """ 3692 if value == None: 3693 return fielddef[LENGTH] * ' ' 3694 try: 3695 value = float(value) 3696 except Exception: 3697 raise DbfError("incompatible type: %s(%s)" % (type(value), value)) 3698 decimalsize = fielddef[DECIMALS] 3699 totalsize = fielddef[LENGTH] 3700 if decimalsize: 3701 decimalsize += 1 3702 maxintegersize = totalsize - decimalsize 3703 integersize = len("%.0f" % floor(value)) 3704 if integersize > maxintegersize: 3705 if integersize != 1: 3706 raise DataOverflowError('Integer portion too big') 3707 string = scinot(value, decimalsize) 3708 if len(string) > totalsize: 3709 raise DataOverflowError('Value representation too long for field') 3710 return "%*.*f" % (fielddef[LENGTH], fielddef[DECIMALS], value)
3711
3712 -def retrieve_vfp_datetime(bytes, fielddef, *ignore):
3713 """ 3714 returns the date/time stored in bytes; dates <= 01/01/1981 00:00:00 3715 may not be accurate; BC dates are nulled. 3716 """ 3717 # two four-byte integers store the date and time. 3718 # millesecords are discarded from time 3719 if bytes == array('c', '\x00' * 8): 3720 cls = fielddef[EMPTY] 3721 if cls is NoneType: 3722 return None 3723 return cls() 3724 cls = fielddef[CLASS] 3725 time = unpack_long_int(bytes[4:]) 3726 microseconds = (time % 1000) * 1000 3727 time = time // 1000 # int(round(time, -3)) // 1000 discard milliseconds 3728 hours = time // 3600 3729 mins = time % 3600 // 60 3730 secs = time % 3600 % 60 3731 time = datetime.time(hours, mins, secs, microseconds) 3732 possible = unpack_long_int(bytes[:4]) 3733 possible -= VFPTIME 3734 possible = max(0, possible) 3735 date = datetime.date.fromordinal(possible) 3736 return cls(date.year, date.month, date.day, time.hour, time.minute, time.second, time.microsecond)
3737
3738 -def update_vfp_datetime(moment, *ignore):
3739 """ 3740 Sets the date/time stored in moment 3741 moment must have fields: 3742 year, month, day, hour, minute, second, microsecond 3743 """ 3744 bytes = ['\x00'] * 8 3745 if moment: 3746 hour = moment.hour 3747 minute = moment.minute 3748 second = moment.second 3749 millisecond = moment.microsecond // 1000 # convert from millionths to thousandths 3750 time = ((hour * 3600) + (minute * 60) + second) * 1000 + millisecond 3751 bytes[4:] = update_integer(time) 3752 bytes[:4] = update_integer(moment.toordinal() + VFPTIME) 3753 return ''.join(bytes)
3754
3755 -def retrieve_vfp_memo(bytes, fielddef, memo, decoder):
3756 """ 3757 Returns the block of data from a memo file 3758 """ 3759 if memo is None: 3760 block = 0 3761 else: 3762 block = struct.unpack('<i', bytes)[0] 3763 if not block: 3764 cls = fielddef[EMPTY] 3765 if cls is NoneType: 3766 return None 3767 return cls() 3768 data = memo.get_memo(block) 3769 if fielddef[FLAGS] & BINARY: 3770 return data 3771 return fielddef[CLASS](decoder(data)[0])
3772
3773 -def update_vfp_memo(string, fielddef, memo, decoder, encoder):
3774 """ 3775 Writes string as a memo, returns the block number it was saved into 3776 """ 3777 if memo is None: 3778 raise DbfError('Memos are being ignored, unable to update') 3779 if string == None: 3780 string = '' 3781 if fielddef[FLAGS] & BINARY: 3782 if not isinstance(string, str): 3783 raise ValueError('binary field: %r not in bytes format' % string) 3784 string = str(string) 3785 else: 3786 if not isinstance(string, unicode): 3787 if not isinstance(string, str): 3788 raise ValueError("unable to coerce %r(%r) to string" % (type(string), string)) 3789 string = decoder(string)[0] 3790 string = encoder(string)[0] 3791 block = memo.put_memo(string) 3792 return struct.pack('<i', block)
3793
3794 -def add_character(format, flags):
3795 if format[0][0] != '(' or format[0][-1] != ')' or any([f not in flags for f in format[1:]]): 3796 raise FieldSpecError("Format for Character field creation is 'C(n)%s', not 'C%s'" % field_spec_error_text(format, flags)) 3797 length = int(format[0][1:-1]) 3798 if not 0 < length < 256: 3799 raise FieldSpecError("Character fields must be between 1 and 255, not %d" % length) 3800 decimals = 0 3801 flag = 0 3802 for f in format[1:]: 3803 flag |= FIELD_FLAGS[f] 3804 return length, decimals, flag
3805
3806 -def add_date(format, flags):
3807 if any([f not in flags for f in format]): 3808 raise FieldSpecError("Format for Date field creation is 'D%s', not 'D%s'" % field_spec_error_text(format, flags)) 3809 length = 8 3810 decimals = 0 3811 flag = 0 3812 for f in format: 3813 flag |= FIELD_FLAGS[f] 3814 return length, decimals, flag
3815
3816 -def add_logical(format, flags):
3817 if any([f not in flags for f in format]): 3818 raise FieldSpecError("Format for Logical field creation is 'L%s', not 'L%s'" % field_spec_error_text(format, flags)) 3819 length = 1 3820 decimals = 0 3821 flag = 0 3822 for f in format: 3823 flag |= FIELD_FLAGS[f] 3824 return length, decimals, flag
3825
3826 -def add_memo(format, flags):
3827 if any(f not in flags for f in format): 3828 raise FieldSpecError("Format for Memo field creation is 'M(n)%s', not 'M%s'" % field_spec_error_text(format, flags)) 3829 length = 10 3830 decimals = 0 3831 flag = 0 3832 for f in format: 3833 flag |= FIELD_FLAGS[f] 3834 return length, decimals, flag
3835
3836 -def add_numeric(format, flags):
3837 if len(format) > 1 or format[0][0] != '(' or format[0][-1] != ')' or any(f not in flags for f in format[1:]): 3838 raise FieldSpecError("Format for Numeric field creation is 'N(s,d)%s', not 'N%s'" % field_spec_error_text(format, flags)) 3839 length, decimals = format[0][1:-1].split(',') 3840 length = int(length) 3841 decimals = int(decimals) 3842 flag = 0 3843 for f in format[1:]: 3844 flag |= FIELD_FLAGS[f] 3845 if not 0 < length < 20: 3846 raise FieldSpecError("Numeric fields must be between 1 and 19 digits, not %d" % length) 3847 if decimals and not 0 < decimals <= length - 2: 3848 raise FieldSpecError("Decimals must be between 0 and Length-2 (Length: %d, Decimals: %d)" % (length, decimals)) 3849 return length, decimals, flag
3850
3851 -def add_clp_character(format, flags):
3852 if format[0][0] != '(' or format[0][-1] != ')' or any([f not in flags for f in format[1:]]): 3853 raise FieldSpecError("Format for Character field creation is 'C(n)%s', not 'C%s'" % field_spec_error_text(format, flags)) 3854 length = int(format[0][1:-1]) 3855 if not 0 < length < 65519: 3856 raise FieldSpecError("Character fields must be between 1 and 65,519") 3857 decimals = 0 3858 flag = 0 3859 for f in format[1:]: 3860 flag |= FIELD_FLAGS[f] 3861 return length, decimals, flag
3862
3863 -def add_vfp_character(format, flags):
3864 if format[0][0] != '(' or format[0][-1] != ')' or any([f not in flags for f in format[1:]]): 3865 raise FieldSpecError("Format for Character field creation is 'C(n)%s', not 'C%s'" % field_spec_error_text(format, flags)) 3866 length = int(format[0][1:-1]) 3867 if not 0 < length < 255: 3868 raise FieldSpecError("Character fields must be between 1 and 255") 3869 decimals = 0 3870 flag = 0 3871 for f in format[1:]: 3872 flag |= FIELD_FLAGS[f] 3873 return length, decimals, flag
3874
3875 -def add_vfp_currency(format, flags):
3876 if any(f not in flags for f in format[1:]): 3877 raise FieldSpecError("Format for Currency field creation is 'Y%s', not 'Y%s'" % field_spec_error_text(format, flags)) 3878 length = 8 3879 decimals = 0 3880 flag = 0 3881 for f in format: 3882 flag |= FIELD_FLAGS[f] 3883 return length, decimals, flag
3884
3885 -def add_vfp_datetime(format, flags):
3886 if any(f not in flags for f in format[1:]): 3887 raise FieldSpecError("Format for DateTime field creation is 'T%s', not 'T%s'" % field_spec_error_text(format, flags)) 3888 length = 8 3889 decimals = 0 3890 flag = 0 3891 for f in format: 3892 flag |= FIELD_FLAGS[f] 3893 return length, decimals, flag
3894
3895 -def add_vfp_double(format, flags):
3896 if any(f not in flags for f in format[1:]): 3897 raise FieldSpecError("Format for Double field creation is 'B%s', not 'B%s'" % field_spec_error_text(format, flags)) 3898 length = 8 3899 decimals = 0 3900 flag = 0 3901 for f in format: 3902 flag |= FIELD_FLAGS[f] 3903 return length, decimals, flag
3904
3905 -def add_vfp_integer(format, flags):
3906 if any(f not in flags for f in format[1:]): 3907 raise FieldSpecError("Format for Integer field creation is 'I%s', not 'I%s'" % field_spec_error_text(format, flags)) 3908 length = 4 3909 decimals = 0 3910 flag = 0 3911 for f in format: 3912 flag |= FIELD_FLAGS[f] 3913 return length, decimals, flag
3914
3915 -def add_vfp_memo(format, flags):
3916 if any(f not in flags for f in format[1:]): 3917 raise FieldSpecError("Format for Memo field creation is 'M%s', not 'M%s'" % field_spec_error_text(format, flags)) 3918 length = 4 3919 decimals = 0 3920 flag = 0 3921 for f in format: 3922 flag |= FIELD_FLAGS[f] 3923 if 'binary' not in flags: # general or picture -- binary is implied 3924 flag |= FIELD_FLAGS['binary'] 3925 return length, decimals, flag
3926
3927 -def add_vfp_numeric(format, flags):
3928 if format[0][0] != '(' or format[0][-1] != ')' or any(f not in flags for f in format[1:]): 3929 raise FieldSpecError("Format for Numeric field creation is 'N(s,d)%s', not 'N%s'" % field_spec_error_text(format, flags)) 3930 length, decimals = format[0][1:-1].split(',') 3931 length = int(length) 3932 decimals = int(decimals) 3933 flag = 0 3934 for f in format[1:]: 3935 flag |= FIELD_FLAGS[f] 3936 if not 0 < length < 21: 3937 raise FieldSpecError("Numeric fields must be between 1 and 20 digits, not %d" % length) 3938 if decimals and not 0 < decimals <= length - 2: 3939 raise FieldSpecError("Decimals must be between 0 and Length-2 (Length: %d, Decimals: %d)" % (length, decimals)) 3940 return length, decimals, flag
3941
3942 -def field_spec_error_text(format, flags):
3943 """ 3944 generic routine for error text for the add...() functions 3945 """ 3946 flg = '' 3947 if flags: 3948 flg = ' [ ' + ' | '.join(flags) + ' ]' 3949 frmt = '' 3950 if format: 3951 frmt = ' ' + ' '.join(format) 3952 return flg, frmt
3953
3954 -def ezip(*iters):
3955 """ 3956 extends all iters to longest one, using last value from each as necessary 3957 """ 3958 iters = [iter(x) for x in iters] 3959 last = [None] * len(iters) 3960 while "any iters have items left": 3961 alive = len(iters) 3962 for i, iterator in enumerate(iters): 3963 try: 3964 value = next(iterator) 3965 last[i] = value 3966 except StopIteration: 3967 alive -= 1 3968 if alive: 3969 yield tuple(last) 3970 alive = len(iters) 3971 continue 3972 break
3973
3974 3975 # Public classes 3976 3977 -class Tables(object):
3978 """ 3979 context manager for multiple tables and/or indices 3980 """
3981 - def __init__(yo, *tables):
3982 if len(tables) == 1 and not isinstance(tables[0], (Table, basestring)): 3983 tables = tables[0] 3984 yo._tables = [] 3985 yo._entered = [] 3986 for table in tables: 3987 if isinstance(table, basestring): 3988 table = Table(table) 3989 yo._tables.append(table)
3990 - def __enter__(yo):
3991 for table in yo._tables: 3992 table.__enter__() 3993 yo._entered.append(table) 3994 return tuple(yo._tables)
3995 - def __exit__(yo, *args):
3996 while yo._entered: 3997 table = yo._entered.pop() 3998 try: 3999 table.__exit__() 4000 except Exception: 4001 pass
4002
4003 -class IndexLocation(long):
4004 """ 4005 Represents the index where the match criteria is if True, 4006 or would be if False 4007 4008 Used by Index.index_search 4009 """ 4010
4011 - def __new__(cls, value, found):
4012 "value is the number, found is True/False" 4013 result = long.__new__(cls, value) 4014 result.found = found 4015 return result
4016
4017 - def __nonzero__(self):
4018 return self.found
4019
4020 4021 -class FieldInfo(tuple):
4022 """ 4023 tuple with named attributes for representing a field's dbf type, 4024 length, decimal portion, and python class 4025 """ 4026 4027 __slots__= () 4028
4029 - def __new__(cls, *args):
4030 if len(args) != 4: 4031 raise TypeError("%s should be called with Type, Length, Decimal size, and Class" % cls.__name__) 4032 return tuple.__new__(cls, args)
4033 4034 @property
4035 - def field_type(self):
4036 return self[0]
4037 4038 @property
4039 - def length(self):
4040 return self[1]
4041 4042 @property
4043 - def decimal(self):
4044 return self[2]
4045 4046 @property
4047 - def py_type(self):
4048 return self[3]
4049
4050 4051 -class CodePage(tuple):
4052 """ 4053 tuple with named attributes for representing a tables codepage 4054 """ 4055 4056 __slots__= () 4057
4058 - def __new__(cls, name):
4059 "call with name of codepage (e.g. 'cp1252')" 4060 code, name, desc = _codepage_lookup(name) 4061 return tuple.__new__(cls, (name, desc, code))
4062
4063 - def __repr__(self):
4064 return "CodePage(%r, %r, %r)" % (self[0], self[1], self[2])
4065
4066 - def __str__(self):
4067 return "%s (%s)" % (self[0], self[1])
4068 4069 @property
4070 - def name(self):
4071 return self[0]
4072 4073 @property
4074 - def desc(self):
4075 return self[1]
4076 4077 @property
4078 - def code(self):
4079 return self[2]
4080
4081 4082 -class Iter(_Navigation):
4083 """ 4084 Provides iterable behavior for a table 4085 """ 4086
4087 - def __init__(self, table, include_vapor=False):
4088 """ 4089 Return a Vapor record as the last record in the iteration 4090 if include_vapor is True 4091 """ 4092 self._table = table 4093 self._record = None 4094 self._include_vapor = include_vapor 4095 self._exhausted = False
4096
4097 - def __iter__(self):
4098 return self
4099
4100 - def next(self):
4101 while not self._exhausted: 4102 if self._index == len(self._table): 4103 break 4104 if self._index >= (len(self._table) - 1): 4105 self._index = max(self._index, len(self._table)) 4106 if self._include_vapor: 4107 return RecordVaporWare('eof', self._table) 4108 break 4109 self._index += 1 4110 record = self._table[self._index] 4111 return record 4112 self._exhausted = True 4113 raise StopIteration
4114
4115 4116 -class Table(_Navigation):
4117 """ 4118 Base class for dbf style tables 4119 """ 4120 4121 _version = 'basic memory table' 4122 _versionabbr = 'dbf' 4123 _max_fields = 255 4124 _max_records = 4294967296 4125 4126 @MutableDefault
4127 - def _field_types():
4128 return { 4129 'C' : { 4130 'Type':'Character', 'Init':add_character, 'Blank':lambda x: ' ' * x, 'Retrieve':retrieve_character, 'Update':update_character, 4131 'Class':unicode, 'Empty':unicode, 'flags':tuple(), 4132 }, 4133 'D' : { 4134 'Type':'Date', 'Init':add_date, 'Blank':lambda x: ' ', 'Retrieve':retrieve_date, 'Update':update_date, 4135 'Class':datetime.date, 'Empty':none, 'flags':tuple(), 4136 }, 4137 'F' : { 4138 'Type':'Numeric', 'Retrieve':retrieve_numeric, 'Update':update_numeric, 'Blank':lambda x: ' ' * x, 'Init':add_numeric, 4139 'Class':'default', 'Empty':none, 'flags':tuple(), 4140 }, 4141 'L' : { 4142 'Type':'Logical', 'Init':add_logical, 'Blank':lambda x: '?', 'Retrieve':retrieve_logical, 'Update':update_logical, 4143 'Class':bool, 'Empty':none, 'flags':tuple(), 4144 }, 4145 'M' : { 4146 'Type':'Memo', 'Init':add_memo, 'Blank':lambda x: ' ', 'Retrieve':retrieve_memo, 'Update':update_memo, 4147 'Class':unicode, 'Empty':unicode, 'flags':tuple(), 4148 }, 4149 'N' : { 4150 'Type':'Numeric', 'Init':add_numeric, 'Blank':lambda x: ' ' * x, 'Retrieve':retrieve_numeric, 'Update':update_numeric, 4151 'Class':'default', 'Empty':none, 'flags':tuple(), 4152 }, 4153 }
4154 @MutableDefault
4155 - def _previous_status():
4156 return []
4157 _memoext = '' 4158 _memoClass = _DbfMemo 4159 _yesMemoMask = '' 4160 _noMemoMask = '' 4161 _binary_types = tuple() # as in non-unicode character, or non-text number 4162 _character_types = ('C', 'D', 'F', 'L', 'M', 'N') # field represented by text data 4163 _currency_types = tuple() # money! 4164 _date_types = ('D', ) # dates 4165 _datetime_types = tuple() # dates w/times 4166 _decimal_types = ('N', 'F') # text-based numeric fields 4167 _fixed_types = ('M', 'D', 'L') # always same length in table 4168 _logical_types = ('L', ) # logicals 4169 _memo_types = tuple('M', ) 4170 _numeric_types = ('N', 'F') # fields representing a number 4171 _variable_types = ('C', 'N', 'F') # variable length in table 4172 _dbfTableHeader = array('c', '\x00' * 32) 4173 _dbfTableHeader[0] = '\x00' # table type - none 4174 _dbfTableHeader[8:10] = array('c', pack_short_int(33)) 4175 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 4176 _dbfTableHeader[29] = '\x00' # code page -- none, using plain ascii 4177 _dbfTableHeader = _dbfTableHeader.tostring() 4178 _dbfTableHeaderExtra = '' 4179 _supported_tables = [] 4180 _pack_count = 0 4181 backup = None 4182
4183 - class _Indexen(object):
4184 """ 4185 implements the weakref structure for seperate indexes 4186 """ 4187
4188 - def __init__(self):
4189 self._indexen = set()
4190
4191 - def __iter__(self):
4192 self._indexen = set([s for s in self._indexen if s() is not None]) 4193 return (s() for s in self._indexen if s() is not None)
4194
4195 - def __len__(self):
4196 self._indexen = set([s for s in self._indexen if s() is not None]) 4197 return len(self._indexen)
4198
4199 - def add(self, new_index):
4200 self._indexen.add(weakref.ref(new_index)) 4201 self._indexen = set([s for s in self._indexen if s() is not None])
4202
4203 - class _MetaData(dict):
4204 """ 4205 Container class for storing per table metadata 4206 """ 4207 blankrecord = None 4208 dfd = None # file handle 4209 fields = None # field names 4210 field_count = 0 # number of fields 4211 field_types = None # dictionary of dbf type field specs 4212 filename = None # name of .dbf file 4213 ignorememos = False # True when memos should be ignored 4214 memoname = None # name of .dbt/.fpt file 4215 mfd = None # file handle 4216 memo = None # memo object 4217 memofields = None # field names of Memo type 4218 newmemofile = False # True when memo file needs to be created 4219 nulls = None # non-None when Nullable fields present 4220 user_fields = None # not counting SYSTEM fields 4221 user_field_count = 0 # also not counting SYSTEM fields
4222
4223 - class _TableHeader(object):
4224 """ 4225 represents the data block that defines a tables type and layout 4226 """ 4227
4228 - def __init__(self, data, pack_date, unpack_date):
4229 if len(data) != 32: 4230 raise BadDataError('table header should be 32 bytes, but is %d bytes' % len(data)) 4231 self.packDate = pack_date 4232 self.unpackDate = unpack_date 4233 self._data = array('c', data + '\x0d')
4234
4235 - def codepage(self, cp=None):
4236 """ 4237 get/set code page of table 4238 """ 4239 if cp is None: 4240 return self._data[29] 4241 else: 4242 cp, sd, ld = _codepage_lookup(cp) 4243 self._data[29] = cp 4244 return cp
4245 4246 @property
4247 - def data(self):
4248 """ 4249 main data structure 4250 """ 4251 date = self.packDate(Date.today()) 4252 self._data[1:4] = array('c', date) 4253 return self._data.tostring()
4254 4255 @data.setter
4256 - def data(self, bytes):
4257 if len(bytes) < 32: 4258 raise BadDataError("length for data of %d is less than 32" % len(bytes)) 4259 self._data[:] = array('c', bytes)
4260 4261 @property
4262 - def extra(self):
4263 "extra dbf info (located after headers, before data records)" 4264 fieldblock = self._data[32:] 4265 for i in range(len(fieldblock) // 32 + 1): 4266 cr = i * 32 4267 if fieldblock[cr] == '\x0d': 4268 break 4269 else: 4270 raise BadDataError("corrupt field structure") 4271 cr += 33 # skip past CR 4272 return self._data[cr:].tostring()
4273 4274 @extra.setter
4275 - def extra(self, data):
4276 fieldblock = self._data[32:] 4277 for i in range(len(fieldblock) // 32 + 1): 4278 cr = i * 32 4279 if fieldblock[cr] == '\x0d': 4280 break 4281 else: 4282 raise BadDataError("corrupt field structure") 4283 cr += 33 # skip past CR 4284 self._data[cr:] = array('c', data) # extra 4285 self._data[8:10] = array('c', pack_short_int(len(self._data))) # start
4286 4287 @property
4288 - def field_count(self):
4289 "number of fields (read-only)" 4290 fieldblock = self._data[32:] 4291 for i in range(len(fieldblock) // 32 + 1): 4292 cr = i * 32 4293 if fieldblock[cr] == '\x0d': 4294 break 4295 else: 4296 raise BadDataError("corrupt field structure") 4297 return len(fieldblock[:cr]) // 32
4298 4299 @property
4300 - def fields(self):
4301 """ 4302 field block structure 4303 """ 4304 fieldblock = self._data[32:] 4305 for i in range(len(fieldblock) // 32 + 1): 4306 cr = i * 32 4307 if fieldblock[cr] == '\x0d': 4308 break 4309 else: 4310 raise BadDataError("corrupt field structure") 4311 return fieldblock[:cr].tostring()
4312 4313 @fields.setter
4314 - def fields(self, block):
4315 fieldblock = self._data[32:] 4316 for i in range(len(fieldblock) // 32 + 1): 4317 cr = i * 32 4318 if fieldblock[cr] == '\x0d': 4319 break 4320 else: 4321 raise BadDataError("corrupt field structure") 4322 cr += 32 # convert to indexing main structure 4323 fieldlen = len(block) 4324 if fieldlen % 32 != 0: 4325 raise BadDataError("fields structure corrupt: %d is not a multiple of 32" % fieldlen) 4326 self._data[32:cr] = array('c', block) # fields 4327 self._data[8:10] = array('c', pack_short_int(len(self._data))) # start 4328 fieldlen = fieldlen // 32 4329 recordlen = 1 # deleted flag 4330 for i in range(fieldlen): 4331 recordlen += ord(block[i*32+16]) 4332 self._data[10:12] = array('c', pack_short_int(recordlen))
4333 4334 @property
4335 - def record_count(self):
4336 """ 4337 number of records (maximum 16,777,215) 4338 """ 4339 return unpack_long_int(self._data[4:8].tostring())
4340 4341 @record_count.setter
4342 - def record_count(self, count):
4343 self._data[4:8] = array('c', pack_long_int(count))
4344 4345 @property
4346 - def record_length(self):
4347 """ 4348 length of a record (read_only) (max of 65,535) 4349 """ 4350 return unpack_short_int(self._data[10:12].tostring())
4351 4352 @record_length.setter
4353 - def record_length(self, length):
4354 """ 4355 to support Clipper large Character fields 4356 """ 4357 self._data[10:12] = array('c', pack_short_int(length))
4358 4359 @property
4360 - def start(self):
4361 """ 4362 starting position of first record in file (must be within first 64K) 4363 """ 4364 return unpack_short_int(self._data[8:10].tostring())
4365 4366 @start.setter
4367 - def start(self, pos):
4368 self._data[8:10] = array('c', pack_short_int(pos))
4369 4370 @property
4371 - def update(self):
4372 """ 4373 date of last table modification (read-only) 4374 """ 4375 return self.unpackDate(self._data[1:4].tostring())
4376 4377 @property
4378 - def version(self):
4379 """ 4380 dbf version 4381 """ 4382 return self._data[0]
4383 4384 @version.setter
4385 - def version(self, ver):
4386 self._data[0] = ver
4387
4388 - class _Table(object):
4389 """ 4390 implements the weakref table for records 4391 """ 4392
4393 - def __init__(self, count, meta):
4394 self._meta = meta 4395 self._max_count = count 4396 self._weakref_list = {} 4397 self._accesses = 0 4398 self._dead_check = 1024
4399
4400 - def __getitem__(self, index):
4401 # maybe = self._weakref_list[index]() 4402 if index < 0: 4403 if self._max_count + index < 0: 4404 raise IndexError('index %d smaller than available records' % index) 4405 index = self._max_count + index 4406 if index >= self._max_count: 4407 raise IndexError('index %d greater than available records' % index) 4408 maybe = self._weakref_list.get(index) 4409 if maybe: 4410 maybe = maybe() 4411 self._accesses += 1 4412 if self._accesses >= self._dead_check: 4413 for key, value in self._weakref_list.items(): 4414 if value() is None: 4415 del self._weakref_list[key] 4416 if not maybe: 4417 meta = self._meta 4418 if meta.status == CLOSED: 4419 raise DbfError("%s is closed; record %d is unavailable" % (meta.filename, index)) 4420 header = meta.header 4421 if index < 0: 4422 index += header.record_count 4423 size = header.record_length 4424 location = index * size + header.start 4425 meta.dfd.seek(location) 4426 if meta.dfd.tell() != location: 4427 raise ValueError("unable to seek to offset %d in file" % location) 4428 bytes = meta.dfd.read(size) 4429 if not bytes: 4430 raise ValueError("unable to read record data from %s at location %d" % (meta.filename, location)) 4431 maybe = Record(recnum=index, layout=meta, kamikaze=bytes, _fromdisk=True) 4432 self._weakref_list[index] = weakref.ref(maybe) 4433 return maybe
4434
4435 - def append(self, record):
4436 self._weakref_list[self._max_count] = weakref.ref(record) 4437 self._max_count += 1
4438
4439 - def clear(self):
4440 for key in self._weakref_list.keys(): 4441 del self._weakref_list[key] 4442 self._max_count = 0
4443
4444 - def flush(self):
4445 for maybe in self._weakref_list.values(): 4446 maybe = maybe() 4447 if maybe and not maybe._write_to_disk: 4448 raise DbfError("some records have not been written to disk")
4449
4450 - def pop(self):
4451 if not self._max_count: 4452 raise IndexError('no records exist') 4453 self._max_count -= 1 4454 return self[self._max_count-1]
4455
4456 - def _build_header_fields(self):
4457 """ 4458 constructs fieldblock for disk table 4459 """ 4460 fieldblock = array('c', '') 4461 memo = False 4462 nulls = False 4463 meta = self._meta 4464 header = meta.header 4465 header.version = chr(ord(header.version) & ord(self._noMemoMask)) 4466 meta.fields = [f for f in meta.fields if f != '_nullflags'] 4467 for field in meta.fields: 4468 layout = meta[field] 4469 if meta.fields.count(field) > 1: 4470 raise BadDataError("corrupted field structure (noticed in _build_header_fields)") 4471 fielddef = array('c', '\x00' * 32) 4472 fielddef[:11] = array('c', pack_str(meta.encoder(field)[0])) 4473 fielddef[11] = layout[TYPE] 4474 fielddef[12:16] = array('c', pack_long_int(layout[START])) 4475 fielddef[16] = chr(layout[LENGTH]) 4476 fielddef[17] = chr(layout[DECIMALS]) 4477 fielddef[18] = chr(layout[FLAGS]) 4478 fieldblock.extend(fielddef) 4479 if layout[TYPE] in meta.memo_types: 4480 memo = True 4481 if layout[FLAGS] & NULLABLE: 4482 nulls = True 4483 if memo: 4484 header.version = chr(ord(header.version) | ord(self._yesMemoMask)) 4485 if meta.memo is None: 4486 meta.memo = self._memoClass(meta) 4487 else: 4488 if os.path.exists(meta.memoname): 4489 if meta.mfd is not None: 4490 meta.mfd.close() 4491 4492 os.remove(meta.memoname) 4493 meta.memo = None 4494 if nulls: 4495 start = layout[START] + layout[LENGTH] 4496 length, one_more = divmod(len(meta.fields), 8) 4497 if one_more: 4498 length += 1 4499 fielddef = array('c', '\x00' * 32) 4500 fielddef[:11] = array('c', pack_str('_nullflags')) 4501 fielddef[11] = '0' 4502 fielddef[12:16] = array('c', pack_long_int(start)) 4503 fielddef[16] = chr(length) 4504 fielddef[17] = chr(0) 4505 fielddef[18] = chr(BINARY | SYSTEM) 4506 fieldblock.extend(fielddef) 4507 meta.fields.append('_nullflags') 4508 nullflags = ( 4509 '0', # type 4510 start, # start 4511 length, # length 4512 start + length, # end 4513 0, # decimals 4514 BINARY | SYSTEM, # flags 4515 none, # class 4516 none, # empty 4517 ) 4518 meta['_nullflags'] = nullflags 4519 header.fields = fieldblock.tostring() 4520 meta.user_fields = [f for f in meta.fields if not meta[f][FLAGS] & SYSTEM] 4521 meta.user_field_count = len(meta.user_fields) 4522 Record._create_blank_data(meta)
4523
4524 - def _check_memo_integrity(self):
4525 """ 4526 checks memo file for problems 4527 """ 4528 raise NotImplementedError("_check_memo_integrity must be implemented by subclass")
4529
4530 - def _initialize_fields(self):
4531 """ 4532 builds the FieldList of names, types, and descriptions from the disk file 4533 """ 4534 raise NotImplementedError("_initialize_fields must be implemented by subclass")
4535
4536 - def _field_layout(self, i):
4537 """ 4538 Returns field information Name Type(Length[, Decimals]) 4539 """ 4540 name = self._meta.fields[i] 4541 fielddef = self._meta[name] 4542 type = fielddef[TYPE] 4543 length = fielddef[LENGTH] 4544 decimals = fielddef[DECIMALS] 4545 set_flags = fielddef[FLAGS] 4546 flags = [] 4547 if type in ('G', 'P'): 4548 printable_flags = NULLABLE, SYSTEM 4549 else: 4550 printable_flags = BINARY, NULLABLE, SYSTEM 4551 for flg in printable_flags: 4552 if flg & set_flags == flg: 4553 flags.append(FIELD_FLAGS[flg]) 4554 set_flags &= 255 ^ flg 4555 if flags: 4556 flags = ' ' + ' '.join(flags) 4557 else: 4558 flags = '' 4559 if type in self._fixed_types: 4560 description = "%s %s%s" % (name, type, flags) 4561 elif type in self._numeric_types: 4562 description = "%s %s(%d,%d)%s" % (name, type, length, decimals, flags) 4563 else: 4564 description = "%s %s(%d)%s" % (name, type, length, flags) 4565 return description
4566
4567 - def _list_fields(self, specs, sep=','):
4568 """ 4569 standardizes field specs 4570 """ 4571 if specs is None: 4572 specs = self.field_names 4573 elif isinstance(specs, str): 4574 specs = specs.strip(sep).split(sep) 4575 else: 4576 specs = list(specs) 4577 specs = [s.strip() for s in specs] 4578 return specs
4579
4580 - def _nav_check(self):
4581 """ 4582 Raises `DbfError` if table is closed 4583 """ 4584 if self._meta.status == CLOSED: 4585 raise DbfError('table %s is closed' % self.filename)
4586 4587 @staticmethod
4588 - def _pack_date(date):
4589 """ 4590 Returns a group of three bytes, in integer form, of the date 4591 """ 4592 return "%c%c%c" % (date.year - 1900, date.month, date.day)
4593 4594 @staticmethod
4595 - def _unpack_date(bytestr):
4596 """ 4597 Returns a Date() of the packed three-byte date passed in 4598 """ 4599 year, month, day = struct.unpack('<BBB', bytestr) 4600 year += 1900 4601 return Date(year, month, day)
4602
4603 - def _update_disk(self, headeronly=False):
4604 """ 4605 synchronizes the disk file with current data 4606 """ 4607 if self._meta.location == IN_MEMORY: 4608 return 4609 meta = self._meta 4610 header = meta.header 4611 fd = meta.dfd 4612 fd.seek(0) 4613 fd.write(header.data) 4614 eof = header.start + header.record_count * header.record_length 4615 if not headeronly: 4616 for record in self: 4617 record._update_disk() 4618 fd.flush() 4619 fd.truncate(eof) 4620 if self._versionabbr in ('db3', 'clp'): 4621 fd.seek(0, SEEK_END) 4622 fd.write('\x1a') # required for dBase III compatibility 4623 fd.flush() 4624 fd.truncate(eof + 1)
4625
4626 - def __contains__(self, data):
4627 """ 4628 data can be a record, template, dict, or tuple 4629 """ 4630 if not isinstance(data, (Record, RecordTemplate, dict, tuple)): 4631 raise TypeError("x should be a record, template, dict, or tuple, not %r" % type(data)) 4632 for record in Iter(self): 4633 if data == record: 4634 return True 4635 return False
4636
4637 - def __enter__(self):
4638 self._previous_status.append(self._meta.status) 4639 self.open() 4640 return self
4641
4642 - def __exit__(self, *exc_info):
4643 if self._previous_status.pop() == CLOSED: 4644 self.close()
4645
4646 - def __getattr__(self, name):
4647 if name in ( 4648 'binary_types', 4649 'character_types', 4650 'currency_types', 4651 'date_types', 4652 'datetime_types', 4653 'decimal_types', 4654 'fixed_types', 4655 'logical_types', 4656 'memo_types', 4657 'numeric_types', 4658 'variable_types', 4659 ): 4660 return getattr(self, '_'+name) 4661 if name in ('_table', ): 4662 if self._meta.location == ON_DISK: 4663 self._table = self._Table(len(self), self._meta) 4664 else: 4665 self._table = [] 4666 return object.__getattribute__(self, name)
4667
4668 - def __getitem__(self, value):
4669 if isinstance(value, baseinteger): 4670 if not -self._meta.header.record_count <= value < self._meta.header.record_count: 4671 raise NotFoundError("Record %d is not in table %s." % (value, self.filename)) 4672 return self._table[value] 4673 elif type(value) == slice: 4674 sequence = List(desc='%s --> %s' % (self.filename, value)) 4675 for index in range(len(self))[value]: 4676 record = self._table[index] 4677 sequence.append(record) 4678 return sequence 4679 else: 4680 raise TypeError('type <%s> not valid for indexing' % type(value))
4681
4682 - def __init__(self, filename, field_specs=None, memo_size=128, ignore_memos=False, 4683 codepage=None, default_data_types=None, field_data_types=None, # e.g. 'name':str, 'age':float 4684 dbf_type=None, on_disk=True, 4685 ):
4686 """ 4687 open/create dbf file 4688 filename should include path if needed 4689 field_specs can be either a ;-delimited string or a list of strings 4690 memo_size is always 512 for db3 memos 4691 ignore_memos is useful if the memo file is missing or corrupt 4692 read_only will load records into memory, then close the disk file 4693 keep_memos will also load any memo fields into memory 4694 meta_only will ignore all records, keeping only basic table information 4695 codepage will override whatever is set in the table itself 4696 """ 4697 4698 if not on_disk: 4699 if field_specs is None: 4700 raise DbfError("field list must be specified for memory tables") 4701 self._indexen = self._Indexen() 4702 self._meta = meta = self._MetaData() 4703 meta.max_fields = self._max_fields 4704 meta.max_records = self._max_records 4705 meta.table = weakref.ref(self) 4706 meta.filename = filename 4707 meta.fields = [] 4708 meta.user_fields = [] 4709 meta.user_field_count = 0 4710 meta.fieldtypes = fieldtypes = self._field_types 4711 meta.fixed_types = self._fixed_types 4712 meta.variable_types = self._variable_types 4713 meta.character_types = self._character_types 4714 meta.currency_types = self._currency_types 4715 meta.decimal_types = self._decimal_types 4716 meta.numeric_types = self._numeric_types 4717 meta.memo_types = self._memo_types 4718 meta.ignorememos = meta.original_ignorememos = ignore_memos 4719 meta.memo_size = memo_size 4720 meta.input_decoder = codecs.getdecoder(input_decoding) # from ascii to unicode 4721 meta.output_encoder = codecs.getencoder(input_decoding) # and back to ascii 4722 meta.header = header = self._TableHeader(self._dbfTableHeader, self._pack_date, self._unpack_date) 4723 header.extra = self._dbfTableHeaderExtra 4724 if default_data_types is None: 4725 default_data_types = dict() 4726 elif default_data_types == 'enhanced': 4727 default_data_types = { 4728 'C' : dbf.Char, 4729 'L' : dbf.Logical, 4730 'D' : dbf.Date, 4731 'T' : dbf.DateTime, 4732 } 4733 4734 self._meta._default_data_types = default_data_types 4735 if field_data_types is None: 4736 field_data_types = dict() 4737 self._meta._field_data_types = field_data_types 4738 for field, types in default_data_types.items(): 4739 if not isinstance(types, tuple): 4740 types = (types, ) 4741 for result_name, result_type in ezip(('Class', 'Empty', 'Null'), types): 4742 fieldtypes[field][result_name] = result_type 4743 if not on_disk: 4744 self._table = [] 4745 meta.location = IN_MEMORY 4746 meta.memoname = filename 4747 meta.header.data 4748 else: 4749 base, ext = os.path.splitext(filename) 4750 if ext.lower() != '.dbf': 4751 meta.filename = filename + '.dbf' 4752 searchname = filename + '.[Db][Bb][Ff]' 4753 else: 4754 meta.filename = filename 4755 searchname = filename 4756 matches = glob(searchname) 4757 if len(matches) == 1: 4758 meta.filename = matches[0] 4759 elif matches: 4760 raise DbfError("please specify exactly which of %r you want" % (matches, )) 4761 case = [('l','u')[c.isupper()] for c in meta.filename[-4:]] 4762 if case == ['l','l','l','l']: 4763 meta.memoname = base + self._memoext.lower() 4764 elif case == ['l','u','u','u']: 4765 meta.memoname = base + self._memoext.upper() 4766 else: 4767 meta.memoname = base + ''.join([c.lower() if case[i] == 'l' else c.upper() for i, c in enumerate(self._memoext)]) 4768 meta.location = ON_DISK 4769 if codepage is not None: 4770 header.codepage(codepage) 4771 cp, sd, ld = _codepage_lookup(codepage) 4772 self._meta.decoder = codecs.getdecoder(sd) 4773 self._meta.encoder = codecs.getencoder(sd) 4774 if field_specs: 4775 if meta.location == ON_DISK: 4776 meta.dfd = open(meta.filename, 'w+b') 4777 meta.newmemofile = True 4778 if codepage is None: 4779 header.codepage(default_codepage) 4780 cp, sd, ld = _codepage_lookup(header.codepage()) 4781 meta.decoder = codecs.getdecoder(sd) 4782 meta.encoder = codecs.getencoder(sd) 4783 meta.status = READ_WRITE 4784 self.add_fields(field_specs) 4785 else: 4786 try: 4787 dfd = meta.dfd = open(meta.filename, 'r+b') 4788 except IOError: 4789 e= sys.exc_info()[1] 4790 raise DbfError(str(e)) 4791 dfd.seek(0) 4792 meta.header = header = self._TableHeader(dfd.read(32), self._pack_date, self._unpack_date) 4793 if not header.version in self._supported_tables: 4794 dfd.close() 4795 dfd = None 4796 raise DbfError( 4797 "%s does not support %s [%x]" % 4798 (self._version, 4799 version_map.get(header.version, 'Unknown: %s' % header.version), 4800 ord(header.version))) 4801 if codepage is None: 4802 cp, sd, ld = _codepage_lookup(header.codepage()) 4803 self._meta.decoder = codecs.getdecoder(sd) 4804 self._meta.encoder = codecs.getencoder(sd) 4805 fieldblock = dfd.read(header.start - 32) 4806 for i in range(len(fieldblock) // 32 + 1): 4807 fieldend = i * 32 4808 if fieldblock[fieldend] == '\x0d': 4809 break 4810 else: 4811 raise BadDataError("corrupt field structure in header") 4812 if len(fieldblock[:fieldend]) % 32 != 0: 4813 raise BadDataError("corrupt field structure in header") 4814 old_length = header.data[10:12] 4815 header.fields = fieldblock[:fieldend] 4816 header.data = header.data[:10] + old_length + header.data[12:] # restore original for testing 4817 header.extra = fieldblock[fieldend + 1:] # skip trailing \r 4818 self._initialize_fields() 4819 self._check_memo_integrity() 4820 dfd.seek(0) 4821 4822 for field in meta.fields: 4823 field_type = meta[field][TYPE] 4824 default_field_type = ( 4825 fieldtypes[field_type]['Class'], 4826 fieldtypes[field_type]['Empty'], 4827 ) 4828 specific_field_type = field_data_types.get(field) 4829 if specific_field_type is not None and not isinstance(specific_field_type, tuple): 4830 specific_field_type = (specific_field_type, ) 4831 classes = [] 4832 for result_name, result_type in ezip( 4833 ('class', 'empty'), 4834 specific_field_type or default_field_type, 4835 ): 4836 classes.append(result_type) 4837 meta[field] = meta[field][:-2] + tuple(classes) 4838 meta.status = READ_ONLY 4839 self.close()
4840
4841 - def __iter__(self):
4842 """ 4843 iterates over the table's records 4844 """ 4845 return Iter(self)
4846
4847 - def __len__(self):
4848 """ 4849 returns number of records in table 4850 """ 4851 return self._meta.header.record_count
4852
4853 - def __new__(cls, filename, field_specs=None, memo_size=128, ignore_memos=False, 4854 codepage=None, default_data_types=None, field_data_types=None, # e.g. 'name':str, 'age':float 4855 dbf_type=None, on_disk=True, 4856 ):
4857 if dbf_type is None and isinstance(filename, Table): 4858 return filename 4859 if field_specs and dbf_type is None: 4860 dbf_type = default_type 4861 if dbf_type is not None: 4862 dbf_type = dbf_type.lower() 4863 table = table_types.get(dbf_type) 4864 if table is None: 4865 raise DbfError("Unknown table type: %s" % dbf_type) 4866 return object.__new__(table) 4867 else: 4868 base, ext = os.path.splitext(filename) 4869 if ext.lower() != '.dbf': 4870 filename = filename + '.dbf' 4871 possibles = guess_table_type(filename) 4872 if len(possibles) == 1: 4873 return object.__new__(possibles[0][2]) 4874 else: 4875 for type, desc, cls in possibles: 4876 if type == default_type: 4877 return object.__new__(cls) 4878 else: 4879 types = ', '.join(["%s" % item[1] for item in possibles]) 4880 abbrs = '[' + ' | '.join(["%s" % item[0] for item in possibles]) + ']' 4881 raise DbfError("Table could be any of %s. Please specify %s when opening" % (types, abbrs))
4882
4883 - def __nonzero__(self):
4884 """ 4885 True if table has any records 4886 """ 4887 return self._meta.header.record_count != 0
4888
4889 - def __repr__(self):
4890 return __name__ + ".Table(%r, status=%r)" % (self._meta.filename, self._meta.status)
4891
4892 - def __str__(self):
4893 status = self._meta.status 4894 version = version_map.get(self._meta.header.version) 4895 if version is not None: 4896 version = self._version 4897 else: 4898 version = 'unknown - ' + hex(ord(self._meta.header.version)) 4899 str = """ 4900 Table: %s 4901 Type: %s 4902 Codepage: %s 4903 Status: %s 4904 Last updated: %s 4905 Record count: %d 4906 Field count: %d 4907 Record length: %d """ % (self.filename, version 4908 , self.codepage, status, 4909 self.last_update, len(self), self.field_count, self.record_length) 4910 str += "\n --Fields--\n" 4911 for i in range(len(self.field_names)): 4912 str += "%11d) %s\n" % (i, self._field_layout(i)) 4913 return str
4914 4915 @property
4916 - def codepage(self):
4917 """ 4918 code page used for text translation 4919 """ 4920 return CodePage(code_pages[self._meta.header.codepage()][0])
4921 4922 @codepage.setter
4923 - def codepage(self, codepage):
4924 if not isinstance(codepage, CodePage): 4925 raise TypeError("codepage should be a CodePage, not a %r" % type(codepage)) 4926 meta = self._meta 4927 if meta.status != READ_WRITE: 4928 raise DbfError('%s not in read/write mode, unable to change codepage' % meta.filename) 4929 meta.header.codepage(codepage.code) 4930 meta.decoder = codecs.getdecoder(codepage.name) 4931 meta.encoder = codecs.getencoder(codepage.name) 4932 self._update_disk(headeronly=True)
4933 4934 @property
4935 - def field_count(self):
4936 """ 4937 the number of user fields in the table 4938 """ 4939 return self._meta.user_field_count
4940 4941 @property
4942 - def field_names(self):
4943 """ 4944 a list of the user fields in the table 4945 """ 4946 return self._meta.user_fields[:]
4947 4948 @property
4949 - def filename(self):
4950 """ 4951 table's file name, including path (if specified on open) 4952 """ 4953 return self._meta.filename
4954 4955 @property
4956 - def last_update(self):
4957 """ 4958 date of last update 4959 """ 4960 return self._meta.header.update
4961 4962 @property
4963 - def memoname(self):
4964 """ 4965 table's memo name (if path included in filename on open) 4966 """ 4967 return self._meta.memoname
4968 4969 @property
4970 - def record_length(self):
4971 """ 4972 number of bytes in a record (including deleted flag and null field size 4973 """ 4974 return self._meta.header.record_length
4975 4976 @property
4977 - def supported_tables(self):
4978 """ 4979 allowable table types 4980 """ 4981 return self._supported_tables
4982 4983 @property
4984 - def status(self):
4985 """ 4986 CLOSED, READ_ONLY, or READ_WRITE 4987 """ 4988 return self._meta.status
4989 4990 @property
4991 - def version(self):
4992 """ 4993 returns the dbf type of the table 4994 """ 4995 return self._version
4996
4997 - def add_fields(self, field_specs):
4998 """ 4999 adds field(s) to the table layout; format is Name Type(Length,Decimals)[; Name Type(Length,Decimals)[...]] 5000 backup table is created with _backup appended to name 5001 then zaps table, recreates current structure, and copies records back from the backup 5002 """ 5003 meta = self._meta 5004 if meta.status != READ_WRITE: 5005 raise DbfError('%s not in read/write mode, unable to add fields (%s)' % (meta.filename, meta.status)) 5006 header = meta.header 5007 fields = self.structure() + self._list_fields(field_specs, sep=';') 5008 if (len(fields) + ('_nullflags' in meta)) > meta.max_fields: 5009 raise DbfError( 5010 "Adding %d more field%s would exceed the limit of %d" 5011 % (len(fields), ('','s')[len(fields)==1], meta.max_fields) 5012 ) 5013 old_table = None 5014 if self: 5015 old_table = self.create_backup() 5016 self.zap() 5017 if meta.mfd is not None and not meta.ignorememos: 5018 meta.mfd.close() 5019 meta.mfd = None 5020 meta.memo = None 5021 if not meta.ignorememos: 5022 meta.newmemofile = True 5023 offset = 1 5024 for name in meta.fields: 5025 del meta[name] 5026 meta.fields[:] = [] 5027 5028 meta.blankrecord = None 5029 for field in fields: 5030 field = field.lower() 5031 pieces = field.split() 5032 name = pieces.pop(0) 5033 if '(' in pieces[0]: 5034 loc = pieces[0].index('(') 5035 pieces.insert(0, pieces[0][:loc]) 5036 pieces[1] = pieces[1][loc:] 5037 format = pieces.pop(0).upper() 5038 if pieces and '(' in pieces[0]: 5039 for i, p in enumerate(pieces): 5040 if ')' in p: 5041 pieces[0:i+1] = [''.join(pieces[0:i+1])] 5042 break 5043 if name[0] == '_' or name[0].isdigit() or not name.replace('_', '').isalnum(): 5044 raise FieldSpecError("%s invalid: field names must start with a letter, and can only contain letters, digits, and _" % name) 5045 name = unicode(name) 5046 if name in meta.fields: 5047 raise DbfError("Field '%s' already exists" % name) 5048 field_type = format.encode('ascii') 5049 if len(name) > 10: 5050 raise FieldSpecError("Maximum field name length is 10. '%s' is %d characters long." % (name, len(name))) 5051 if not field_type in meta.fieldtypes.keys(): 5052 raise FieldSpecError("Unknown field type: %s" % field_type) 5053 init = self._meta.fieldtypes[field_type]['Init'] 5054 flags = self._meta.fieldtypes[field_type]['flags'] 5055 try: 5056 length, decimals, flags = init(pieces, flags) 5057 except FieldSpecError: 5058 exc = sys.exc_info()[1] 5059 raise FieldSpecError(exc.message + ' (%s:%s)' % (meta.filename, name)) 5060 start = offset 5061 end = offset + length 5062 offset = end 5063 meta.fields.append(name) 5064 cls = meta.fieldtypes[field_type]['Class'] 5065 empty = meta.fieldtypes[field_type]['Empty'] 5066 meta[name] = ( 5067 field_type, 5068 start, 5069 length, 5070 end, 5071 decimals, 5072 flags, 5073 cls, 5074 empty, 5075 ) 5076 self._build_header_fields() 5077 self._update_disk() 5078 if old_table is not None: 5079 old_table.open() 5080 for record in old_table: 5081 self.append(scatter(record)) 5082 old_table.close()
5083
5084 - def allow_nulls(self, fields):
5085 """ 5086 set fields to allow null values 5087 """ 5088 meta = self._meta 5089 if meta.status != READ_WRITE: 5090 raise DbfError('%s not in read/write mode, unable to change field types' % meta.filename) 5091 elif self._versionabbr in ('db3', ): 5092 raise DbfError("Nullable fields are not allowed in %s tables" % self._version) 5093 header = meta.header 5094 fields = self._list_fields(fields) 5095 missing = set(fields) - set(self.field_names) 5096 if missing: 5097 raise FieldMissingError(', '.join(missing)) 5098 if len(self.field_names) + 1 > meta.max_fields: 5099 raise DbfError( 5100 "Adding the hidden _nullflags field would exceed the limit of %d fields for this table" 5101 % (meta.max_fields, ) 5102 ) 5103 old_table = None 5104 if self: 5105 old_table = self.create_backup() 5106 self.zap() 5107 if meta.mfd is not None and not meta.ignorememos: 5108 meta.mfd.close() 5109 meta.mfd = None 5110 meta.memo = None 5111 if not meta.ignorememos: 5112 meta.newmemofile = True 5113 for field in fields: 5114 specs = list(meta[field]) 5115 specs[FLAGS] |= NULLABLE 5116 meta[field] = tuple(specs) 5117 meta.blankrecord = None 5118 self._build_header_fields() 5119 self._update_disk() 5120 if old_table is not None: 5121 old_table.open() 5122 for record in old_table: 5123 self.append(scatter(record)) 5124 old_table.close()
5125
5126 - def append(self, data='', drop=False, multiple=1):
5127 """ 5128 adds <multiple> blank records, and fills fields with dict/tuple values if present 5129 """ 5130 meta = self._meta 5131 if meta.status != READ_WRITE: 5132 raise DbfError('%s not in read/write mode, unable to append records' % meta.filename) 5133 if not self.field_count: 5134 raise DbfError("No fields defined, cannot append") 5135 empty_table = len(self) == 0 5136 dictdata = False 5137 tupledata = False 5138 header = meta.header 5139 kamikaze = '' 5140 if header.record_count == meta.max_records: 5141 raise DbfError("table %r is full; unable to add any more records" % self) 5142 if isinstance(data, (Record, RecordTemplate)): 5143 if data._meta.record_sig[0] == self._meta.record_sig[0]: 5144 kamikaze = data._data 5145 else: 5146 if isinstance(data, dict): 5147 dictdata = data 5148 data = '' 5149 elif isinstance(data, tuple): 5150 if len(data) > self.field_count: 5151 raise DbfError("incoming data has too many values") 5152 tupledata = data 5153 data = '' 5154 elif data: 5155 raise TypeError("data to append must be a tuple, dict, record, or template; not a %r" % type(data)) 5156 newrecord = Record(recnum=header.record_count, layout=meta, kamikaze=kamikaze) 5157 if kamikaze and meta.memofields: 5158 newrecord._start_flux() 5159 for field in meta.memofields: 5160 newrecord[field] = data[field] 5161 newrecord._commit_flux() 5162 5163 self._table.append(newrecord) 5164 header.record_count += 1 5165 if not kamikaze: 5166 try: 5167 if dictdata: 5168 gather(newrecord, dictdata, drop=drop) 5169 elif tupledata: 5170 newrecord._start_flux() 5171 for index, item in enumerate(tupledata): 5172 newrecord[index] = item 5173 newrecord._commit_flux() 5174 elif data: 5175 newrecord._start_flux() 5176 data_fields = field_names(data) 5177 my_fields = self.field_names 5178 for field in data_fields: 5179 if field not in my_fields: 5180 if not drop: 5181 raise DbfError("field %r not in table %r" % (field, self)) 5182 else: 5183 newrecord[field] = data[field] 5184 newrecord._commit_flux() 5185 except Exception: 5186 self._table.pop() # discard failed record 5187 header.record_count = header.record_count - 1 5188 self._update_disk() 5189 raise 5190 multiple -= 1 5191 if multiple: 5192 data = newrecord._data 5193 single = header.record_count 5194 total = single + multiple 5195 while single < total: 5196 multi_record = Record(single, meta, kamikaze=data) 5197 multi_record._start_flux() 5198 self._table.append(multi_record) 5199 for field in meta.memofields: 5200 multi_record[field] = newrecord[field] 5201 single += 1 5202 multi_record._commit_flux() 5203 header.record_count = total # += multiple 5204 newrecord = multi_record 5205 self._update_disk(headeronly=True)
5206
5207 - def close(self):
5208 """ 5209 closes disk files, flushing record data to disk 5210 ensures table data is available if keep_table 5211 ensures memo data is available if keep_memos 5212 """ 5213 if self._meta.location == ON_DISK and self._meta.status != CLOSED: 5214 self._table.flush() 5215 if self._meta.mfd is not None: 5216 self._meta.mfd.close() 5217 self._meta.mfd = None 5218 self._meta.dfd.close() 5219 self._meta.dfd = None 5220 self._meta.status = CLOSED
5221
5222 - def create_backup(self, new_name=None, on_disk=None):
5223 """ 5224 creates a backup table 5225 """ 5226 meta = self._meta 5227 already_open = meta.status != CLOSED 5228 if not already_open: 5229 self.open() 5230 if on_disk is None: 5231 on_disk = meta.location 5232 if not on_disk and new_name is None: 5233 new_name = self.filename + '_backup' 5234 if new_name is None: 5235 upper = self.filename.isupper() 5236 directory, filename = os.path.split(self.filename) 5237 name, ext = os.path.splitext(filename) 5238 extra = ('_backup', '_BACKUP')[upper] 5239 new_name = os.path.join(temp_dir or directory, name + extra + ext) 5240 bkup = Table(new_name, self.structure(), codepage=self.codepage.name, dbf_type=self._versionabbr, on_disk=on_disk) 5241 bkup.open() 5242 for record in self: 5243 bkup.append(record) 5244 bkup.close() 5245 self.backup = new_name 5246 if not already_open: 5247 self.close() 5248 return bkup
5249
5250 - def create_index(self, key):
5251 """ 5252 creates an in-memory index using the function key 5253 """ 5254 meta = self._meta 5255 if meta.status == CLOSED: 5256 raise DbfError('%s is closed' % meta.filename) 5257 return Index(self, key)
5258
5259 - def create_template(self, record=None, defaults=None):
5260 """ 5261 returns a record template that can be used like a record 5262 """ 5263 return RecordTemplate(self._meta, original_record=record, defaults=defaults)
5264
5265 - def delete_fields(self, doomed):
5266 """ 5267 removes field(s) from the table 5268 creates backup files with _backup appended to the file name, 5269 then modifies current structure 5270 """ 5271 meta = self._meta 5272 if meta.status != READ_WRITE: 5273 raise DbfError('%s not in read/write mode, unable to delete fields' % meta.filename) 5274 doomed = self._list_fields(doomed) 5275 header = meta.header 5276 for victim in doomed: 5277 if victim not in meta.user_fields: 5278 raise DbfError("field %s not in table -- delete aborted" % victim) 5279 old_table = None 5280 if self: 5281 old_table = self.create_backup() 5282 self.zap() 5283 if meta.mfd is not None and not meta.ignorememos: 5284 meta.mfd.close() 5285 meta.mfd = None 5286 meta.memo = None 5287 if not meta.ignorememos: 5288 meta.newmemofile = True 5289 if '_nullflags' in meta.fields: 5290 doomed.append('_nullflags') 5291 for victim in doomed: 5292 layout = meta[victim] 5293 meta.fields.pop(meta.fields.index(victim)) 5294 start = layout[START] 5295 end = layout[END] 5296 for field in meta.fields: 5297 if meta[field][START] == end: 5298 specs = list(meta[field]) 5299 end = specs[END] #self._meta[field][END] 5300 specs[START] = start #self._meta[field][START] = start 5301 specs[END] = start + specs[LENGTH] #self._meta[field][END] = start + self._meta[field][LENGTH] 5302 start = specs[END] #self._meta[field][END] 5303 meta[field] = tuple(specs) 5304 self._build_header_fields() 5305 self._update_disk() 5306 for name in list(meta): 5307 if name not in meta.fields: 5308 del meta[name] 5309 if old_table is not None: 5310 old_table.open() 5311 for record in old_table: 5312 self.append(scatter(record), drop=True) 5313 old_table.close()
5314
5315 - def disallow_nulls(self, fields):
5316 """ 5317 set fields to not allow null values 5318 """ 5319 meta = self._meta 5320 if meta.status != READ_WRITE: 5321 raise DbfError('%s not in read/write mode, unable to change field types' % meta.filename) 5322 fields = self._list_fields(fields) 5323 missing = set(fields) - set(self.field_names) 5324 if missing: 5325 raise FieldMissingError(', '.join(missing)) 5326 old_table = None 5327 if self: 5328 old_table = self.create_backup() 5329 self.zap() 5330 if meta.mfd is not None and not meta.ignorememos: 5331 meta.mfd.close() 5332 meta.mfd = None 5333 meta.memo = None 5334 if not meta.ignorememos: 5335 meta.newmemofile = True 5336 for field in fields: 5337 specs = list(meta[field]) 5338 specs[FLAGS] &= 0xff ^ NULLABLE 5339 meta[field] = tuple(specs) 5340 meta.blankrecord = None 5341 self._build_header_fields() 5342 self._update_disk() 5343 if old_table is not None: 5344 old_table.open() 5345 for record in old_table: 5346 self.append(scatter(record)) 5347 old_table.close()
5348
5349 - def field_info(self, field):
5350 """ 5351 returns (dbf type, class, size, dec) of field 5352 """ 5353 if field in self.field_names: 5354 field = self._meta[field] 5355 return FieldInfo(field[TYPE], field[LENGTH], field[DECIMALS], field[CLASS]) 5356 raise FieldMissingError("%s is not a field in %s" % (field, self.filename))
5357
5358 - def index(self, record, start=None, stop=None):
5359 """ 5360 returns the index of record between start and stop 5361 start and stop default to the first and last record 5362 """ 5363 if not isinstance(record, (Record, RecordTemplate, dict, tuple)): 5364 raise TypeError("x should be a record, template, dict, or tuple, not %r" % type(record)) 5365 meta = self._meta 5366 if meta.status == CLOSED: 5367 raise DbfError('%s is closed' % meta.filename) 5368 if start is None: 5369 start = 0 5370 if stop is None: 5371 stop = len(self) 5372 for i in range(start, stop): 5373 if record == (self[i]): 5374 return i 5375 else: 5376 raise NotFoundError("dbf.Table.index(x): x not in table", data=record)
5377
5378 - def new(self, filename, field_specs=None, memo_size=None, ignore_memos=None, codepage=None, default_data_types=None, field_data_types=None, on_disk=True):
5379 """ 5380 returns a new table of the same type 5381 """ 5382 if field_specs is None: 5383 field_specs = self.structure() 5384 if on_disk: 5385 path, name = os.path.split(filename) 5386 if path == "": 5387 filename = os.path.join(os.path.split(self.filename)[0], filename) 5388 elif name == "": 5389 filename = os.path.join(path, os.path.split(self.filename)[1]) 5390 if memo_size is None: 5391 memo_size = self._meta.memo_size 5392 if ignore_memos is None: 5393 ignore_memos = self._meta.ignorememos 5394 if codepage is None: 5395 codepage = self._meta.header.codepage()[0] 5396 if default_data_types is None: 5397 default_data_types = self._meta._default_data_types 5398 if field_data_types is None: 5399 field_data_types = self._meta._field_data_types 5400 return Table(filename, field_specs, memo_size, ignore_memos, codepage, default_data_types, field_data_types, dbf_type=self._versionabbr, on_disk=on_disk)
5401
5402 - def nullable_field(self, field):
5403 """ 5404 returns True if field allows Nulls 5405 """ 5406 if field not in self.field_names: 5407 raise MissingField(field) 5408 return bool(self._meta[field][FLAGS] & NULLABLE)
5409
5410 - def open(self, mode=READ_WRITE):
5411 """ 5412 (re)opens disk table, (re)initializes data structures 5413 """ 5414 if mode not in (READ_WRITE, READ_ONLY): 5415 raise DbfError("mode for open must be 'read-write' or 'read-only', not %r" % mode) 5416 meta = self._meta 5417 if meta.status == mode: 5418 return self # no-op 5419 meta.status = mode 5420 if meta.location == IN_MEMORY: 5421 return self 5422 if '_table' in dir(self): 5423 del self._table 5424 dfd = meta.dfd = open(meta.filename, 'r+b') 5425 dfd.seek(0) 5426 header = meta.header = self._TableHeader(dfd.read(32), self._pack_date, self._unpack_date) 5427 if not header.version in self._supported_tables: 5428 dfd.close() 5429 dfd = None 5430 raise DbfError("Unsupported dbf type: %s [%x]" % (version_map.get(header.version, 'Unknown: %s' % header.version), ord(header.version))) 5431 fieldblock = dfd.read(header.start - 32) 5432 for i in range(len(fieldblock) // 32 + 1): 5433 fieldend = i * 32 5434 if fieldblock[fieldend] == '\x0d': 5435 break 5436 else: 5437 raise BadDataError("corrupt field structure in header") 5438 if len(fieldblock[:fieldend]) % 32 != 0: 5439 raise BadDataError("corrupt field structure in header") 5440 header.fields = fieldblock[:fieldend] 5441 header.extra = fieldblock[fieldend + 1:] # skip trailing \r 5442 self._meta.ignorememos = self._meta.original_ignorememos 5443 self._initialize_fields() 5444 self._check_memo_integrity() 5445 self._index = -1 5446 dfd.seek(0) 5447 return self
5448
5449 - def pack(self):
5450 """ 5451 physically removes all deleted records 5452 """ 5453 meta = self._meta 5454 if meta.status != READ_WRITE: 5455 raise DbfError('%s not in read/write mode, unable to pack records' % meta.filename) 5456 for dbfindex in self._indexen: 5457 dbfindex._clear() 5458 newtable = [] 5459 index = 0 5460 for record in self._table: 5461 if is_deleted(record): 5462 record._recnum = -1 5463 else: 5464 record._recnum = index 5465 newtable.append(record) 5466 index += 1 5467 if meta.location == ON_DISK: 5468 self._table.clear() 5469 else: 5470 self._table[:] = [] 5471 for record in newtable: 5472 self._table.append(record) 5473 self._pack_count += 1 5474 self._meta.header.record_count = index 5475 self._index = -1 5476 self._update_disk() 5477 self.reindex()
5478
5479 - def query(self, criteria):
5480 """ 5481 criteria is a string that will be converted into a function that returns 5482 a List of all matching records 5483 """ 5484 meta = self._meta 5485 if meta.status == CLOSED: 5486 raise DbfError('%s is closed' % meta.filename) 5487 return pql(self, criteria)
5488
5489 - def reindex(self):
5490 """ 5491 reprocess all indices for this table 5492 """ 5493 meta = self._meta 5494 if meta.status == CLOSED: 5495 raise DbfError('%s is closed' % meta.filename) 5496 for dbfindex in self._indexen: 5497 dbfindex._reindex()
5498
5499 - def rename_field(self, oldname, newname):
5500 """ 5501 renames an existing field 5502 """ 5503 meta = self._meta 5504 if meta.status != READ_WRITE: 5505 raise DbfError('%s not in read/write mode, unable to change field names' % meta.filename) 5506 if self: 5507 self.create_backup() 5508 if not oldname in self._meta.user_fields: 5509 raise FieldMissingError("field --%s-- does not exist -- cannot rename it." % oldname) 5510 if newname[0] == '_' or newname[0].isdigit() or not newname.replace('_', '').isalnum(): 5511 raise FieldSpecError("field names cannot start with _ or digits, and can only contain the _, letters, and digits") 5512 newname = newname.lower() 5513 if newname in self._meta.fields: 5514 raise DbfError("field --%s-- already exists" % newname) 5515 if len(newname) > 10: 5516 raise FieldSpecError("maximum field name length is 10. '%s' is %d characters long." % (newname, len(newname))) 5517 self._meta[newname] = self._meta[oldname] 5518 self._meta.fields[self._meta.fields.index(oldname)] = newname 5519 self._build_header_fields() 5520 self._update_disk(headeronly=True)
5521
5522 - def resize_field(self, chosen, new_size):
5523 """ 5524 resizes field (C only at this time) 5525 creates backup file, then modifies current structure 5526 """ 5527 meta = self._meta 5528 if meta.status != READ_WRITE: 5529 raise DbfError('%s not in read/write mode, unable to change field size' % meta.filename) 5530 if not 0 < new_size < 256: 5531 raise DbfError("new_size must be between 1 and 255 (use delete_fields to remove a field)") 5532 chosen = self._list_fields(chosen) 5533 for candidate in chosen: 5534 if candidate not in self._meta.user_fields: 5535 raise DbfError("field %s not in table -- resize aborted" % candidate) 5536 elif self.field_info(candidate).field_type != 'C': 5537 raise DbfError("field %s is not Character -- resize aborted" % candidate) 5538 if self: 5539 old_table = self.create_backup() 5540 self.zap() 5541 if meta.mfd is not None and not meta.ignorememos: 5542 meta.mfd.close() 5543 meta.mfd = None 5544 meta.memo = None 5545 if not meta.ignorememos: 5546 meta.newmemofile = True 5547 struct = self.structure() 5548 meta.user_fields[:] = [] 5549 new_struct = [] 5550 for field_spec in struct: 5551 name, spec = field_spec.split(' ', 1) 5552 if name in chosen: 5553 spec = "C(%d)" % new_size 5554 new_struct.append(' '.join([name, spec])) 5555 self.add_fields(';'.join(new_struct)) 5556 if old_table is not None: 5557 old_table.open() 5558 for record in old_table: 5559 self.append(scatter(record), drop=True) 5560 old_table.close()
5561
5562 - def structure(self, fields=None):
5563 """ 5564 return field specification list suitable for creating same table layout 5565 fields should be a list of fields or None for all fields in table 5566 """ 5567 field_specs = [] 5568 fields = self._list_fields(fields) 5569 try: 5570 for name in fields: 5571 field_specs.append(self._field_layout(self.field_names.index(name))) 5572 except ValueError: 5573 raise DbfError("field %s does not exist" % name) 5574 return field_specs
5575
5576 - def zap(self):
5577 """ 5578 removes all records from table -- this cannot be undone! 5579 """ 5580 meta = self._meta 5581 if meta.status != READ_WRITE: 5582 raise DbfError('%s not in read/write mode, unable to zap table' % meta.filename) 5583 if meta.location == IN_MEMORY: 5584 self._table[:] = [] 5585 else: 5586 self._table.clear() 5587 if meta.memo: 5588 meta.memo._zap() 5589 meta.header.record_count = 0 5590 self._index = -1 5591 self._update_disk()
5592
5593 5594 -class Db3Table(Table):
5595 """ 5596 Provides an interface for working with dBase III tables. 5597 """ 5598 5599 _version = 'dBase III Plus' 5600 _versionabbr = 'db3' 5601 5602 @MutableDefault
5603 - def _field_types():
5604 return { 5605 'C' : { 5606 'Type':'Character', 'Retrieve':retrieve_character, 'Update':update_character, 'Blank':lambda x: ' ' * x, 'Init':add_character, 5607 'Class':unicode, 'Empty':unicode, 'flags':tuple(), 5608 }, 5609 'D' : { 5610 'Type':'Date', 'Retrieve':retrieve_date, 'Update':update_date, 'Blank':lambda x: ' ', 'Init':add_date, 5611 'Class':datetime.date, 'Empty':none, 'flags':tuple(), 5612 }, 5613 'F' : { 5614 'Type':'Numeric', 'Retrieve':retrieve_numeric, 'Update':update_numeric, 'Blank':lambda x: ' ' * x, 'Init':add_numeric, 5615 'Class':'default', 'Empty':none, 'flags':tuple(), 5616 }, 5617 'L' : { 5618 'Type':'Logical', 'Retrieve':retrieve_logical, 'Update':update_logical, 'Blank':lambda x: '?', 'Init':add_logical, 5619 'Class':bool, 'Empty':none, 'flags':tuple(), 5620 }, 5621 'M' : { 5622 'Type':'Memo', 'Retrieve':retrieve_memo, 'Update':update_memo, 'Blank':lambda x: ' ', 'Init':add_memo, 5623 'Class':unicode, 'Empty':unicode, 'flags':tuple(), 5624 }, 5625 'N' : { 5626 'Type':'Numeric', 'Retrieve':retrieve_numeric, 'Update':update_numeric, 'Blank':lambda x: ' ' * x, 'Init':add_numeric, 5627 'Class':'default', 'Empty':none, 'flags':tuple(), 5628 } }
5629 5630 _memoext = '.dbt' 5631 _memoClass = _Db3Memo 5632 _yesMemoMask = '\x80' 5633 _noMemoMask = '\x7f' 5634 _binary_types = () 5635 _character_types = ('C', 'M') 5636 _currency_types = tuple() 5637 _date_types = ('D',) 5638 _datetime_types = tuple() 5639 _decimal_types = ('N', 'F') 5640 _fixed_types = ('D', 'L', 'M') 5641 _logical_types = ('L',) 5642 _memo_types = ('M',) 5643 _numeric_types = ('N', 'F') 5644 _variable_types = ('C', 'N') 5645 _dbfTableHeader = array('c', '\x00' * 32) 5646 _dbfTableHeader[0] = '\x03' # version - dBase III w/o memo's 5647 _dbfTableHeader[8:10] = array('c', pack_short_int(33)) 5648 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 5649 _dbfTableHeader[29] = '\x03' # code page -- 437 US-MS DOS 5650 _dbfTableHeader = _dbfTableHeader.tostring() 5651 _dbfTableHeaderExtra = '' 5652 _supported_tables = ['\x03', '\x83'] 5653
5654 - def _check_memo_integrity(self):
5655 """ 5656 dBase III and Clipper 5657 """ 5658 if not self._meta.ignorememos: 5659 memo_fields = False 5660 for field in self._meta.fields: 5661 if self._meta[field][TYPE] in self._memo_types: 5662 memo_fields = True 5663 break 5664 if memo_fields and self._meta.header.version != '\x83': 5665 self._meta.dfd.close() 5666 self._meta.dfd = None 5667 raise BadDataError("Table structure corrupt: memo fields exist, header declares no memos") 5668 elif memo_fields and not os.path.exists(self._meta.memoname): 5669 self._meta.dfd.close() 5670 self._meta.dfd = None 5671 raise BadDataError("Table structure corrupt: memo fields exist without memo file") 5672 if memo_fields: 5673 try: 5674 self._meta.memo = self._memoClass(self._meta) 5675 except Exception: 5676 exc = sys.exc_info()[1] 5677 self._meta.dfd.close() 5678 self._meta.dfd = None 5679 raise BadDataError("Table structure corrupt: unable to use memo file (%s)" % exc.args[-1])
5680
5681 - def _initialize_fields(self):
5682 """ 5683 builds the FieldList of names, types, and descriptions 5684 """ 5685 old_fields = defaultdict(dict) 5686 meta = self._meta 5687 for name in meta.fields: 5688 old_fields[name]['type'] = meta[name][TYPE] 5689 old_fields[name]['empty'] = meta[name][EMPTY] 5690 old_fields[name]['class'] = meta[name][CLASS] 5691 meta.fields[:] = [] 5692 offset = 1 5693 fieldsdef = meta.header.fields 5694 if len(fieldsdef) % 32 != 0: 5695 raise BadDataError("field definition block corrupt: %d bytes in size" % len(fieldsdef)) 5696 if len(fieldsdef) // 32 != meta.header.field_count: 5697 raise BadDataError("Header shows %d fields, but field definition block has %d fields" % (meta.header.field_count, len(fieldsdef) // 32)) 5698 total_length = meta.header.record_length 5699 for i in range(meta.header.field_count): 5700 fieldblock = fieldsdef[i*32:(i+1)*32] 5701 name = unpack_str(fieldblock[:11]) 5702 type = fieldblock[11] 5703 if not type in meta.fieldtypes: 5704 raise BadDataError("Unknown field type: %s" % type) 5705 start = offset 5706 length = ord(fieldblock[16]) 5707 offset += length 5708 end = start + length 5709 decimals = ord(fieldblock[17]) 5710 flags = ord(fieldblock[18]) 5711 if name in meta.fields: 5712 raise BadDataError('Duplicate field name found: %s' % name) 5713 meta.fields.append(name) 5714 if name in old_fields and old_fields[name]['type'] == type: 5715 cls = old_fields[name]['class'] 5716 empty = old_fields[name]['empty'] 5717 else: 5718 cls = meta.fieldtypes[type]['Class'] 5719 empty = meta.fieldtypes[type]['Empty'] 5720 meta[name] = ( 5721 type, 5722 start, 5723 length, 5724 end, 5725 decimals, 5726 flags, 5727 cls, 5728 empty, 5729 ) 5730 if offset != total_length: 5731 raise BadDataError("Header shows record length of %d, but calculated record length is %d" % (total_length, offset)) 5732 meta.user_fields = [f for f in meta.fields if not meta[f][FLAGS] & SYSTEM] 5733 meta.user_field_count = len(meta.user_fields) 5734 Record._create_blank_data(meta)
5735
5736 5737 -class ClpTable(Db3Table):
5738 """ 5739 Provides an interface for working with Clipper tables. 5740 """ 5741 5742 _version = 'Clipper 5' 5743 _versionabbr = 'clp' 5744 5745 @MutableDefault
5746 - def _field_types():
5747 return { 5748 'C' : { 5749 'Type':'Character', 'Retrieve':retrieve_character, 'Update':update_character, 'Blank':lambda x: ' ' * x, 'Init':add_clp_character, 5750 'Class':unicode, 'Empty':unicode, 'flags':tuple(), 5751 }, 5752 'D' : { 5753 'Type':'Date', 'Retrieve':retrieve_date, 'Update':update_date, 'Blank':lambda x: ' ', 'Init':add_date, 5754 'Class':datetime.date, 'Empty':none, 'flags':tuple(), 5755 }, 5756 'F' : { 5757 'Type':'Numeric', 'Retrieve':retrieve_numeric, 'Update':update_numeric, 'Blank':lambda x: ' ' * x, 'Init':add_numeric, 5758 'Class':'default', 'Empty':none, 'flags':tuple(), 5759 }, 5760 'L' : { 5761 'Type':'Logical', 'Retrieve':retrieve_logical, 'Update':update_logical, 'Blank':lambda x: '?', 'Init':add_logical, 5762 'Class':bool, 'Empty':none, 'flags':tuple(), 5763 }, 5764 'M' : { 5765 'Type':'Memo', 'Retrieve':retrieve_memo, 'Update':update_memo, 'Blank':lambda x: ' ', 'Init':add_memo, 5766 'Class':unicode, 'Empty':unicode, 'flags':tuple(), 5767 }, 5768 'N' : { 5769 'Type':'Numeric', 'Retrieve':retrieve_numeric, 'Update':update_numeric, 'Blank':lambda x: ' ' * x, 'Init':add_numeric, 5770 'Class':'default', 'Empty':none, 'flags':tuple(), 5771 } }
5772 5773 _memoext = '.dbt' 5774 _memoClass = _Db3Memo 5775 _yesMemoMask = '\x80' 5776 _noMemoMask = '\x7f' 5777 _binary_types = () 5778 _character_types = ('C', 'M') 5779 _currency_types = tuple() 5780 _date_types = ('D',) 5781 _datetime_types = tuple() 5782 _decimal_types = ('N', 'F') 5783 _fixed_types = ('D', 'L', 'M') 5784 _logical_types = ('L',) 5785 _memo_types = ('M',) 5786 _numeric_types = ('N', 'F') 5787 _variable_types = ('C', 'N') 5788 _dbfTableHeader = array('c', '\x00' * 32) 5789 _dbfTableHeader[0] = '\x03' # version - dBase III w/o memo's 5790 _dbfTableHeader[8:10] = array('c', pack_short_int(33)) 5791 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 5792 _dbfTableHeader[29] = '\x03' # code page -- 437 US-MS DOS 5793 _dbfTableHeader = _dbfTableHeader.tostring() 5794 _dbfTableHeaderExtra = '' 5795 _supported_tables = ['\x03', '\x83'] 5796
5797 - class _TableHeader(Table._TableHeader):
5798 """ 5799 represents the data block that defines a tables type and layout 5800 """ 5801 5802 @property
5803 - def fields(self):
5804 "field block structure" 5805 fieldblock = self._data[32:] 5806 for i in range(len(fieldblock)//32+1): 5807 cr = i * 32 5808 if fieldblock[cr] == '\x0d': 5809 break 5810 else: 5811 raise BadDataError("corrupt field structure") 5812 return fieldblock[:cr].tostring()
5813 5814 @fields.setter
5815 - def fields(self, block):
5816 fieldblock = self._data[32:] 5817 for i in range(len(fieldblock)//32+1): 5818 cr = i * 32 5819 if fieldblock[cr] == '\x0d': 5820 break 5821 else: 5822 raise BadDataError("corrupt field structure") 5823 cr += 32 # convert to indexing main structure 5824 fieldlen = len(block) 5825 if fieldlen % 32 != 0: 5826 raise BadDataError("fields structure corrupt: %d is not a multiple of 32" % fieldlen) 5827 self._data[32:cr] = array('c', block) # fields 5828 self._data[8:10] = array('c', pack_short_int(len(self._data))) # start 5829 fieldlen = fieldlen // 32 5830 recordlen = 1 # deleted flag 5831 for i in range(fieldlen): 5832 recordlen += ord(block[i*32+16]) 5833 if block[i*32+11] == 'C': 5834 recordlen += ord(block[i*32+17]) * 256 5835 self._data[10:12] = array('c', pack_short_int(recordlen))
5836 5837
5838 - def _build_header_fields(self):
5839 """ 5840 constructs fieldblock for disk table 5841 """ 5842 fieldblock = array('c', '') 5843 memo = False 5844 nulls = False 5845 meta = self._meta 5846 header = meta.header 5847 header.version = chr(ord(header.version) & ord(self._noMemoMask)) 5848 meta.fields = [f for f in meta.fields if f != '_nullflags'] 5849 total_length = 1 # delete flag 5850 for field in meta.fields: 5851 layout = meta[field] 5852 if meta.fields.count(field) > 1: 5853 raise BadDataError("corrupted field structure (noticed in _build_header_fields)") 5854 fielddef = array('c', '\x00' * 32) 5855 fielddef[:11] = array('c', pack_str(meta.encoder(field)[0])) 5856 fielddef[11] = layout[TYPE] 5857 fielddef[12:16] = array('c', pack_long_int(layout[START])) 5858 total_length += layout[LENGTH] 5859 if layout[TYPE] == 'C': # long character field 5860 fielddef[16] = chr(layout[LENGTH] % 256) 5861 fielddef[17] = chr(layout[LENGTH] // 256) 5862 else: 5863 fielddef[16] = chr(layout[LENGTH]) 5864 fielddef[17] = chr(layout[DECIMALS]) 5865 fielddef[18] = chr(layout[FLAGS]) 5866 fieldblock.extend(fielddef) 5867 if layout[TYPE] in meta.memo_types: 5868 memo = True 5869 if layout[FLAGS] & NULLABLE: 5870 nulls = True 5871 if memo: 5872 header.version = chr(ord(header.version) | ord(self._yesMemoMask)) 5873 if meta.memo is None: 5874 meta.memo = self._memoClass(meta) 5875 else: 5876 if os.path.exists(meta.memoname): 5877 if meta.mfd is not None: 5878 meta.mfd.close() 5879 5880 os.remove(meta.memoname) 5881 meta.memo = None 5882 if nulls: 5883 start = layout[START] + layout[LENGTH] 5884 length, one_more = divmod(len(meta.fields), 8) 5885 if one_more: 5886 length += 1 5887 fielddef = array('c', '\x00' * 32) 5888 fielddef[:11] = array('c', pack_str('_nullflags')) 5889 fielddef[11] = '0' 5890 fielddef[12:16] = array('c', pack_long_int(start)) 5891 fielddef[16] = chr(length) 5892 fielddef[17] = chr(0) 5893 fielddef[18] = chr(BINARY | SYSTEM) 5894 fieldblock.extend(fielddef) 5895 meta.fields.append('_nullflags') 5896 nullflags = ( 5897 '0', # type 5898 start, # start 5899 length, # length 5900 start + length, # end 5901 0, # decimals 5902 BINARY | SYSTEM, # flags 5903 none, # class 5904 none, # empty 5905 ) 5906 meta['_nullflags'] = nullflags 5907 header.fields = fieldblock.tostring() 5908 header.record_length = total_length 5909 meta.user_fields = [f for f in meta.fields if not meta[f][FLAGS] & SYSTEM] 5910 meta.user_field_count = len(meta.user_fields) 5911 Record._create_blank_data(meta)
5912
5913 - def _initialize_fields(self):
5914 """ 5915 builds the FieldList of names, types, and descriptions 5916 """ 5917 meta = self._meta 5918 old_fields = defaultdict(dict) 5919 for name in meta.fields: 5920 old_fields[name]['type'] = meta[name][TYPE] 5921 old_fields[name]['empty'] = meta[name][EMPTY] 5922 old_fields[name]['class'] = meta[name][CLASS] 5923 meta.fields[:] = [] 5924 offset = 1 5925 fieldsdef = meta.header.fields 5926 if len(fieldsdef) % 32 != 0: 5927 raise BadDataError("field definition block corrupt: %d bytes in size" % len(fieldsdef)) 5928 if len(fieldsdef) // 32 != meta.header.field_count: 5929 raise BadDataError("Header shows %d fields, but field definition block has %d fields" 5930 (meta.header.field_count, len(fieldsdef) // 32)) 5931 total_length = meta.header.record_length 5932 for i in range(meta.header.field_count): 5933 fieldblock = fieldsdef[i*32:(i+1)*32] 5934 name = unpack_str(fieldblock[:11]) 5935 type = fieldblock[11] 5936 if not type in meta.fieldtypes: 5937 raise BadDataError("Unknown field type: %s" % type) 5938 start = offset 5939 length = ord(fieldblock[16]) 5940 decimals = ord(fieldblock[17]) 5941 if type == 'C': 5942 length += decimals * 256 5943 offset += length 5944 end = start + length 5945 flags = ord(fieldblock[18]) 5946 if name in meta.fields: 5947 raise BadDataError('Duplicate field name found: %s' % name) 5948 meta.fields.append(name) 5949 if name in old_fields and old_fields[name]['type'] == type: 5950 cls = old_fields[name]['class'] 5951 empty = old_fields[name]['empty'] 5952 else: 5953 cls = meta.fieldtypes[type]['Class'] 5954 empty = meta.fieldtypes[type]['Empty'] 5955 meta[name] = ( 5956 type, 5957 start, 5958 length, 5959 end, 5960 decimals, 5961 flags, 5962 cls, 5963 empty, 5964 ) 5965 if offset != total_length: 5966 raise BadDataError("Header shows record length of %d, but calculated record length is %d" 5967 (total_length, offset)) 5968 meta.user_fields = [f for f in meta.fields if not meta[f][FLAGS] & SYSTEM] 5969 meta.user_field_count = len(meta.user_fields) 5970 Record._create_blank_data(meta)
5971
5972 5973 -class FpTable(Table):
5974 """ 5975 Provides an interface for working with FoxPro 2 tables 5976 """ 5977 5978 _version = 'Foxpro' 5979 _versionabbr = 'fp' 5980 5981 @MutableDefault
5982 - def _field_types():
5983 return { 5984 'C' : { 5985 'Type':'Character', 'Retrieve':retrieve_character, 'Update':update_character, 'Blank':lambda x: ' ' * x, 'Init':add_vfp_character, 5986 'Class':unicode, 'Empty':unicode, 'flags':('binary', 'nocptrans', 'null', ), 5987 }, 5988 'F' : { 5989 'Type':'Float', 'Retrieve':retrieve_numeric, 'Update':update_numeric, 'Blank':lambda x: ' ' * x, 'Init':add_vfp_numeric, 5990 'Class':'default', 'Empty':none, 'flags':('null', ), 5991 }, 5992 'N' : { 5993 'Type':'Numeric', 'Retrieve':retrieve_numeric, 'Update':update_numeric, 'Blank':lambda x: ' ' * x, 'Init':add_vfp_numeric, 5994 'Class':'default', 'Empty':none, 'flags':('null', ), 5995 }, 5996 'L' : { 5997 'Type':'Logical', 'Retrieve':retrieve_logical, 'Update':update_logical, 'Blank':lambda x: '?', 'Init':add_logical, 5998 'Class':bool, 'Empty':none, 'flags':('null', ), 5999 }, 6000 'D' : { 6001 'Type':'Date', 'Retrieve':retrieve_date, 'Update':update_date, 'Blank':lambda x: ' ', 'Init':add_date, 6002 'Class':datetime.date, 'Empty':none, 'flags':('null', ), 6003 }, 6004 'M' : { 6005 'Type':'Memo', 'Retrieve':retrieve_memo, 'Update':update_memo, 'Blank':lambda x: ' ', 'Init':add_memo, 6006 'Class':unicode, 'Empty':unicode, 'flags':('binary', 'nocptrans', 'null', ), 6007 }, 6008 'G' : { 6009 'Type':'General', 'Retrieve':retrieve_memo, 'Update':update_memo, 'Blank':lambda x: ' ', 'Init':add_memo, 6010 'Class':bytes, 'Empty':bytes, 'flags':('null', ), 6011 }, 6012 'P' : { 6013 'Type':'Picture', 'Retrieve':retrieve_memo, 'Update':update_memo, 'Blank':lambda x: ' ', 'Init':add_memo, 6014 'Class':bytes, 'Empty':bytes, 'flags':('null', ), 6015 }, 6016 '0' : { 6017 'Type':'_NullFlags', 'Retrieve':unsupported_type, 'Update':unsupported_type, 'Blank':lambda x: '\x00' * x, 'Init':None, 6018 'Class':none, 'Empty':none, 'flags':('binary', 'system', ), 6019 } }
6020 6021 _memoext = '.fpt' 6022 _memoClass = _VfpMemo 6023 _yesMemoMask = '\xf5' # 1111 0101 6024 _noMemoMask = '\x03' # 0000 0011 6025 _binary_types = ('G', 'P') 6026 _character_types = ('C', 'D', 'F', 'L', 'M', 'N') # field representing character data 6027 _currency_types = tuple() 6028 _date_types = ('D',) 6029 _datetime_types = tuple() 6030 _fixed_types = ('D', 'G', 'L', 'M', 'P') 6031 _logical_types = ('L',) 6032 _memo_types = ('G', 'M', 'P') 6033 _numeric_types = ('F', 'N') 6034 _text_types = ('C', 'M') 6035 _variable_types = ('C', 'F', 'N') 6036 _supported_tables = ('\x03', '\xf5') 6037 _dbfTableHeader = array('c', '\x00' * 32) 6038 _dbfTableHeader[0] = '\x30' # version - Foxpro 6 0011 0000 6039 _dbfTableHeader[8:10] = array('c', pack_short_int(33 + 263)) 6040 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 6041 _dbfTableHeader[29] = '\x03' # code page -- 437 US-MS DOS 6042 _dbfTableHeader = _dbfTableHeader.tostring() 6043 _dbfTableHeaderExtra = '\x00' * 263 6044
6045 - def _check_memo_integrity(self):
6046 if not self._meta.ignorememos: 6047 memo_fields = False 6048 for field in self._meta.fields: 6049 if self._meta[field][TYPE] in self._memo_types: 6050 memo_fields = True 6051 break 6052 if memo_fields and not os.path.exists(self._meta.memoname): 6053 self._meta.dfd.close() 6054 self._meta.dfd = None 6055 raise BadDataError("Table structure corrupt: memo fields exist without memo file") 6056 elif not memo_fields and os.path.exists(self._meta.memoname): 6057 self._meta.dfd.close() 6058 self._meta.dfd = None 6059 raise BadDataError("Table structure corrupt: no memo fields exist but memo file does") 6060 if memo_fields: 6061 try: 6062 self._meta.memo = self._memoClass(self._meta) 6063 except Exception: 6064 exc = sys.exc_info()[1] 6065 self._meta.dfd.close() 6066 self._meta.dfd = None 6067 raise BadDataError("Table structure corrupt: unable to use memo file (%s)" % exc.args[-1])
6068
6069 - def _initialize_fields(self):
6070 """ 6071 builds the FieldList of names, types, and descriptions 6072 """ 6073 meta = self._meta 6074 old_fields = defaultdict(dict) 6075 for name in meta.fields: 6076 old_fields[name]['type'] = meta[name][TYPE] 6077 old_fields[name]['class'] = meta[name][CLASS] 6078 old_fields[name]['empty'] = meta[name][EMPTY] 6079 meta.fields[:] = [] 6080 offset = 1 6081 fieldsdef = meta.header.fields 6082 if len(fieldsdef) % 32 != 0: 6083 raise BadDataError("field definition block corrupt: %d bytes in size" % len(fieldsdef)) 6084 if len(fieldsdef) // 32 != meta.header.field_count: 6085 raise BadDataError("Header shows %d fields, but field definition block has %d fields" 6086 (meta.header.field_count, len(fieldsdef) // 32)) 6087 total_length = meta.header.record_length 6088 for i in range(meta.header.field_count): 6089 fieldblock = fieldsdef[i*32:(i+1)*32] 6090 name = unpack_str(fieldblock[:11]) 6091 type = fieldblock[11] 6092 if not type in meta.fieldtypes: 6093 raise BadDataError("Unknown field type: %s" % type) 6094 start = offset 6095 length = ord(fieldblock[16]) 6096 offset += length 6097 end = start + length 6098 decimals = ord(fieldblock[17]) 6099 flags = ord(fieldblock[18]) 6100 if name in meta.fields: 6101 raise BadDataError('Duplicate field name found: %s' % name) 6102 meta.fields.append(name) 6103 if name in old_fields and old_fields[name]['type'] == type: 6104 cls = old_fields[name]['class'] 6105 empty = old_fields[name]['empty'] 6106 else: 6107 cls = meta.fieldtypes[type]['Class'] 6108 empty = meta.fieldtypes[type]['Empty'] 6109 meta[name] = ( 6110 type, 6111 start, 6112 length, 6113 end, 6114 decimals, 6115 flags, 6116 cls, 6117 empty, 6118 ) 6119 if offset != total_length: 6120 raise BadDataError("Header shows record length of %d, but calculated record length is %d" % (total_length, offset)) 6121 meta.user_fields = [f for f in meta.fields if not meta[f][FLAGS] & SYSTEM] 6122 meta.user_field_count = len(meta.user_fields) 6123 Record._create_blank_data(meta)
6124 6125 @staticmethod
6126 - def _pack_date(date):
6127 """ 6128 Returns a group of three bytes, in integer form, of the date 6129 """ 6130 return "%c%c%c" % (date.year - 2000, date.month, date.day)
6131 6132 @staticmethod
6133 - def _unpack_date(bytestr):
6134 """ 6135 Returns a Date() of the packed three-byte date passed in 6136 """ 6137 year, month, day = struct.unpack('<BBB', bytestr) 6138 year += 2000 6139 return Date(year, month, day)
6140
6141 -class VfpTable(FpTable):
6142 """ 6143 Provides an interface for working with Visual FoxPro 6 tables 6144 """ 6145 6146 _version = 'Visual Foxpro' 6147 _versionabbr = 'vfp' 6148 6149 @MutableDefault
6150 - def _field_types():
6151 return { 6152 'C' : { 6153 'Type':'Character', 'Retrieve':retrieve_character, 'Update':update_character, 'Blank':lambda x: ' ' * x, 'Init':add_vfp_character, 6154 'Class':unicode, 'Empty':unicode, 'flags':('binary', 'nocptrans', 'null', ), 6155 }, 6156 'Y' : { 6157 'Type':'Currency', 'Retrieve':retrieve_currency, 'Update':update_currency, 'Blank':lambda x: '\x00' * 8, 'Init':add_vfp_currency, 6158 'Class':Decimal, 'Empty':none, 'flags':('null', ), 6159 }, 6160 'B' : { 6161 'Type':'Double', 'Retrieve':retrieve_double, 'Update':update_double, 'Blank':lambda x: '\x00' * 8, 'Init':add_vfp_double, 6162 'Class':float, 'Empty':none, 'flags':('null', ), 6163 }, 6164 'F' : { 6165 'Type':'Float', 'Retrieve':retrieve_numeric, 'Update':update_numeric, 'Blank':lambda x: ' ' * x, 'Init':add_vfp_numeric, 6166 'Class':'default', 'Empty':none, 'flags':('null', ), 6167 }, 6168 'N' : { 6169 'Type':'Numeric', 'Retrieve':retrieve_numeric, 'Update':update_numeric, 'Blank':lambda x: ' ' * x, 'Init':add_vfp_numeric, 6170 'Class':'default', 'Empty':none, 'flags':('null', ), 6171 }, 6172 'I' : { 6173 'Type':'Integer', 'Retrieve':retrieve_integer, 'Update':update_integer, 'Blank':lambda x: '\x00' * 4, 'Init':add_vfp_integer, 6174 'Class':int, 'Empty':none, 'flags':('null', ), 6175 }, 6176 'L' : { 6177 'Type':'Logical', 'Retrieve':retrieve_logical, 'Update':update_logical, 'Blank':lambda x: '?', 'Init':add_logical, 6178 'Class':bool, 'Empty':none, 'flags':('null', ), 6179 }, 6180 'D' : { 6181 'Type':'Date', 'Retrieve':retrieve_date, 'Update':update_date, 'Blank':lambda x: ' ', 'Init':add_date, 6182 'Class':datetime.date, 'Empty':none, 'flags':('null', ), 6183 }, 6184 'T' : { 6185 'Type':'DateTime', 'Retrieve':retrieve_vfp_datetime, 'Update':update_vfp_datetime, 'Blank':lambda x: '\x00' * 8, 'Init':add_vfp_datetime, 6186 'Class':datetime.datetime, 'Empty':none, 'flags':('null', ), 6187 }, 6188 'M' : { 6189 'Type':'Memo', 'Retrieve':retrieve_vfp_memo, 'Update':update_vfp_memo, 'Blank':lambda x: '\x00\x00\x00\x00', 'Init':add_vfp_memo, 6190 'Class':unicode, 'Empty':unicode, 'flags':('binary', 'nocptrans', 'null', ), 6191 }, 6192 'G' : { 6193 'Type':'General', 'Retrieve':retrieve_vfp_memo, 'Update':update_vfp_memo, 'Blank':lambda x: '\x00\x00\x00\x00', 'Init':add_vfp_memo, 6194 'Class':bytes, 'Empty':bytes, 'flags':('null', ), 6195 }, 6196 'P' : { 6197 'Type':'Picture', 'Retrieve':retrieve_vfp_memo, 'Update':update_vfp_memo, 'Blank':lambda x: '\x00\x00\x00\x00', 'Init':add_vfp_memo, 6198 'Class':bytes, 'Empty':bytes, 'flags':('null', ), 6199 }, 6200 '0' : { 6201 'Type':'_NullFlags', 'Retrieve':unsupported_type, 'Update':unsupported_type, 'Blank':lambda x: '\x00' * x, 'Init':int, 6202 'Class':none, 'Empty':none, 'flags':('binary', 'system',), 6203 } }
6204 6205 _memoext = '.fpt' 6206 _memoClass = _VfpMemo 6207 _yesMemoMask = '\x30' # 0011 0000 6208 _noMemoMask = '\x30' # 0011 0000 6209 _binary_types = ('B', 'G', 'I', 'P', 'T', 'Y') 6210 _character_types = ('C', 'D', 'F', 'L', 'M', 'N') # field representing character data 6211 _currency_types = ('Y',) 6212 _date_types = ('D', 'T') 6213 _datetime_types = ('T',) 6214 _fixed_types = ('B', 'D', 'G', 'I', 'L', 'M', 'P', 'T', 'Y') 6215 _logical_types = ('L',) 6216 _memo_types = ('G', 'M', 'P') 6217 _numeric_types = ('B', 'F', 'I', 'N', 'Y') 6218 _variable_types = ('C', 'F', 'N') 6219 _supported_tables = ('\x30', '\x31') 6220 _dbfTableHeader = array('c', '\x00' * 32) 6221 _dbfTableHeader[0] = '\x30' # version - Foxpro 6 0011 0000 6222 _dbfTableHeader[8:10] = array('c', pack_short_int(33 + 263)) 6223 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 6224 _dbfTableHeader[29] = '\x03' # code page -- 437 US-MS DOS 6225 _dbfTableHeader = _dbfTableHeader.tostring() 6226 _dbfTableHeaderExtra = '\x00' * 263 6227
6228 - def _initialize_fields(self):
6229 """ 6230 builds the FieldList of names, types, and descriptions 6231 """ 6232 meta = self._meta 6233 old_fields = defaultdict(dict) 6234 for name in meta.fields: 6235 old_fields[name]['type'] = meta[name][TYPE] 6236 old_fields[name]['class'] = meta[name][CLASS] 6237 old_fields[name]['empty'] = meta[name][EMPTY] 6238 meta.fields[:] = [] 6239 offset = 1 6240 fieldsdef = meta.header.fields 6241 meta.nullflags = None 6242 total_length = meta.header.record_length 6243 for i in range(meta.header.field_count): 6244 fieldblock = fieldsdef[i*32:(i+1)*32] 6245 name = unpack_str(fieldblock[:11]) 6246 type = fieldblock[11] 6247 if not type in meta.fieldtypes: 6248 raise BadDataError("Unknown field type: %s" % type) 6249 start = unpack_long_int(fieldblock[12:16]) 6250 length = ord(fieldblock[16]) 6251 offset += length 6252 end = start + length 6253 decimals = ord(fieldblock[17]) 6254 flags = ord(fieldblock[18]) 6255 if name in meta.fields: 6256 raise BadDataError('Duplicate field name found: %s' % name) 6257 meta.fields.append(name) 6258 if name in old_fields and old_fields[name]['type'] == type: 6259 cls = old_fields[name]['class'] 6260 empty = old_fields[name]['empty'] 6261 else: 6262 cls = meta.fieldtypes[type]['Class'] 6263 empty = meta.fieldtypes[type]['Empty'] 6264 meta[name] = ( 6265 type, 6266 start, 6267 length, 6268 end, 6269 decimals, 6270 flags, 6271 cls, 6272 empty, 6273 ) 6274 if offset != total_length: 6275 raise BadDataError("Header shows record length of %d, but calculated record length is %d" % (total_length, offset)) 6276 meta.user_fields = [f for f in meta.fields if not meta[f][FLAGS] & SYSTEM] 6277 meta.user_field_count = len(meta.user_fields) 6278 Record._create_blank_data(meta)
6279
6280 6281 -class List(_Navigation):
6282 """ 6283 list of Dbf records, with set-like behavior 6284 """ 6285 6286 _desc = '' 6287
6288 - def __init__(self, records=None, desc=None, key=None):
6289 self._list = [] 6290 self._set = set() 6291 self._tables = dict() 6292 if key is not None: 6293 self.key = key 6294 if key.__doc__ is None: 6295 key.__doc__ = 'unknown' 6296 key = self.key 6297 self._current = -1 6298 if isinstance(records, self.__class__) and key is records.key: 6299 self._list = records._list[:] 6300 self._set = records._set.copy() 6301 self._current = 0 6302 elif records is not None: 6303 for record in records: 6304 value = key(record) 6305 item = (source_table(record), recno(record), value) 6306 if value not in self._set: 6307 self._set.add(value) 6308 self._list.append(item) 6309 self._current = 0 6310 if desc is not None: 6311 self._desc = desc
6312
6313 - def __add__(self, other):
6314 self._still_valid_check() 6315 key = self.key 6316 if isinstance(other, (Table, list)): 6317 other = self.__class__(other, key=key) 6318 if isinstance(other, self.__class__): 6319 other._still_valid_check() 6320 result = self.__class__() 6321 result._set = self._set.copy() 6322 result._list[:] = self._list[:] 6323 result._tables = {} 6324 result._tables.update(self._tables) 6325 result.key = self.key 6326 if key is other.key: # same key? just compare key values 6327 for item in other._list: 6328 result._maybe_add(item) 6329 else: # different keys, use this list's key on other's records 6330 for rec in other: 6331 result._maybe_add((source_table(rec), recno(rec), key(rec))) 6332 return result 6333 return NotImplemented
6334
6335 - def __contains__(self, data):
6336 self._still_valid_check() 6337 if not isinstance(data, (Record, RecordTemplate, tuple, dict)): 6338 raise TypeError("%r is not a record, templace, tuple, nor dict" % (data, )) 6339 try: # attempt quick method 6340 item = self.key(data) 6341 if not isinstance(item, tuple): 6342 item = (item, ) 6343 return item in self._set 6344 except Exception: # argh, try brute force method 6345 for record in self: 6346 if record == data: 6347 return True 6348 return False
6349
6350 - def __delitem__(self, key):
6351 self._still_valid_check() 6352 if isinstance(key, int): 6353 item = self._list.pop[key] 6354 self._set.remove(item[2]) 6355 elif isinstance(key, slice): 6356 self._set.difference_update([item[2] for item in self._list[key]]) 6357 self._list.__delitem__(key) 6358 elif isinstance(key, (Record, RecordTemplate, dict, tuple)): 6359 index = self.index(key) 6360 item = self._list.pop[index] 6361 self._set.remove(item[2]) 6362 else: 6363 raise TypeError('%r should be an int, slice, record, template, tuple, or dict -- not a %r' % (key, type(key)))
6364
6365 - def __getitem__(self, key):
6366 self._still_valid_check() 6367 if isinstance(key, int): 6368 count = len(self._list) 6369 if not -count <= key < count: 6370 raise NotFoundError("Record %d is not in list." % key) 6371 return self._get_record(*self._list[key]) 6372 elif isinstance(key, slice): 6373 result = self.__class__() 6374 result._list[:] = self._list[key] 6375 result._set = set(result._list) 6376 result.key = self.key 6377 return result 6378 elif isinstance(key, (Record, RecordTemplate, dict, tuple)): 6379 index = self.index(key) 6380 return self._get_record(*self._list[index]) 6381 else: 6382 raise TypeError('%r should be an int, slice, record, record template, tuple, or dict -- not a %r' % (key, type(key)))
6383
6384 - def __iter__(self):
6385 self._still_valid_check() 6386 return Iter(self)
6387
6388 - def __len__(self):
6389 self._still_valid_check() 6390 return len(self._list)
6391
6392 - def __nonzero__(self):
6393 self._still_valid_check() 6394 return len(self) > 0
6395
6396 - def __radd__(self, other):
6397 self._still_valid_check() 6398 key = self.key 6399 if isinstance(other, (Table, list)): 6400 other = self.__class__(other, key=key) 6401 if isinstance(other, self.__class__): 6402 other._still_valid_check() 6403 result = other.__class__() 6404 result._set = other._set.copy() 6405 result._list[:] = other._list[:] 6406 result._tables = {} 6407 result._tables.update(self._tables) 6408 result.key = other.key 6409 if key is other.key: # same key? just compare key values 6410 for item in self._list: 6411 result._maybe_add(item) 6412 else: # different keys, use this list's key on other's records 6413 for rec in self: 6414 result._maybe_add((source_table(rec), recno(rec), key(rec))) 6415 return result 6416 return NotImplemented
6417
6418 - def __repr__(self):
6419 self._still_valid_check() 6420 if self._desc: 6421 return "%s(key=(%s), desc=%s)" % (self.__class__, self.key.__doc__, self._desc) 6422 else: 6423 return "%s(key=(%s))" % (self.__class__, self.key.__doc__)
6424
6425 - def __rsub__(self, other):
6426 self._still_valid_check() 6427 key = self.key 6428 if isinstance(other, (Table, list)): 6429 other = self.__class__(other, key=key) 6430 if isinstance(other, self.__class__): 6431 other._still_valid_check() 6432 result = other.__class__() 6433 result._list[:] = other._list[:] 6434 result._set = other._set.copy() 6435 result._tables = {} 6436 result._tables.update(other._tables) 6437 result.key = key 6438 lost = set() 6439 if key is other.key: 6440 for item in self._list: 6441 if item[2] in result._list: 6442 result._set.remove(item[2]) 6443 lost.add(item) 6444 else: 6445 for rec in self: 6446 value = key(rec) 6447 if value in result._set: 6448 result._set.remove(value) 6449 lost.add((source_table(rec), recno(rec), value)) 6450 result._list = [item for item in result._list if item not in lost] 6451 lost = set(result._tables.keys()) 6452 for table, _1, _2 in result._list: 6453 if table in result._tables: 6454 lost.remove(table) 6455 if not lost: 6456 break 6457 for table in lost: 6458 del result._tables[table] 6459 return result 6460 return NotImplemented
6461
6462 - def __sub__(self, other):
6463 self._still_valid_check() 6464 key = self.key 6465 if isinstance(other, (Table, list)): 6466 other = self.__class__(other, key=key) 6467 if isinstance(other, self.__class__): 6468 other._still_valid_check() 6469 result = self.__class__() 6470 result._list[:] = self._list[:] 6471 result._set = self._set.copy() 6472 result._tables = {} 6473 result._tables.update(self._tables) 6474 result.key = key 6475 lost = set() 6476 if key is other.key: 6477 for item in other._list: 6478 if item[2] in result._set: 6479 result._set.remove(item[2]) 6480 lost.add(item[2]) 6481 else: 6482 for rec in other: 6483 value = key(rec) 6484 if value in result._set: 6485 result._set.remove(value) 6486 lost.add(value) 6487 result._list = [item for item in result._list if item[2] not in lost] 6488 lost = set(result._tables.keys()) 6489 for table, _1, _2 in result._list: 6490 if table in result._tables: 6491 lost.remove(table) 6492 if not lost: 6493 break 6494 for table in lost: 6495 del result._tables[table] 6496 return result 6497 return NotImplemented
6498
6499 - def _maybe_add(self, item):
6500 self._still_valid_check() 6501 table, recno, key = item 6502 self._tables[table] = table._pack_count # TODO: check that _pack_count is the same if already in table 6503 if key not in self._set: 6504 self._set.add(key) 6505 self._list.append(item)
6506
6507 - def _get_record(self, table=None, rec_no=None, value=None):
6508 if table is rec_no is None: 6509 table, rec_no, value = self._list[self._index] 6510 return table[rec_no]
6511
6512 - def _purge(self, record, old_record_number, offset):
6513 partial = source_table(record), old_record_number 6514 records = sorted(self._list, key=lambda item: (item[0], item[1])) 6515 for item in records: 6516 if partial == item[:2]: 6517 found = True 6518 break 6519 elif partial[0] is item[0] and partial[1] < item[1]: 6520 found = False 6521 break 6522 else: 6523 found = False 6524 if found: 6525 self._list.pop(self._list.index(item)) 6526 self._set.remove(item[2]) 6527 start = records.index(item) + found 6528 for item in records[start:]: 6529 if item[0] is not partial[0]: # into other table's records 6530 break 6531 i = self._list.index(item) 6532 self._set.remove(item[2]) 6533 item = item[0], (item[1] - offset), item[2] 6534 self._list[i] = item 6535 self._set.add(item[2]) 6536 return found
6537
6538 - def _still_valid_check(self):
6539 for table, last_pack in self._tables.items(): 6540 if last_pack != getattr(table, '_pack_count'): 6541 raise DbfError("table has been packed; list is invalid")
6542 6543 _nav_check = _still_valid_check 6544
6545 - def append(self, record):
6546 self._still_valid_check() 6547 self._maybe_add((source_table(record), recno(record), self.key(record)))
6548
6549 - def clear(self):
6550 self._list = [] 6551 self._set = set() 6552 self._index = -1 6553 self._tables.clear()
6554
6555 - def extend(self, records):
6556 self._still_valid_check() 6557 key = self.key 6558 if isinstance(records, self.__class__): 6559 if key is records.key: # same key? just compare key values 6560 for item in records._list: 6561 self._maybe_add(item) 6562 else: # different keys, use this list's key on other's records 6563 for rec in records: 6564 value = key(rec) 6565 self._maybe_add((source_table(rec), recno(rec), value)) 6566 else: 6567 for rec in records: 6568 value = key(rec) 6569 self._maybe_add((source_table(rec), recno(rec), value))
6570
6571 - def index(self, record, start=None, stop=None):
6572 """ 6573 returns the index of record between start and stop 6574 start and stop default to the first and last record 6575 """ 6576 if not isinstance(record, (Record, RecordTemplate, dict, tuple)): 6577 raise TypeError("x should be a record, template, dict, or tuple, not %r" % type(record)) 6578 self._still_valid_check() 6579 if start is None: 6580 start = 0 6581 if stop is None: 6582 stop = len(self) 6583 for i in range(start, stop): 6584 if record == (self[i]): 6585 return i 6586 else: 6587 raise NotFoundError("dbf.List.index(x): x not in List", data=record)
6588
6589 - def insert(self, i, record):
6590 self._still_valid_check() 6591 item = source_table(record), recno(record), self.key(record) 6592 if item not in self._set: 6593 self._set.add(item[2]) 6594 self._list.insert(i, item)
6595
6596 - def key(self, record):
6597 """ 6598 table_name, record_number 6599 """ 6600 self._still_valid_check() 6601 return source_table(record), recno(record)
6602
6603 - def pop(self, index=None):
6604 self._still_valid_check() 6605 if index is None: 6606 table, recno, value = self._list.pop() 6607 else: 6608 table, recno, value = self._list.pop(index) 6609 self._set.remove(value) 6610 return self._get_record(table, recno, value)
6611
6612 - def query(self, criteria):
6613 """ 6614 criteria is a callback that returns a truthy value for matching record 6615 """ 6616 return pql(self, criteria)
6617
6618 - def remove(self, data):
6619 self._still_valid_check() 6620 if not isinstance(data, (Record, RecordTemplate, dict, tuple)): 6621 raise TypeError("%r(%r) is not a record, template, tuple, nor dict" % (type(data), data)) 6622 index = self.index(data) 6623 record = self[index] 6624 item = source_table(record), recno(record), self.key(record) 6625 self._list.remove(item) 6626 self._set.remove(item[2])
6627
6628 - def reverse(self):
6629 self._still_valid_check() 6630 return self._list.reverse()
6631
6632 - def sort(self, key=None, reverse=False):
6633 self._still_valid_check() 6634 if key is None: 6635 return self._list.sort(reverse=reverse) 6636 return self._list.sort(key=lambda item: key(item[0][item[1]]), reverse=reverse)
6637
6638 6639 -class Index(_Navigation):
6640 """ 6641 non-persistent index for a table 6642 """ 6643
6644 - def __init__(self, table, key):
6645 self._table = table 6646 self._values = [] # ordered list of values 6647 self._rec_by_val = [] # matching record numbers 6648 self._records = {} # record numbers:values 6649 self.__doc__ = key.__doc__ or 'unknown' 6650 self._key = key 6651 self._previous_status = [] 6652 for record in table: 6653 value = key(record) 6654 if value is DoNotIndex: 6655 continue 6656 rec_num = recno(record) 6657 if not isinstance(value, tuple): 6658 value = (value, ) 6659 vindex = bisect_right(self._values, value) 6660 self._values.insert(vindex, value) 6661 self._rec_by_val.insert(vindex, rec_num) 6662 self._records[rec_num] = value 6663 table._indexen.add(self)
6664
6665 - def __call__(self, record):
6666 rec_num = recno(record) 6667 key = self.key(record) 6668 if rec_num in self._records: 6669 if self._records[rec_num] == key: 6670 return 6671 old_key = self._records[rec_num] 6672 vindex = bisect_left(self._values, old_key) 6673 self._values.pop(vindex) 6674 self._rec_by_val.pop(vindex) 6675 del self._records[rec_num] 6676 assert rec_num not in self._records 6677 if key == (DoNotIndex, ): 6678 return 6679 vindex = bisect_right(self._values, key) 6680 self._values.insert(vindex, key) 6681 self._rec_by_val.insert(vindex, rec_num) 6682 self._records[rec_num] = key
6683
6684 - def __contains__(self, data):
6685 if not isinstance(data, (Record, RecordTemplate, tuple, dict)): 6686 raise TypeError("%r is not a record, templace, tuple, nor dict" % (data, )) 6687 try: 6688 value = self.key(data) 6689 return value in self._values 6690 except Exception: 6691 for record in self: 6692 if record == data: 6693 return True 6694 return False
6695
6696 - def __getitem__(self, key):
6697 '''if key is an integer, returns the matching record; 6698 if key is a [slice | string | tuple | record] returns a List; 6699 raises NotFoundError on failure''' 6700 if isinstance(key, int): 6701 count = len(self._values) 6702 if not -count <= key < count: 6703 raise NotFoundError("Record %d is not in list." % key) 6704 rec_num = self._rec_by_val[key] 6705 return self._table[rec_num] 6706 elif isinstance(key, slice): 6707 result = List() 6708 start, stop, step = key.start, key.stop, key.step 6709 if start is None: start = 0 6710 if stop is None: stop = len(self._rec_by_val) 6711 if step is None: step = 1 6712 if step < 0: 6713 start, stop = stop - 1, -(stop - start + 1) 6714 for loc in range(start, stop, step): 6715 record = self._table[self._rec_by_val[loc]] 6716 result._maybe_add(item=(self._table, self._rec_by_val[loc], result.key(record))) 6717 return result 6718 elif isinstance (key, (basestring, tuple, Record, RecordTemplate)): 6719 if isinstance(key, (Record, RecordTemplate)): 6720 key = self.key(key) 6721 elif isinstance(key, basestring): 6722 key = (key, ) 6723 lo = self._search(key, where='left') 6724 hi = self._search(key, where='right') 6725 if lo == hi: 6726 raise NotFoundError(key) 6727 result = List(desc='match = %r' % (key, )) 6728 for loc in range(lo, hi): 6729 record = self._table[self._rec_by_val[loc]] 6730 result._maybe_add(item=(self._table, self._rec_by_val[loc], result.key(record))) 6731 return result 6732 else: 6733 raise TypeError('indices must be integers, match objects must by strings or tuples')
6734
6735 - def __enter__(self):
6736 self._table.__enter__() 6737 return self
6738
6739 - def __exit__(self, *exc_info):
6740 self._table.__exit__() 6741 return False
6742
6743 - def __iter__(self):
6744 return Iter(self)
6745
6746 - def __len__(self):
6747 return len(self._records)
6748
6749 - def _clear(self):
6750 """ 6751 removes all entries from index 6752 """ 6753 self._values[:] = [] 6754 self._rec_by_val[:] = [] 6755 self._records.clear()
6756
6757 - def _key(self, record):
6758 """ 6759 table_name, record_number 6760 """ 6761 self._still_valid_check() 6762 return source_table(record), recno(record)
6763
6764 - def _nav_check(self):
6765 """ 6766 raises error if table is closed 6767 """ 6768 if self._table._meta.status == CLOSED: 6769 raise DbfError('indexed table %s is closed' % self.filename)
6770
6771 - def _partial_match(self, target, match):
6772 target = target[:len(match)] 6773 if isinstance(match[-1], basestring): 6774 target = list(target) 6775 target[-1] = target[-1][:len(match[-1])] 6776 target = tuple(target) 6777 return target == match
6778
6779 - def _purge(self, rec_num):
6780 value = self._records.get(rec_num) 6781 if value is not None: 6782 vindex = bisect_left(self._values, value) 6783 del self._records[rec_num] 6784 self._values.pop(vindex) 6785 self._rec_by_val.pop(vindex)
6786
6787 - def _reindex(self):
6788 """ 6789 reindexes all records 6790 """ 6791 for record in self._table: 6792 self(record)
6793
6794 - def _search(self, match, lo=0, hi=None, where=None):
6795 if hi is None: 6796 hi = len(self._values) 6797 if where == 'left': 6798 return bisect_left(self._values, match, lo, hi) 6799 elif where == 'right': 6800 return bisect_right(self._values, match, lo, hi)
6801
6802 - def index(self, record, start=None, stop=None):
6803 """ 6804 returns the index of record between start and stop 6805 start and stop default to the first and last record 6806 """ 6807 if not isinstance(record, (Record, RecordTemplate, dict, tuple)): 6808 raise TypeError("x should be a record, template, dict, or tuple, not %r" % type(record)) 6809 self._nav_check() 6810 if start is None: 6811 start = 0 6812 if stop is None: 6813 stop = len(self) 6814 for i in range(start, stop): 6815 if record == (self[i]): 6816 return i 6817 else: 6818 raise NotFoundError("dbf.Index.index(x): x not in Index", data=record)
6819
6820 - def index_search(self, match, start=None, stop=None, nearest=False, partial=False):
6821 """ 6822 returns the index of match between start and stop 6823 start and stop default to the first and last record. 6824 if nearest is true returns the location of where the match should be 6825 otherwise raises NotFoundError 6826 """ 6827 self._nav_check() 6828 if not isinstance(match, tuple): 6829 match = (match, ) 6830 if start is None: 6831 start = 0 6832 if stop is None: 6833 stop = len(self) 6834 loc = self._search(match, start, stop, where='left') 6835 if loc == len(self._values): 6836 if nearest: 6837 return IndexLocation(loc, False) 6838 raise NotFoundError("dbf.Index.index_search(x): x not in index", data=match) 6839 if self._values[loc] == match \ 6840 or partial and self._partial_match(self._values[loc], match): 6841 return IndexLocation(loc, True) 6842 elif nearest: 6843 return IndexLocation(loc, False) 6844 else: 6845 raise NotFoundError("dbf.Index.index_search(x): x not in Index", data=match)
6846
6847 - def key(self, record):
6848 result = self._key(record) 6849 if not isinstance(result, tuple): 6850 result = (result, ) 6851 return result
6852
6853 - def query(self, criteria):
6854 """ 6855 criteria is a callback that returns a truthy value for matching record 6856 """ 6857 self._nav_check() 6858 return pql(self, criteria)
6859
6860 - def search(self, match, partial=False):
6861 """ 6862 returns dbf.List of all (partially) matching records 6863 """ 6864 self._nav_check() 6865 result = List() 6866 if not isinstance(match, tuple): 6867 match = (match, ) 6868 loc = self._search(match, where='left') 6869 if loc == len(self._values): 6870 return result 6871 while loc < len(self._values) and self._values[loc] == match: 6872 record = self._table[self._rec_by_val[loc]] 6873 result._maybe_add(item=(self._table, self._rec_by_val[loc], result.key(record))) 6874 loc += 1 6875 if partial: 6876 while loc < len(self._values) and self._partial_match(self._values[loc], match): 6877 record = self._table[self._rec_by_val[loc]] 6878 result._maybe_add(item=(self._table, self._rec_by_val[loc], result.key(record))) 6879 loc += 1 6880 return result
6881
6882 6883 -class Relation(object):
6884 """ 6885 establishes a relation between two dbf tables (not persistent) 6886 """ 6887 6888 relations = {} 6889
6890 - def __new__(cls, src, tgt, src_names=None, tgt_names=None):
6891 if (len(src) != 2 or len(tgt) != 2): 6892 raise DbfError("Relation should be called with ((src_table, src_field), (tgt_table, tgt_field))") 6893 if src_names and len(src_names) !=2 or tgt_names and len(tgt_names) != 2: 6894 raise DbfError('src_names and tgt_names, if specified, must be ("table","field")') 6895 src_table, src_field = src 6896 tgt_table, tgt_field = tgt 6897 try: 6898 if isinstance(src_field, baseinteger): 6899 table, field = src_table, src_field 6900 src_field = table.field_names[field] 6901 else: 6902 src_table.field_names.index(src_field) 6903 if isinstance(tgt_field, baseinteger): 6904 table, field = tgt_table, tgt_field 6905 tgt_field = table.field_names[field] 6906 else: 6907 tgt_table.field_names.index(tgt_field) 6908 except (IndexError, ValueError): 6909 raise DbfError('%r not in %r' % (field, table)) 6910 if src_names: 6911 src_table_name, src_field_name = src_names 6912 else: 6913 src_table_name, src_field_name = src_table.filename, src_field 6914 if src_table_name[-4:].lower() == '.dbf': 6915 src_table_name = src_table_name[:-4] 6916 if tgt_names: 6917 tgt_table_name, tgt_field_name = tgt_names 6918 else: 6919 tgt_table_name, tgt_field_name = tgt_table.filename, tgt_field 6920 if tgt_table_name[-4:].lower() == '.dbf': 6921 tgt_table_name = tgt_table_name[:-4] 6922 relation = cls.relations.get(((src_table, src_field), (tgt_table, tgt_field))) 6923 if relation is not None: 6924 return relation 6925 obj = object.__new__(cls) 6926 obj._src_table, obj._src_field = src_table, src_field 6927 obj._tgt_table, obj._tgt_field = tgt_table, tgt_field 6928 obj._src_table_name, obj._src_field_name = src_table_name, src_field_name 6929 obj._tgt_table_name, obj._tgt_field_name = tgt_table_name, tgt_field_name 6930 obj._tables = dict() 6931 cls.relations[((src_table, src_field), (tgt_table, tgt_field))] = obj 6932 return obj
6933
6934 - def __eq__(yo, other):
6935 if (yo.src_table == other.src_table 6936 and yo.src_field == other.src_field 6937 and yo.tgt_table == other.tgt_table 6938 and yo.tgt_field == other.tgt_field): 6939 return True 6940 return False
6941
6942 - def __getitem__(yo, record):
6943 """ 6944 record should be from the source table 6945 """ 6946 key = (record[yo.src_field], ) 6947 try: 6948 return yo.index[key] 6949 except NotFoundError: 6950 return List(desc='%s not found' % key)
6951
6952 - def __hash__(yo):
6953 return hash((yo.src_table, yo.src_field, yo.tgt_table, yo.tgt_field))
6954
6955 - def __ne__(yo, other):
6956 if (yo.src_table != other.src_table 6957 or yo.src_field != other.src_field 6958 or yo.tgt_table != other.tgt_table 6959 or yo.tgt_field != other.tgt_field): 6960 return True 6961 return False
6962
6963 - def __repr__(yo):
6964 return "Relation((%r, %r), (%r, %r))" % (yo.src_table_name, yo.src_field, yo.tgt_table_name, yo.tgt_field)
6965
6966 - def __str__(yo):
6967 return "%s:%s --> %s:%s" % (yo.src_table_name, yo.src_field_name, yo.tgt_table_name, yo.tgt_field_name)
6968 6969 @property
6970 - def src_table(yo):
6971 "name of source table" 6972 return yo._src_table
6973 6974 @property
6975 - def src_field(yo):
6976 "name of source field" 6977 return yo._src_field
6978 6979 @property
6980 - def src_table_name(yo):
6981 return yo._src_table_name
6982 6983 @property
6984 - def src_field_name(yo):
6985 return yo._src_field_name
6986 6987 @property
6988 - def tgt_table(yo):
6989 "name of target table" 6990 return yo._tgt_table
6991 6992 @property
6993 - def tgt_field(yo):
6994 "name of target field" 6995 return yo._tgt_field
6996 6997 @property
6998 - def tgt_table_name(yo):
6999 return yo._tgt_table_name
7000 7001 @property
7002 - def tgt_field_name(yo):
7003 return yo._tgt_field_name
7004 7005 @LazyAttr
7006 - def index(yo):
7007 def index(record, field=yo._tgt_field): 7008 return record[field]
7009 index.__doc__ = "%s:%s --> %s:%s" % (yo.src_table_name, yo.src_field_name, yo.tgt_table_name, yo.tgt_field_name) 7010 yo.index = yo._tgt_table.create_index(index) 7011 source = dbf.List(yo._src_table, key=lambda rec, field=yo._src_field: rec[field]) 7012 target = dbf.List(yo._tgt_table, key=lambda rec, field=yo._tgt_field: rec[field]) 7013 if len(source) != len(yo._src_table): 7014 yo._tables[yo._src_table] = 'many' 7015 else: 7016 yo._tables[yo._src_table] = 'one' 7017 if len(target) != len(yo._tgt_table): 7018 yo._tables[yo._tgt_table] = 'many' 7019 else: 7020 yo._tables[yo._tgt_table] = 'one' 7021 return yo.index
7022
7023 - def one_or_many(yo, table):
7024 yo.index # make sure yo._tables has been populated 7025 try: 7026 if isinstance(table, basestring): 7027 table = (yo._src_table, yo._tgt_table)[yo._tgt_table_name == table] 7028 return yo._tables[table] 7029 except IndexError: 7030 raise NotFoundError("table %s not in relation" % table)
7031
7032 7033 -class IndexFile(_Navigation):
7034 pass
7035
7036 -class BytesType(object):
7037
7038 - def __init__(self, offset):
7039 self.offset = offset
7040
7041 - def __get__(self, inst, cls=None):
7042 if inst is None: 7043 return self 7044 start = self.offset 7045 end = start + self.size 7046 byte_data = inst._data[start:end] 7047 return self.from_bytes(byte_data)
7048
7049 - def __set__(self, inst, value):
7050 start = self.offset 7051 end = start + self.size 7052 byte_data = self.to_bytes(value) 7053 inst._data = inst._data[:start] + byte_data + inst._data[end:]
7054
7055 7056 -class IntBytesType(BytesType):
7057 """ 7058 add big_endian and neg_one to __init__ 7059 """ 7060
7061 - def __init__(self, offset, big_endian=False, neg_one_is_none=False, one_based=False):
7062 self.offset = offset 7063 self.big_endian = big_endian 7064 self.neg_one_is_none = neg_one_is_none 7065 self.one_based = one_based
7066
7067 - def from_bytes(self, byte_data):
7068 if self.neg_one_is_none and byte_data == '\xff' * self.size: 7069 return None 7070 if self.big_endian: 7071 value = struct.unpack('>%s' % self.code, byte_data)[0] 7072 else: 7073 value = struct.unpack('<%s' % self.code, byte_data)[0] 7074 if self.one_based: 7075 # values are stored one based, convert to standard Python zero-base 7076 value -= 1 7077 return value
7078
7079 - def to_bytes(self, value):
7080 if value is None: 7081 if self.neg_one_is_none: 7082 return '\xff\xff' 7083 raise DbfError('unable to store None in %r' % self.__name__) 7084 limit = 2 ** (self.size * 8) - 1 7085 if self.one_based: 7086 limit -= 1 7087 if value > 2 ** limit: 7088 raise DataOverflowError("Maximum Integer size exceeded. Possible: %d. Attempted: %d" % (limit, value)) 7089 if self.one_based: 7090 value += 1 7091 if self.big_endian: 7092 return struct.pack('>%s' % self.code, value) 7093 else: 7094 return struct.pack('<%s' % self.code, value)
7095
7096 7097 -class Int8(IntBytesType):
7098 """ 7099 1-byte integer 7100 """ 7101 7102 size = 1 7103 code = 'B'
7104
7105 7106 -class Int16(IntBytesType):
7107 """ 7108 2-byte integer 7109 """ 7110 7111 size = 2 7112 code = 'H'
7113
7114 7115 -class Int32(IntBytesType):
7116 """ 7117 4-byte integer 7118 """ 7119 7120 size = 4 7121 code = 'L'
7122
7123 7124 -class Bytes(BytesType):
7125
7126 - def __init__(self, offset, size=0, fill_to=0, strip_null=False):
7127 if not (size or fill_to): 7128 raise DbfError("either size or fill_to must be specified") 7129 self.offset = offset 7130 self.size = size 7131 self.fill_to = fill_to 7132 self.strip_null = strip_null
7133
7134 - def from_bytes(self, byte_data):
7135 if self.strip_null: 7136 return byte_data.rstrip('\x00') 7137 else: 7138 return byte_data
7139
7140 - def to_bytes(self, value):
7141 if not isinstance(value, bytes): 7142 raise DbfError('value must be bytes [%r]' % value) 7143 if self.strip_null and len(value) < self.size: 7144 value += '\x00' * (self.size - len(value)) 7145 return value
7146
7147 7148 -class DataBlock(object):
7149 """ 7150 adds _data as a str to class 7151 binds variable name to BytesType descriptor 7152 """ 7153
7154 - def __init__(self, size):
7155 self.size = size
7156
7157 - def __call__(self, cls):
7158 fields = [] 7159 initialized = stringified = False 7160 for name, thing in cls.__dict__.items(): 7161 if isinstance(thing, BytesType): 7162 thing.__name__ = name 7163 fields.append((name, thing)) 7164 elif name in ('__init__', '__new__'): 7165 initialized = True 7166 elif name in ('__repr__', ): 7167 stringified = True 7168 fields.sort(key=lambda t: t[1].offset) 7169 for _, field in fields: 7170 offset = field.offset 7171 if not field.size: 7172 field.size = field.fill_to - offset 7173 total_field_size = field.offset + field.size 7174 if self.size and total_field_size > self.size: 7175 raise DbfError('Fields in %r are using %d bytes, but only %d allocated' % (cls, self.size)) 7176 total_field_size = self.size or total_field_size 7177 cls._data = str('\x00' * total_field_size) 7178 cls.__len__ = lambda s: len(s._data) 7179 cls._size_ = total_field_size 7180 if not initialized: 7181 def init(self, data): 7182 if len(data) != self._size_: 7183 raise Exception('%d bytes required, received %d' % (self._size_, len(data))) 7184 self._data = data
7185 cls.__init__ = init 7186 if not stringified: 7187 def repr(self): 7188 clauses = [] 7189 for name, _ in fields: 7190 value = getattr(self, name) 7191 if isinstance(value, str) and len(value) > 12: 7192 value = value[:9] + '...' 7193 clauses.append('%s=%r' % (name, value)) 7194 return ('%s(%s)' % (cls.__name__, ', '.join(clauses)))
7195 cls.__repr__ = repr 7196 return cls 7197
7198 7199 -class LruCache(object):
7200 """ 7201 keep the most recent n items in the dict 7202 7203 based on code from Raymond Hettinger: http://stackoverflow.com/a/8334739/208880 7204 """ 7205 7219
7220 - def __init__(self, maxsize, func=None):
7221 self.maxsize = maxsize 7222 self.mapping = {} 7223 self.tail = self.Link() # oldest 7224 self.head = self.Link(self.tail) # newest 7225 self.head.prev_link = self.tail 7226 self.func = func 7227 if func is not None: 7228 self.__name__ = func.__name__ 7229 self.__doc__ = func.__doc__
7230
7231 - def __call__(self, *func):
7232 if self.func is None: 7233 [self.func] = func 7234 self.__name__ = func.__name__ 7235 self.__doc__ = func.__doc__ 7236 return self 7237 mapping, head, tail = self.mapping, self.head, self.tail 7238 link = mapping.get(func, head) 7239 if link is head: 7240 value = self.func(*func) 7241 if len(mapping) >= self.maxsize: 7242 old_prev, old_next, old_key, old_value = tail.next_link 7243 tail.next_link = old_next 7244 old_next.prev_link = tail 7245 del mapping[old_key] 7246 behind = head.prev_link 7247 link = self.Link(behind, head, func, value) 7248 mapping[func] = behind.next_link = head.prev_link = link 7249 else: 7250 link_prev, link_next, func, value = link 7251 link_prev.next_link = link_next 7252 link_next.prev_link = link_prev 7253 behind = head.prev_link 7254 behind.next_link = head.prev_link = link 7255 link.prev_link = behind 7256 link.next_link = head 7257 return value
7258
7259 7260 -class Idx(object):
7261 # default numeric storage is little-endian 7262 # numbers used as key values, and the 4-byte numbers in leaf nodes are big-endian 7263 7264 @DataBlock(512)
7265 - class Header(object):
7266 root_node = Int32(0) 7267 free_node_list = Int32(4, neg_one_is_none=True) 7268 file_size = Int32(8) 7269 key_length = Int16(12) 7270 index_options = Int8(14) 7271 index_signature = Int8(15) 7272 key_expr = Bytes(16, 220, strip_null=True) 7273 for_expr = Bytes(236, 220, strip_null=True)
7274 7275 @DataBlock(512)
7276 - class Node(object):
7277 attributes = Int16(0) 7278 num_keys = Int16(2) 7279 left_peer = Int32(4, neg_one_is_none=True) 7280 right_peer = Int32(8, neg_one_is_none=True) 7281 pool = Bytes(12, fill_to=512)
7282 - def __init__(self, byte_data, node_key, record_key):
7283 if len(byte_data) != 512: 7284 raise DbfError("incomplete header: only received %d bytes" % len(byte_data)) 7285 self._data = byte_data 7286 self._node_key = node_key 7287 self._record_key = record_key
7288 - def is_leaf(self):
7289 return self.attributes in (2, 3)
7290 - def is_root(self):
7291 return self.attributes in (1, 3)
7292 - def is_interior(self):
7293 return self.attributes in (0, 1)
7294 - def keys(self):
7295 result = [] 7296 if self.is_leaf(): 7297 key = self._record_key 7298 else: 7299 key = self._node_key 7300 key_len = key._size_ 7301 for i in range(self.num_keys): 7302 start = i * key_len 7303 end = start + key_len 7304 result.append(key(self.pool[start:end])) 7305 return result
7306
7307 - def __init__(self, table, filename, size_limit=100):
7308 self.table = weakref.ref(table) 7309 self.filename = filename 7310 self.limit = size_limit 7311 with open(filename, 'rb') as idx: 7312 self.header = header = self.Header(idx.read(512)) 7313 offset = 512 7314 @DataBlock(header.key_length+4) 7315 class NodeKey(object): 7316 key = Bytes(0, header.key_length) 7317 rec_no = Int32(header.key_length, big_endian=True)
7318 @DataBlock(header.key_length+4) 7319 class RecordKey(object): 7320 key = Bytes(0, header.key_length) 7321 rec_no = Int32(header.key_length, big_endian=True, one_based=True)
7322 self.NodeKey = NodeKey 7323 self.RecordKey = RecordKey 7324 # set up root node 7325 idx.seek(header.root_node) 7326 self.root_node = self.Node(idx.read(512), self.NodeKey, self.RecordKey) 7327 # set up node reader 7328 self.read_node = LruCache(maxsize=size_limit, func=self.read_node) 7329 # set up iterating members 7330 self.current_node = None 7331 self.current_key = None 7332
7333 - def __iter__(self):
7334 # find the first leaf node 7335 table = self.table() 7336 if table is None: 7337 raise DbfError('the database linked to %r has been closed' % self.filename) 7338 node = self.root_node 7339 if not node.num_keys: 7340 yield 7341 return 7342 while "looking for a leaf": 7343 # travel the links down to the first leaf node 7344 if node.is_leaf(): 7345 break 7346 node = self.read_node(node.keys()[0].rec_no) 7347 while "traversing nodes": 7348 for key in node.keys(): 7349 yield table[key.rec_no] 7350 next_node = node.right_peer 7351 if next_node is None: 7352 return 7353 node = self.read_node(next_node)
7354 forward = __iter__ 7355
7356 - def read_node(self, offset):
7357 """ 7358 reads the sector indicated, and returns a Node object 7359 """ 7360 with open(self.filename, 'rb') as idx: 7361 idx.seek(offset) 7362 return self.Node(idx.read(512), self.NodeKey, self.RecordKey)
7363
7364 - def backward(self):
7365 # find the last leaf node 7366 table = self.table() 7367 if table is None: 7368 raise DbfError('the database linked to %r has been closed' % self.filename) 7369 node = self.root_node 7370 if not node.num_keys: 7371 yield 7372 return 7373 while "looking for last leaf": 7374 # travel the links down to the last leaf node 7375 if node.is_leaf(): 7376 break 7377 node = self.read_node(node.keys()[-1].rec_no) 7378 while "traversing nodes": 7379 for key in reversed(node.keys()): 7380 yield table[key.rec_no] 7381 prev_node = node.left_peer 7382 if prev_node is None: 7383 return 7384 node = self.read_node(prev_node)
7385 7386 7387 # table meta 7388 7389 table_types = { 7390 'db3' : Db3Table, 7391 'clp' : ClpTable, 7392 'fp' : FpTable, 7393 'vfp' : VfpTable, 7394 } 7395 7396 version_map = { 7397 '\x02' : 'FoxBASE', 7398 '\x03' : 'dBase III Plus', 7399 '\x04' : 'dBase IV', 7400 '\x05' : 'dBase V', 7401 '\x30' : 'Visual FoxPro', 7402 '\x31' : 'Visual FoxPro (auto increment field)', 7403 '\x32' : 'Visual FoxPro (VarChar, VarBinary, or BLOB enabled)', 7404 '\x43' : 'dBase IV SQL table files', 7405 '\x63' : 'dBase IV SQL system files', 7406 '\x83' : 'dBase III Plus w/memos', 7407 '\x8b' : 'dBase IV w/memos', 7408 '\x8e' : 'dBase IV w/SQL table', 7409 '\xf5' : 'FoxPro w/memos'} 7410 7411 code_pages = { 7412 '\x00' : ('ascii', "plain ol' ascii"), 7413 '\x01' : ('cp437', 'U.S. MS-DOS'), 7414 '\x02' : ('cp850', 'International MS-DOS'), 7415 '\x03' : ('cp1252', 'Windows ANSI'), 7416 '\x04' : ('mac_roman', 'Standard Macintosh'), 7417 '\x08' : ('cp865', 'Danish OEM'), 7418 '\x09' : ('cp437', 'Dutch OEM'), 7419 '\x0A' : ('cp850', 'Dutch OEM (secondary)'), 7420 '\x0B' : ('cp437', 'Finnish OEM'), 7421 '\x0D' : ('cp437', 'French OEM'), 7422 '\x0E' : ('cp850', 'French OEM (secondary)'), 7423 '\x0F' : ('cp437', 'German OEM'), 7424 '\x10' : ('cp850', 'German OEM (secondary)'), 7425 '\x11' : ('cp437', 'Italian OEM'), 7426 '\x12' : ('cp850', 'Italian OEM (secondary)'), 7427 '\x13' : ('cp932', 'Japanese Shift-JIS'), 7428 '\x14' : ('cp850', 'Spanish OEM (secondary)'), 7429 '\x15' : ('cp437', 'Swedish OEM'), 7430 '\x16' : ('cp850', 'Swedish OEM (secondary)'), 7431 '\x17' : ('cp865', 'Norwegian OEM'), 7432 '\x18' : ('cp437', 'Spanish OEM'), 7433 '\x19' : ('cp437', 'English OEM (Britain)'), 7434 '\x1A' : ('cp850', 'English OEM (Britain) (secondary)'), 7435 '\x1B' : ('cp437', 'English OEM (U.S.)'), 7436 '\x1C' : ('cp863', 'French OEM (Canada)'), 7437 '\x1D' : ('cp850', 'French OEM (secondary)'), 7438 '\x1F' : ('cp852', 'Czech OEM'), 7439 '\x22' : ('cp852', 'Hungarian OEM'), 7440 '\x23' : ('cp852', 'Polish OEM'), 7441 '\x24' : ('cp860', 'Portugese OEM'), 7442 '\x25' : ('cp850', 'Potugese OEM (secondary)'), 7443 '\x26' : ('cp866', 'Russian OEM'), 7444 '\x37' : ('cp850', 'English OEM (U.S.) (secondary)'), 7445 '\x40' : ('cp852', 'Romanian OEM'), 7446 '\x4D' : ('cp936', 'Chinese GBK (PRC)'), 7447 '\x4E' : ('cp949', 'Korean (ANSI/OEM)'), 7448 '\x4F' : ('cp950', 'Chinese Big 5 (Taiwan)'), 7449 '\x50' : ('cp874', 'Thai (ANSI/OEM)'), 7450 '\x57' : ('cp1252', 'ANSI'), 7451 '\x58' : ('cp1252', 'Western European ANSI'), 7452 '\x59' : ('cp1252', 'Spanish ANSI'), 7453 '\x64' : ('cp852', 'Eastern European MS-DOS'), 7454 '\x65' : ('cp866', 'Russian MS-DOS'), 7455 '\x66' : ('cp865', 'Nordic MS-DOS'), 7456 '\x67' : ('cp861', 'Icelandic MS-DOS'), 7457 '\x68' : (None, 'Kamenicky (Czech) MS-DOS'), 7458 '\x69' : (None, 'Mazovia (Polish) MS-DOS'), 7459 '\x6a' : ('cp737', 'Greek MS-DOS (437G)'), 7460 '\x6b' : ('cp857', 'Turkish MS-DOS'), 7461 '\x78' : ('cp950', 'Traditional Chinese (Hong Kong SAR, Taiwan) Windows'), 7462 '\x79' : ('cp949', 'Korean Windows'), 7463 '\x7a' : ('cp936', 'Chinese Simplified (PRC, Singapore) Windows'), 7464 '\x7b' : ('cp932', 'Japanese Windows'), 7465 '\x7c' : ('cp874', 'Thai Windows'), 7466 '\x7d' : ('cp1255', 'Hebrew Windows'), 7467 '\x7e' : ('cp1256', 'Arabic Windows'), 7468 '\xc8' : ('cp1250', 'Eastern European Windows'), 7469 '\xc9' : ('cp1251', 'Russian Windows'), 7470 '\xca' : ('cp1254', 'Turkish Windows'), 7471 '\xcb' : ('cp1253', 'Greek Windows'), 7472 '\x96' : ('mac_cyrillic', 'Russian Macintosh'), 7473 '\x97' : ('mac_latin2', 'Macintosh EE'), 7474 '\x98' : ('mac_greek', 'Greek Macintosh'), 7475 '\xf0' : ('utf8', '8-bit unicode'), 7476 } 7477 7478 7479 default_codepage = code_pages.get(default_codepage, code_pages.get('\x00'))[0]
7480 7481 7482 # SQL functions 7483 7484 -def pql_select(records, chosen_fields, condition, field_names):
7485 if chosen_fields != '*': 7486 field_names = chosen_fields.replace(' ', '').split(',') 7487 result = condition(records) 7488 result.modified = 0, 'record' + ('', 's')[len(result)>1] 7489 result.field_names = field_names 7490 return result
7491
7492 -def pql_update(records, command, condition, field_names):
7493 possible = condition(records) 7494 modified = pql_cmd(command, field_names)(possible) 7495 possible.modified = modified, 'record' + ('', 's')[modified>1] 7496 return possible
7497
7498 -def pql_delete(records, dead_fields, condition, field_names):
7499 deleted = condition(records) 7500 deleted.modified = len(deleted), 'record' + ('', 's')[len(deleted)>1] 7501 deleted.field_names = field_names 7502 if dead_fields == '*': 7503 for record in deleted: 7504 record.delete_record() 7505 record.write_record() 7506 else: 7507 keep = [f for f in field_names if f not in dead_fields.replace(' ', '').split(',')] 7508 for record in deleted: 7509 record.reset_record(keep_fields=keep) 7510 record.write_record() 7511 return deleted
7512
7513 -def pql_recall(records, all_fields, condition, field_names):
7514 if all_fields != '*': 7515 raise DbfError('SQL RECALL: fields must be * (only able to recover at the record level)') 7516 revivified = List() 7517 for record in condition(records): 7518 if is_deleted(record): 7519 revivified.append(record) 7520 undelete(record) 7521 revivified.modfied = len(revivified), 'record' + ('', 's')[len(revivified)>1] 7522 return revivified
7523
7524 -def pql_add(records, new_fields, condition, field_names):
7525 tables = set() 7526 possible = condition(records) 7527 for record in possible: 7528 tables.add(source_table(record)) 7529 for table in tables: 7530 table.add_fields(new_fields) 7531 possible.modified = len(tables), 'table' + ('', 's')[len(tables)>1] 7532 possible.field_names = field_names 7533 return possible
7534
7535 -def pql_drop(records, dead_fields, condition, field_names):
7536 tables = set() 7537 possible = condition(records) 7538 for record in possible: 7539 tables.add(source_table(record)) 7540 for table in tables: 7541 table.delete_fields(dead_fields) 7542 possible.modified = len(tables), 'table' + ('', 's')[len(tables)>1] 7543 possible.field_names = field_names 7544 return possible
7545
7546 -def pql_pack(records, command, condition, field_names):
7547 tables = set() 7548 possible = condition(records) 7549 for record in possible: 7550 tables.add(source_table(record)) 7551 for table in tables: 7552 table.pack() 7553 possible.modified = len(tables), 'table' + ('', 's')[len(tables)>1] 7554 possible.field_names = field_names 7555 return possible
7556
7557 -def pql_resize(records, fieldname_newsize, condition, field_names):
7558 tables = set() 7559 possible = condition(records) 7560 for record in possible: 7561 tables.add(source_table(record)) 7562 fieldname, newsize = fieldname_newsize.split() 7563 newsize = int(newsize) 7564 for table in tables: 7565 table.resize_field(fieldname, newsize) 7566 possible.modified = len(tables), 'table' + ('', 's')[len(tables)>1] 7567 possible.field_names = field_names 7568 return possible
7569
7570 -def pql_criteria(records, criteria):
7571 """ 7572 creates a function matching the pql criteria 7573 """ 7574 function = """def func(records): 7575 '''%s 7576 ''' 7577 _matched = dbf.List() 7578 for _rec in records: 7579 %s 7580 7581 if %s: 7582 _matched.append(_rec) 7583 return _matched""" 7584 fields = [] 7585 for field in field_names(records): 7586 if field in criteria: 7587 fields.append(field) 7588 criteria = criteria.replace('recno()', 'recno(_rec)').replace('is_deleted()', 'is_deleted(_rec)') 7589 fields = '\n '.join(['%s = _rec.%s' % (field, field) for field in fields]) 7590 g = dict() 7591 g['dbf'] = dbf 7592 g.update(pql_user_functions) 7593 function %= (criteria, fields, criteria) 7594 exec function in g 7595 return g['func']
7596
7597 -def pql_cmd(command, field_names):
7598 """ 7599 creates a function matching to apply command to each record in records 7600 """ 7601 function = """def func(records): 7602 '''%s 7603 ''' 7604 _changed = 0 7605 for _rec in records: 7606 _tmp = dbf.create_template(_rec) 7607 %s 7608 7609 %s 7610 7611 %s 7612 if _tmp != _rec: 7613 dbf.gather(_rec, _tmp) 7614 _changed += 1 7615 return _changed""" 7616 fields = [] 7617 for field in field_names: 7618 if field in command: 7619 fields.append(field) 7620 command = command.replace('recno()', 'recno(_rec)').replace('is_deleted()', 'is_deleted(_rec)') 7621 pre_fields = '\n '.join(['%s = _tmp.%s' % (field, field) for field in fields]) 7622 post_fields = '\n '.join(['_tmp.%s = %s' % (field, field) for field in fields]) 7623 g = pql_user_functions.copy() 7624 g['dbf'] = dbf 7625 g['recno'] = recno 7626 g['create_template'] = create_template 7627 g['gather'] = gather 7628 if ' with ' in command.lower(): 7629 offset = command.lower().index(' with ') 7630 command = command[:offset] + ' = ' + command[offset + 6:] 7631 function %= (command, pre_fields, command, post_fields) 7632 exec function in g 7633 return g['func']
7634
7635 -def pql(records, command):
7636 """ 7637 recognized pql commands are SELECT, UPDATE | REPLACE, DELETE, RECALL, ADD, DROP 7638 """ 7639 close_table = False 7640 if isinstance(records, basestring): 7641 records = Table(records) 7642 close_table = True 7643 try: 7644 if not records: 7645 return List() 7646 pql_command = command 7647 if ' where ' in command: 7648 command, condition = command.split(' where ', 1) 7649 condition = pql_criteria(records, condition) 7650 else: 7651 def condition(records): 7652 return records[:]
7653 name, command = command.split(' ', 1) 7654 command = command.strip() 7655 name = name.lower() 7656 fields = field_names(records) 7657 if pql_functions.get(name) is None: 7658 raise DbfError('unknown SQL command: %s' % name.upper()) 7659 result = pql_functions[name](records, command, condition, fields) 7660 tables = set() 7661 for record in result: 7662 tables.add(source_table(record)) 7663 finally: 7664 if close_table: 7665 records.close() 7666 return result 7667 7668 pql_functions = { 7669 'select' : pql_select, 7670 'update' : pql_update, 7671 'replace': pql_update, 7672 'insert' : None, 7673 'delete' : pql_delete, 7674 'recall' : pql_recall, 7675 'add' : pql_add, 7676 'drop' : pql_drop, 7677 'count' : None, 7678 'pack' : pql_pack, 7679 'resize' : pql_resize, 7680 }
7681 7682 7683 -def _nop(value):
7684 """ 7685 returns parameter unchanged 7686 """ 7687 return value
7688
7689 -def _normalize_tuples(tuples, length, filler):
7690 """ 7691 ensures each tuple is the same length, using filler[-missing] for the gaps 7692 """ 7693 final = [] 7694 for t in tuples: 7695 if len(t) < length: 7696 final.append( tuple([item for item in t] + filler[len(t)-length:]) ) 7697 else: 7698 final.append(t) 7699 return tuple(final)
7700
7701 -def _codepage_lookup(cp):
7702 if cp not in code_pages: 7703 for code_page in sorted(code_pages.keys()): 7704 sd, ld = code_pages[code_page] 7705 if cp == sd or cp == ld: 7706 if sd is None: 7707 raise DbfError("Unsupported codepage: %s" % ld) 7708 cp = code_page 7709 break 7710 else: 7711 raise DbfError("Unsupported codepage: %s" % cp) 7712 sd, ld = code_pages[cp] 7713 return cp, sd, ld
7714
7715 7716 # miscellany 7717 7718 -class _Db4Table(Table):
7719 """ 7720 under development 7721 """ 7722 7723 version = 'dBase IV w/memos (non-functional)' 7724 _versionabbr = 'db4' 7725 7726 @MutableDefault
7727 - def _field_types():
7728 return { 7729 'C' : {'Type':'Character', 'Retrieve':retrieve_character, 'Update':update_character, 'Blank':str, 'Init':add_vfp_character}, 7730 'Y' : {'Type':'Currency', 'Retrieve':retrieve_currency, 'Update':update_currency, 'Blank':Decimal, 'Init':add_vfp_currency}, 7731 'B' : {'Type':'Double', 'Retrieve':retrieve_double, 'Update':update_double, 'Blank':float, 'Init':add_vfp_double}, 7732 'F' : {'Type':'Float', 'Retrieve':retrieve_numeric, 'Update':update_numeric, 'Blank':float, 'Init':add_vfp_numeric}, 7733 'N' : {'Type':'Numeric', 'Retrieve':retrieve_numeric, 'Update':update_numeric, 'Blank':int, 'Init':add_vfp_numeric}, 7734 'I' : {'Type':'Integer', 'Retrieve':retrieve_integer, 'Update':update_integer, 'Blank':int, 'Init':add_vfp_integer}, 7735 'L' : {'Type':'Logical', 'Retrieve':retrieve_logical, 'Update':update_logical, 'Blank':Logical, 'Init':add_logical}, 7736 'D' : {'Type':'Date', 'Retrieve':retrieve_date, 'Update':update_date, 'Blank':Date, 'Init':add_date}, 7737 'T' : {'Type':'DateTime', 'Retrieve':retrieve_vfp_datetime, 'Update':update_vfp_datetime, 'Blank':DateTime, 'Init':add_vfp_datetime}, 7738 'M' : {'Type':'Memo', 'Retrieve':retrieve_memo, 'Update':update_memo, 'Blank':str, 'Init':add_memo}, 7739 'G' : {'Type':'General', 'Retrieve':retrieve_memo, 'Update':update_memo, 'Blank':str, 'Init':add_memo}, 7740 'P' : {'Type':'Picture', 'Retrieve':retrieve_memo, 'Update':update_memo, 'Blank':str, 'Init':add_memo}, 7741 '0' : {'Type':'_NullFlags', 'Retrieve':unsupported_type, 'Update':unsupported_type, 'Blank':int, 'Init':None} }
7742 7743 _memoext = '.dbt' 7744 _memotypes = ('G', 'M', 'P') 7745 _memoClass = _VfpMemo 7746 _yesMemoMask = '\x8b' # 0011 0000 7747 _noMemoMask = '\x04' # 0011 0000 7748 _fixed_fields = ('B', 'D', 'G', 'I', 'L', 'M', 'P', 'T', 'Y') 7749 _variable_fields = ('C', 'F', 'N') 7750 _binary_fields = ('G', 'P') 7751 _character_fields = ('C', 'M') # field representing character data 7752 _decimal_fields = ('F', 'N') 7753 _numeric_fields = ('B', 'F', 'I', 'N', 'Y') 7754 _currency_fields = ('Y',) 7755 _supported_tables = ('\x04', '\x8b') 7756 _dbfTableHeader = ['\x00'] * 32 7757 _dbfTableHeader[0] = '\x8b' # version - Foxpro 6 0011 0000 7758 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 7759 _dbfTableHeader[29] = '\x03' # code page -- 437 US-MS DOS 7760 _dbfTableHeader = ''.join(_dbfTableHeader) 7761 _dbfTableHeaderExtra = '' 7762
7763 - def _check_memo_integrity(self):
7764 """ 7765 dBase IV specific 7766 """ 7767 if self._meta.header.version == '\x8b': 7768 try: 7769 self._meta.memo = self._memoClass(self._meta) 7770 except: 7771 self._meta.dfd.close() 7772 self._meta.dfd = None 7773 raise 7774 if not self._meta.ignorememos: 7775 for field in self._meta.fields: 7776 if self._meta[field][TYPE] in self._memotypes: 7777 if self._meta.header.version != '\x8b': 7778 self._meta.dfd.close() 7779 self._meta.dfd = None 7780 raise BadDataError("Table structure corrupt: memo fields exist, header declares no memos") 7781 elif not os.path.exists(self._meta.memoname): 7782 self._meta.dfd.close() 7783 self._meta.dfd = None 7784 raise BadDataError("Table structure corrupt: memo fields exist without memo file") 7785 break
7786
7787 7788 # utility functions 7789 7790 -def create_template(table_or_record, defaults=None):
7791 if isinstance(table_or_record, Table): 7792 return RecordTemplate(table_or_record._meta, defaults) 7793 else: 7794 return RecordTemplate(table_or_record._meta, table_or_record, defaults)
7795
7796 -def delete(record):
7797 """ 7798 marks record as deleted 7799 """ 7800 template = isinstance(record, RecordTemplate) 7801 if not template and record._meta.status == CLOSED: 7802 raise DbfError("%s is closed; cannot delete record" % record._meta.filename) 7803 record_in_flux = not record._write_to_disk 7804 if not template and not record_in_flux: 7805 record._start_flux() 7806 try: 7807 record._data[0] = '*' 7808 if not template: 7809 record._dirty = True 7810 except: 7811 if not template and not record_in_flux: 7812 record._rollback_flux() 7813 raise 7814 if not template and not record_in_flux: 7815 record._commit_flux()
7816
7817 -def export(table_or_records, filename=None, field_names=None, format='csv', header=True, dialect='dbf', encoding=None):
7818 """ 7819 writes the records using CSV or tab-delimited format, using the filename 7820 given if specified, otherwise the table name 7821 if table_or_records is a collection of records (not an actual table) they 7822 should all be of the same format 7823 """ 7824 table = source_table(table_or_records[0]) 7825 if filename is None: 7826 filename = table.filename 7827 if field_names is None: 7828 field_names = table.field_names 7829 if isinstance(field_names, basestring): 7830 field_names = [f.strip() for f in field_names.split(',')] 7831 format = format.lower() 7832 if format not in ('csv', 'tab', 'fixed'): 7833 raise DbfError("export format: csv, tab, or fixed -- not %s" % format) 7834 if format == 'fixed': 7835 format = 'txt' 7836 if encoding is None: 7837 encoding = table.codepage.name 7838 encoder = codecs.getencoder(encoding) 7839 if isinstance(field_names[0], unicode): 7840 header_names = [encoder(f) for f in field_names] 7841 else: 7842 header_names = field_names 7843 base, ext = os.path.splitext(filename) 7844 if ext.lower() in ('', '.dbf'): 7845 filename = base + "." + format 7846 try: 7847 if format == 'csv': 7848 fd = open(filename, 'wb') 7849 csvfile = csv.writer(fd, dialect=dialect) 7850 if header: 7851 csvfile.writerow(header_names) 7852 for record in table_or_records: 7853 fields = [] 7854 for fieldname in field_names: 7855 data = record[fieldname] 7856 if isinstance(data, unicode): 7857 fields.append(encoder(data)[0]) 7858 else: 7859 fields.append(data) 7860 csvfile.writerow(fields) 7861 elif format == 'tab': 7862 fd = open(filename, 'w') 7863 if header: 7864 fd.write('\t'.join(header_names) + '\n') 7865 for record in table_or_records: 7866 fields = [] 7867 for fieldname in field_names: 7868 data = record[fieldname] 7869 if isinstance(data, unicode): 7870 fields.append(encoder(data)[0]) 7871 else: 7872 fields.append(str(data)) 7873 fd.write('\t'.join(fields) + '\n') 7874 else: # format == 'fixed' 7875 fd = open(filename, 'w') 7876 header = open("%s_layout.txt" % os.path.splitext(filename)[0], 'w') 7877 header.write("%-15s Size\n" % "Field Name") 7878 header.write("%-15s ----\n" % ("-" * 15)) 7879 sizes = [] 7880 for field in field_names: 7881 size = table.field_info(field).length 7882 sizes.append(size) 7883 field = encoder(field)[0] 7884 header.write("%-15s %3d\n" % (field, size)) 7885 header.write('\nTotal Records in file: %d\n' % len(table_or_records)) 7886 header.close() 7887 for record in table_or_records: 7888 fields = [] 7889 for i, fieldname in enumerate(field_names): 7890 data = record[fieldname] 7891 if isinstance(data, unicode): 7892 fields.append("%-*s" % (sizes[i], encoder(data)[0])) 7893 else: 7894 fields.append("%-*s" % (sizes[i], data)) 7895 fd.write(''.join(fields) + '\n') 7896 finally: 7897 fd.close() 7898 fd = None 7899 return len(table_or_records)
7900
7901 -def field_names(thing):
7902 """ 7903 fields in table/record, keys in dict 7904 """ 7905 if isinstance(thing, dict): 7906 return thing.keys() 7907 elif isinstance(thing, (Table, Record, RecordTemplate)): 7908 return thing._meta.user_fields[:] 7909 elif isinstance(thing, Index): 7910 return thing._table._meta.user_fields[:] 7911 else: 7912 for record in thing: # grab any record 7913 return record._meta.user_fields[:]
7914
7915 -def is_deleted(record):
7916 """ 7917 marked for deletion? 7918 """ 7919 return record._data[0] == '*'
7920
7921 -def recno(record):
7922 """ 7923 physical record number 7924 """ 7925 return record._recnum
7926
7927 -def reset(record, keep_fields=None):
7928 """ 7929 sets record's fields back to original, except for fields in keep_fields 7930 """ 7931 template = record_in_flux = False 7932 if isinstance(record, RecordTemplate): 7933 template = True 7934 else: 7935 record_in_flux = not record._write_to_disk 7936 if record._meta.status == CLOSED: 7937 raise DbfError("%s is closed; cannot modify record" % record._meta.filename) 7938 if keep_fields is None: 7939 keep_fields = [] 7940 keep = {} 7941 for field in keep_fields: 7942 keep[field] = record[field] 7943 record._data[:] = record._meta.blankrecord[:] 7944 for field in keep_fields: 7945 record[field] = keep[field] 7946 if not template: 7947 if record._write_to_disk: 7948 record._write() 7949 else: 7950 record._dirty = True
7951
7952 -def source_table(thingie):
7953 """ 7954 table associated with table | record | index 7955 """ 7956 table = thingie._meta.table() 7957 if table is None: 7958 raise DbfError("table is no longer available") 7959 return table
7960
7961 -def undelete(record):
7962 """ 7963 marks record as active 7964 """ 7965 template = isinstance(record, RecordTemplate) 7966 if not template and record._meta.status == CLOSED: 7967 raise DbfError("%s is closed; cannot undelete record" % record._meta.filename) 7968 record_in_flux = not record._write_to_disk 7969 if not template and not record_in_flux: 7970 record._start_flux() 7971 try: 7972 record._data[0] = ' ' 7973 if not template: 7974 record._dirty = True 7975 except: 7976 if not template and not record_in_flux: 7977 record._rollback_flux() 7978 raise 7979 if not template and not record_in_flux: 7980 record._commit_flux()
7981 -def write(record, **kwargs):
7982 """ 7983 write record data to disk (updates indices) 7984 """ 7985 if record._meta.status == CLOSED: 7986 raise DbfError("%s is closed; cannot update record" % record._meta.filename) 7987 elif not record._write_to_disk: 7988 raise DbfError("unable to use .write_record() while record is in flux") 7989 if kwargs: 7990 gather(record, kwargs) 7991 if record._dirty: 7992 record._write()
7993
7994 -def Process(records, start=0, stop=None, filter=None):
7995 """commits each record to disk before returning the next one; undoes all changes to that record if exception raised 7996 if records is a table, it will be opened and closed if necessary 7997 filter function should return True to skip record, False to keep""" 7998 already_open = True 7999 if isinstance(records, Table): 8000 already_open = records.status != CLOSED 8001 if not already_open: 8002 records.open() 8003 try: 8004 if stop is None: 8005 stop = len(records) 8006 for record in records[start:stop]: 8007 if filter is not None and filter(record): 8008 continue 8009 try: 8010 record._start_flux() 8011 yield record 8012 except: 8013 record._rollback_flux() 8014 raise 8015 else: 8016 record._commit_flux() 8017 finally: 8018 if not already_open: 8019 records.close()
8020
8021 -def Templates(records, start=0, stop=None, filter=None):
8022 """ 8023 returns a template of each record instead of the record itself 8024 if records is a table, it will be opened and closed if necessary 8025 """ 8026 already_open = True 8027 if isinstance(records, Table): 8028 already_open = records.status != CLOSED 8029 if not already_open: 8030 records.open() 8031 try: 8032 if stop is None: 8033 stop = len(records) 8034 for record in records[start:stop]: 8035 if filter is not None and filter(record): 8036 continue 8037 yield(create_template(record)) 8038 finally: 8039 if not already_open: 8040 records.close()
8041
8042 -def index(sequence):
8043 """ 8044 returns integers 0 - len(sequence) 8045 """ 8046 for i in xrange(len(sequence)): 8047 yield i
8048
8049 -def guess_table_type(filename):
8050 reported = table_type(filename) 8051 possibles = [] 8052 version = reported[0] 8053 for tabletype in (Db3Table, ClpTable, FpTable, VfpTable): 8054 if version in tabletype._supported_tables: 8055 possibles.append((tabletype._versionabbr, tabletype._version, tabletype)) 8056 if not possibles: 8057 raise DbfError("Tables of type %s not supported" % str(reported)) 8058 return possibles
8059
8060 -def table_type(filename):
8061 """ 8062 returns text representation of a table's dbf version 8063 """ 8064 base, ext = os.path.splitext(filename) 8065 if ext == '': 8066 filename = base + '.[Dd][Bb][Ff]' 8067 matches = glob(filename) 8068 if matches: 8069 filename = matches[0] 8070 else: 8071 filename = base + '.dbf' 8072 if not os.path.exists(filename): 8073 raise DbfError('File %s not found' % filename) 8074 fd = open(filename) 8075 version = fd.read(1) 8076 fd.close() 8077 fd = None 8078 if not version in version_map: 8079 raise DbfError("Unknown dbf type: %s (%x)" % (version, ord(version))) 8080 return version, version_map[version]
8081
8082 -def add_fields(table_name, field_specs):
8083 """ 8084 adds fields to an existing table 8085 """ 8086 table = Table(table_name) 8087 table.open() 8088 try: 8089 table.add_fields(field_specs) 8090 finally: 8091 table.close()
8092
8093 -def delete_fields(table_name, field_names):
8094 """ 8095 deletes fields from an existing table 8096 """ 8097 table = Table(table_name) 8098 table.open() 8099 try: 8100 table.delete_fields(field_names) 8101 finally: 8102 table.close()
8103
8104 -def first_record(table_name):
8105 """ 8106 prints the first record of a table 8107 """ 8108 table = Table(table_name) 8109 table.open() 8110 try: 8111 print(str(table[0])) 8112 finally: 8113 table.close()
8114
8115 -def from_csv(csvfile, to_disk=False, filename=None, field_names=None, extra_fields=None, 8116 dbf_type='db3', memo_size=64, min_field_size=1, 8117 encoding=None, errors=None):
8118 """ 8119 creates a Character table from a csv file 8120 to_disk will create a table with the same name 8121 filename will be used if provided 8122 field_names default to f0, f1, f2, etc, unless specified (list) 8123 extra_fields can be used to add additional fields -- should be normal field specifiers (list) 8124 """ 8125 with codecs.open(csvfile, 'r', encoding='latin-1', errors=errors) as fd: 8126 reader = csv.reader(fd) 8127 if field_names: 8128 if isinstance(field_names, basestring): 8129 field_names = field_names.split() 8130 if ' ' not in field_names[0]: 8131 field_names = ['%s M' % fn for fn in field_names] 8132 else: 8133 field_names = ['f0 M'] 8134 mtable = Table(':memory:', [field_names[0]], dbf_type=dbf_type, memo_size=memo_size, codepage=encoding, on_disk=False) 8135 mtable.open() 8136 fields_so_far = 1 8137 #for row in reader: 8138 while reader: 8139 try: 8140 row = next(reader) 8141 except UnicodeEncodeError: 8142 row = [''] 8143 except StopIteration: 8144 break 8145 while fields_so_far < len(row): 8146 if fields_so_far == len(field_names): 8147 field_names.append('f%d M' % fields_so_far) 8148 mtable.add_fields(field_names[fields_so_far]) 8149 fields_so_far += 1 8150 mtable.append(tuple(row)) 8151 if filename: 8152 to_disk = True 8153 if not to_disk: 8154 if extra_fields: 8155 mtable.add_fields(extra_fields) 8156 else: 8157 if not filename: 8158 filename = os.path.splitext(csvfile)[0] 8159 length = [min_field_size] * len(field_names) 8160 for record in mtable: 8161 for i in index(mtable.field_names): 8162 length[i] = max(length[i], len(record[i])) 8163 fields = mtable.field_names 8164 fielddef = [] 8165 for i in index(length): 8166 if length[i] < 255: 8167 fielddef.append('%s C(%d)' % (fields[i], length[i])) 8168 else: 8169 fielddef.append('%s M' % (fields[i])) 8170 if extra_fields: 8171 fielddef.extend(extra_fields) 8172 csvtable = Table(filename, fielddef, dbf_type=dbf_type, codepage=encoding) 8173 csvtable.open() 8174 for record in mtable: 8175 csvtable.append(scatter(record)) 8176 csvtable.close() 8177 return csvtable 8178 mtable.close() 8179 return mtable
8180
8181 -def get_fields(table_name):
8182 """ 8183 returns the list of field names of a table 8184 """ 8185 table = Table(table_name) 8186 return table.field_names
8187
8188 -def info(table_name):
8189 """ 8190 prints table info 8191 """ 8192 table = Table(table_name) 8193 print(str(table))
8194
8195 -def rename_field(table_name, oldfield, newfield):
8196 """ 8197 renames a field in a table 8198 """ 8199 table = Table(table_name) 8200 try: 8201 table.rename_field(oldfield, newfield) 8202 finally: 8203 table.close()
8204
8205 -def structure(table_name, field=None):
8206 """ 8207 returns the definition of a field (or all fields) 8208 """ 8209 table = Table(table_name) 8210 return table.structure(field)
8211
8212 -def hex_dump(records):
8213 """ 8214 just what it says ;) 8215 """ 8216 for index, dummy in enumerate(records): 8217 chars = dummy._data 8218 print("%2d: " % (index,)) 8219 for char in chars[1:]: 8220 print(" %2x " % (ord(char),)) 8221 print()
8222
8223 8224 # Foxpro functions 8225 8226 -def gather(record, data, drop=False):
8227 """ 8228 saves data into a record's fields; writes to disk if not in flux 8229 keys with no matching field will raise a FieldMissingError 8230 exception unless drop_missing == True; 8231 if an Exception occurs the record is restored before reraising 8232 """ 8233 if isinstance(record, Record) and record._meta.status == CLOSED: 8234 raise DbfError("%s is closed; cannot modify record" % record._meta.filename) 8235 record_in_flux = not record._write_to_disk 8236 if not record_in_flux: 8237 record._start_flux() 8238 try: 8239 record_fields = field_names(record) 8240 for key in field_names(data): 8241 value = data[key] 8242 if not key in record_fields: 8243 if drop: 8244 continue 8245 raise FieldMissingError(key) 8246 record[key] = value 8247 except: 8248 if not record_in_flux: 8249 record._rollback_flux() 8250 raise 8251 if not record_in_flux: 8252 record._commit_flux()
8253
8254 -def scan(table, direction='forward', filter=lambda rec: True):
8255 """ 8256 moves record pointer forward 1; returns False if Eof/Bof reached 8257 table must be derived from _Navigation or have skip() method 8258 """ 8259 if direction not in ('forward', 'reverse'): 8260 raise TypeError("direction should be 'forward' or 'reverse', not %r" % direction) 8261 if direction == 'forward': 8262 n = +1 8263 no_more_records = Eof 8264 else: 8265 n = -1 8266 no_more_records = Bof 8267 try: 8268 while True: 8269 table.skip(n) 8270 if filter(table.current_record): 8271 return True 8272 except no_more_records: 8273 return False
8274
8275 -def scatter(record, as_type=create_template, _mappings=getattr(collections, 'Mapping', dict)):
8276 """ 8277 returns as_type() of [fieldnames and] values. 8278 """ 8279 if isinstance(as_type, types.FunctionType): 8280 return as_type(record) 8281 elif issubclass(as_type, _mappings): 8282 return as_type(zip(field_names(record), record)) 8283 else: 8284 return as_type(record)
8285