Sunday, October 19, 2008

Pure Python Dictionary Implementation

For those curious about how CPython's dict implementation works, I've written a Python implementation using the same algorithms. Aside from the education value, it's pretty useless because it doesn't support None as a value and is extremely slow. You can get the source in a Bazaar repo:

A Python dict implementation.

import collections

dummy = "<dummy key>"

class Entry(object):
A hash table entry.

* key - The key for this entry.
* hash - The has of the key.
* value - The value associated with the key.

__slots__ = ("key", "value", "hash")

def __init__(self):
self.key = None
self.value = None
self.hash = 0

def __repr__(self):
return "<Entry: key={0} value={1}>".format(self.key, self.value)

class Dict(object):
A mapping interface implemented as a hash table.

* used - The number of entires used in the table.
* filled - used + number of entries with a dummy key.
* table - List of entries; contains the actual dict data.
* mask - Length of table - 1. Used to fetch values.

__slots__ = ("filled", "used", "mask", "table")

def __init__(self, arg=None, **kwargs):
self._update(arg, kwargs)

def fromkeys(cls, keys, value=0):
Return a new dictionary from a sequence of keys.
d = cls()
for key in keys:
d[key] = value
return d

def clear(self):
Clear the dictionary of all data.
self.filled = 0
self.used = 0
self.mask = MINSIZE - 1
self.table = []
# Initialize the table to a clean slate of entries.
for i in range(MINSIZE):

def pop(self, *args):
Remove and return the value for a key.
have_default = len(args) == 2
v = self[args[0]]
except KeyError:
if have_default:
return args[1]
del self[args[0]]
return v

def popitem(self):
Remove and return any key-value pair from the dictionary.
if self.used == 0:
raise KeyError("empty dictionary")
entry0 = self.table[0]
entry = entry0
i = 0
if entry0.value is None:
# The first entry in the table's hash is abused to hold the index to
# the next place to look for a value to pop.
i = entry0.hash
if i > self.mask or i < i:
i = 1
entry = self.table[i]
while entry.value is None:
i += 1
if i > self.mask:
i = 1
entry = self.table[i]
res = entry.key, entry.value
# Set the next place to start.
entry0.hash = i + 1
return res

def setdefault(self, key, default=0):
If key is in the dictionary, return it. Otherwise, set it to the default
val = self._lookup(key).value
if val is None:
self[key] = default
return default
return val

def _lookup(self, key):
Find the entry for a key.
key_hash = hash(key)
i = key_hash & self.mask
entry = self.table[i]
if entry.key is None or entry is key:
return entry
free = None
if entry.key is dummy:
free = entry
elif entry.hash == key_hash and key == entry.key:
return entry

perturb = key_hash
while True:
i = (i << 2) + i + perturb + 1;
entry = self.table[i & self.mask]
if entry.key is None:
return entry if free is None else free
if entry.key is key or \
(entry.hash == key_hash and key == entry.key):
return entry
elif entry.key is dummy and free is None:
free = dummy
perturb >>= PERTURB_SHIFT

assert False, "not reached"

def _resize(self, minused):
Resize the dictionary to at least minused.
newsize = MINSIZE
# Find the smalled value for newsize.
while newsize <= minused and newsize > 0:
newsize <<= 1
oldtable = self.table
# Create a new table newsize long.
newtable = []
while len(newtable) < newsize:
# Replace the old table.
self.table = newtable
self.used = 0
self.filled = 0
# Copy the old data into the new table.
for entry in oldtable:
if entry.value is not None:
elif entry.key is dummy:
entry.key = None
self.mask = newsize - 1

def _insert_into_clean(self, entry):
Insert an item in a clean dict. This is a helper for resizing.
i = entry.hash & self.mask
new_entry = self.table[i]
perturb = entry.hash
while new_entry.key is not None:
i = (i << 2) + i + perturb + 1
new_entry = self.table[i & self.mask]
perturb >>= PERTURB_SHIFT
new_entry.key = entry.key
new_entry.value = entry.value
new_entry.hash = entry.hash
self.used += 1
self.filled += 1

def _insert(self, key, value):
Add a new value to the dictionary or replace an old one.
entry = self._lookup(key)
if entry.value is None:
self.used += 1
if entry.key is not dummy:
self.filled += 1
entry.key = key
entry.hash = hash(key)
entry.value = value

def _del(self, entry):
Mark an entry as free with the dummy key.
entry.key = dummy
entry.value = None
self.used -= 1

def __getitem__(self, key):
value = self._lookup(key).value
if value is None:
# Check if we're a subclass.
if type(self) is not Dict:
# Try to call the __missing__ method.
missing = getattr(self, "__missing__")
if missing is not None:
return missing(key)
raise KeyError("no such key: {0!r}".format(key))
return value

def __setitem__(self, key, what):
# None is used as a marker for empty entries, so it can't be in a
# dictionary.
assert what is not None and key is not None, \
"key and value must not be None"
old_used = self.used
self._insert(key, what)
# Maybe resize the dict.
if not (self.used > old_used and
self.filled*3 >= (self.mask + 1)*2):
# Large dictionaries (< 5000) are only doubled in size.
factor = 2 if self.used > 5000 else 4

def __delitem__(self, key):
entry = self._lookup(key)
if entry.value is None:
raise KeyError("no such key: {0!r}".format(key))

def __contains__(self, key):
Check if a key is in the dictionary.
return self._lookup(key).value is not None

def __eq__(self, other):
if not isinstance(other, Dict):
# Try to coerce the other to a Dict, so we can compare it.
other = Dict(other)
except TypeError:
return NotImplemented
if self.used != other.used:
# They're not the same size.
return False
# Look through the table and compare every entry, breaking out early if
# we find a difference.
for entry in self.table:
if entry.value is not None:
bval = other[entry.key]
except KeyError:
return False
if not bval == entry.value:
return False
return True

def __ne__(self, other):
return not self == other

def keys(self):
Return a list of keys in the dictionary.
return [entry.key for entry in self.table if entry.value is not None]

def values(self):
Return a list of values in the dictionary.
return [entry.value for entry in self.table if entry.value is not None]

def items(self):
Return a list of key-value pairs.
return [(entry.key, entry.value) for entry in self.table
if entry.value is not None]

def __iter__(self):
return DictKeysIterator(self)

def itervalues(self):
Return an iterator over the values in the dictionary.
return DictValuesIterator(self)

def iterkeys(self):
Return an iterator over the keys in the dictionary.
return DictKeysIterator(self)

def iteritems(self):
Return an iterator over key-value pairs.
return DictItemsIterator(self)

def _merge(self, mapping):
Update the dictionary from a mapping.
for key in mapping.keys():
self[key] = mapping[key]

def _from_sequence(self, seq):
for double in seq:
if len(double) != 2:
raise ValueError("{0!r} doesn't have a length of 2".format(
self[double[0]] = double[1]

def _update(self, arg, kwargs):
if arg:
if isinstance(arg, collections.Mapping):
if kwargs:

def update(self, arg=None, **kwargs):
Update the dictionary from a mapping or sequence containing key-value
pairs. Any existing values are overwritten.
self._update(arg, kwargs)

def get(self, key, default=0):
Return the value for key if it exists otherwise the default.
return self[key]
except KeyError:
return default

def __len__(self):
return self.used

def __repr__(self):
r = ["{0!r} : {1!r}".format(k, v) for k, v in self.iteritems()]
return "Dict({" + ", ".join(r) + "})"


class DictIterator(object):

def __init__(self, d):
self.d = d
self.used = self.d.used
self.len = self.d.used
self.pos = 0

def __iter__(self):
return self

def next(self):
# Check if the dictionary has been mutated under us.
if self.used != self.d.used:
# Make this state permanent.
self.used = -1
raise RuntimeError("dictionary size changed during interation")
i = self.pos
while i <= self.d.mask and self.d.table[i].value is None:
i += 1
self.pos = i + 1
if i > self.d.mask:
# We're done.
raise StopIteration
self.len -= 1
return self._extract(self.d.table[i])

__next__ = next

def _extract(self, entry):
return getattr(entry, self.kind)

def __len__(self):
return self.len

class DictKeysIterator(DictIterator):
kind = "key"

class DictValuesIterator(DictIterator):
kind = "value"

class DictItemsIterator(DictIterator):

def _extract(self, entry):
return entry.key, entry.value

Monday, October 13, 2008

First impressions of darcs

This week, I've been playing around with the relatively little known distributed version control system, darcs. (That stands for David's Advanced Revision Control System.)

Darcs is based on David Roundy's, its creator, theory of patches. Simply put, darcs' fundamental type is a difference between two trees, a patch.

Creating a simple repo was quick and painless with "darcs initialize". I recorded a few patches easily, and was feeling quite happy about the fast pace with which darcs went about its business. Then, I decided to review my work. Apparently, darcs has no concept of a revision number; every "commit" is just a patch. This makes selecting patches to review rather difficult since everything is relative to the current state of the repo. Perhaps this isn't a problem in practice, though, because advanced patch matching (with regular expressions) is provided. Another thing I disliked was the lack of history in merging between repos. Although it is simple to do, no evidence besides the author's name in the log indicates that the patch was pulled.

Obviously, this is just a first step into the exciting darcs world; I'll continue to use it for some of my projects, and report back later.

Wednesday, October 1, 2008

Python 2.6 released!

Fire the cannons! Begin the fireworks! Scream at the top of your lungs! Clink your glasses! Python 2.6 is here! Download it, and learn what's new.

This is my first final python release on the core team, and I'm quite proud of our baby. :)