summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorsand <daniel@spatof.org>2013-05-30 18:05:55 (GMT)
committer sand <daniel@spatof.org>2013-05-30 18:05:55 (GMT)
commit76eeae8fe016e48e937a19b4919e7e3ba8c0c1e4 (patch)
treea0d1cb0351f29bf5128e31e77a2ba19f271ebc01
parentcfe0bc4e1f60cb3842adf23a1c06e95a4dccc0b2 (diff)
import coil 0.3.21
-rw-r--r--coil/__init__.py40
-rw-r--r--coil/errors.py80
-rw-r--r--coil/parser.py470
-rw-r--r--coil/struct.py840
-rw-r--r--coil/test/__init__.py1
-rw-r--r--coil/test/test_parser.py310
-rw-r--r--coil/test/test_struct.py301
-rw-r--r--coil/test/test_text.py253
-rw-r--r--coil/test/test_tokenizer.py125
-rw-r--r--coil/text.py39
-rw-r--r--coil/tokenizer.py289
11 files changed, 2748 insertions, 0 deletions
diff --git a/coil/__init__.py b/coil/__init__.py
new file mode 100644
index 0000000..47b7caf
--- /dev/null
+++ b/coil/__init__.py
@@ -0,0 +1,40 @@
+# Copyright (c) 2005-2006 Itamar Shtull-Trauring.
+# Copyright (c) 2008-2009 ITA Software, Inc.
+# See LICENSE.txt for details.
+
+"""Coil: A Configuration Library."""
+
+__version_info__ = (0,3,21)
+__version__ = ".".join([str(x) for x in __version_info__])
+__all__ = ['struct', 'parser', 'tokenizer', 'errors']
+
+from coil.parser import Parser
+
+def parse_file(file_name, **kwargs):
+ """Open and parse a coil file.
+
+ See :class:`Parser <coil.parser.Parser>` for possible keyword arguments.
+
+ :param file_name: Name of file to parse.
+ :type file_name: str
+
+ :return: The root object.
+ :rtype: :class:`Struct <coil.struct.Struct>`
+ """
+ file_fd = open(file_name)
+ parser = Parser(file_fd, file_name, **kwargs)
+ file_fd.close()
+ return parser.root()
+
+def parse(string, **kwargs):
+ """Parse a coil string.
+
+ See :class:`Parser <coil.parser.Parser>` for possible keyword arguments.
+
+ :param file_name: String containing data to parse.
+ :type file_name: str
+
+ :return: The root object.
+ :rtype: :class:`Struct <coil.struct.Struct>`
+ """
+ return Parser(string.splitlines(), **kwargs).root()
diff --git a/coil/errors.py b/coil/errors.py
new file mode 100644
index 0000000..b0b050f
--- /dev/null
+++ b/coil/errors.py
@@ -0,0 +1,80 @@
+# Copyright (c) 2008-2009 ITA Software, Inc.
+# See LICENSE.txt for details.
+
+class CoilError(Exception):
+ """Generic error for Coil"""
+
+ def __init__(self, location, reason):
+ self.reason = reason
+ self.location(location)
+ Exception.__init__(self, reason)
+
+ def location(self, location):
+ """Update the parser location for this exception.
+ This is useful for properly tagging :exc:`StructError`
+ instances that are raised during parse time.
+ """
+
+ self.filePath = location.filePath
+ self.line = location.line
+ self.column = location.column
+
+ def __str__(self):
+ if self.filePath or self.line:
+ return "<%s:%s> %s" % (self.filePath, self.line, self.reason)
+ else:
+ return self.reason
+
+class StructError(CoilError):
+ """Generic error for :class:`coil.struct.Struct` objects,
+ used by various Key errors.
+ """
+
+ def __init__(self, struct, reason):
+ self.structPath = struct.path()
+ CoilError.__init__(self, struct, reason)
+
+ def __str__(self):
+ if self.filePath or self.line:
+ return "<%s %s:%s> %s" % (self.structPath,
+ self.filePath, self.line, self.reason)
+ else:
+ return "<%s> %s" % (self.structPath, self.reason)
+
+class KeyMissingError(StructError, KeyError):
+ """The given key was not found"""
+
+ def __init__(self, struct, key):
+ msg = "The key %s was not found" % repr(key)
+ self.key = key
+ StructError.__init__(self, struct, msg)
+
+class KeyTypeError(StructError, TypeError):
+ """The given key was not a string"""
+
+ def __init__(self, struct, key):
+ msg = "Keys must be strings, got %s" % type(key)
+ StructError.__init__(self, struct, msg)
+
+class KeyValueError(StructError, ValueError):
+ """The given key contained invalid characters"""
+
+ def __init__(self, struct, key):
+ msg = "The key %s contains invalid characters" % repr(key)
+ StructError.__init__(self, struct, msg)
+
+class ValueTypeError(StructError):
+ """The given item in a path was not the correct type"""
+
+ def __init__(self, struct, key, item_type, need_type):
+ msg = "the item at %s is a %s, expected %s" % (
+ repr(key), item_type.__name__, need_type.__name__)
+ StructError.__init__(self, struct, msg)
+
+class CoilParseError(CoilError):
+ """General error during parsing"""
+ pass
+
+class CoilUnicodeError(CoilParseError):
+ """Invalid unicode string"""
+ pass
diff --git a/coil/parser.py b/coil/parser.py
new file mode 100644
index 0000000..a0b8b5b
--- /dev/null
+++ b/coil/parser.py
@@ -0,0 +1,470 @@
+# Copyright (c) 2008-2009 ITA Software, Inc.
+# See LICENSE.txt for details.
+
+"""Coil Parser"""
+
+import os
+import sys
+
+from coil import tokenizer, struct, errors
+
+class StructPrototype(struct.Struct):
+ """A temporary struct used for parsing only.
+
+ This Struct tracks links and inheritance so they can be processed
+ when parsing is all done. This is important because it allows us
+ to do fancy things with inheritance and catch errors during
+ parse-time rather than run-time.
+ """
+
+ def __init__(self, base=(), container=None, name=None, location=None,
+ permissive=False):
+ struct.Struct.__init__(self, base, container, name, location)
+
+ # Secondary items are ones that are inherited via @extends or @file
+ # They must be tracked separately so we can raise errors on
+ # double adds and deletes in the primary values.
+ self._secondary_values = {}
+ self._secondary_order = []
+ # _deleted is a list of items that exist in one of the parents
+ # but have been removed from this Struct by ~foo tokens.
+ self._deleted = []
+ self._permissive = permissive
+
+ def _get(self, key):
+ try:
+ return self._values[key]
+ except KeyError:
+ return self._secondary_values[key]
+
+ def _set(self, key, value):
+ self._validate_doubleset(key)
+
+ self._values[key] = value
+
+ if key in self._secondary_values:
+ del self._secondary_values[key]
+ elif key not in self._order:
+ self._order.append(key)
+
+ def _del(self, key):
+ self._validate_doubleset(key)
+
+ if key in self._values:
+ del self._values[key]
+ if key in self._order:
+ self._order.remove(key)
+ else:
+ self._secondary_order.remove(key)
+ elif key in self._secondary_values:
+ del self._secondary_values[key]
+ self._secondary_order.remove(key)
+ else:
+ raise KeyError
+
+ def __contains__(self, key):
+ return key in self._values or key in self._secondary_values
+
+ def __iter__(self):
+ for key in self._secondary_order:
+ yield key
+ for key in self._order:
+ yield key
+
+ def __len__(self):
+ return len(self._values) + len(self._secondary_values)
+
+ def extends(self, base, relative=False):
+ """Add a struct as another parent.
+
+ :param base: A Struct or dict to extend.
+ :param relative: Convert @root links to relative links.
+ Used when extending a Struct from another file.
+ """
+
+ def relativeize(path):
+ if not path.startswith("@root"):
+ return path
+ else:
+ new = ""
+ container = base
+ while container.container:
+ container = container.container
+ new += "."
+ new += path[5:]
+ return new
+
+ def relativestr(match):
+ return "${%s}" % relativeize(match.group(1))
+
+ def relativelist(old):
+ new = []
+ for item in old:
+ if isinstance(item, basestring):
+ item = struct.Struct.EXPAND.sub(relativestr, item)
+ elif isinstance(item, list):
+ item = relativelist(item)
+ new.append(item)
+ return new
+
+ def copylist(old):
+ new = []
+ for item in old:
+ if isinstance(item, list):
+ item = copylist(item)
+ new.append(item)
+ return new
+
+ if base is self:
+ raise errors.StructError(self,
+ "Struct cannot extend itself.")
+
+ if not isinstance(base, struct.Struct):
+ raise errors.StructError(self,
+ "attempting to extend a value which is NOT a struct.")
+
+ if base._map is not None and self._map is None:
+ self._map = list(base._map)
+
+ for key, value in base.iteritems():
+ if key in self or key in self._deleted:
+ continue
+
+ # Copy child Structs so that they can be edited independently
+ if isinstance(value, struct.Struct):
+ new = self.__class__(container=self, name=key)
+ new.extends(value, relative)
+ value = new
+
+ # Convert absolute to relative links if required
+ if relative:
+ if isinstance(value, struct.Link):
+ value.path = relativeize(value.path)
+ elif isinstance(value, basestring):
+ value = struct.Struct.EXPAND.sub(relativestr, value)
+ elif isinstance(value, list):
+ value = relativelist(value)
+ # Otherwise we still need to recursively copy lists
+ elif isinstance(value, list):
+ value = copylist(value)
+
+ self._secondary_values[key] = value
+ self._secondary_order.append(key)
+
+ def _validate_doubleset(self, key):
+ """Private: check that key has not been used (excluding parents)"""
+
+ if self._permissive:
+ return
+ elif key in self._deleted or key in self._values:
+ raise errors.StructError(self, "Setting/deleting %r twice" % key)
+
+
+class Parser(object):
+ """The standard coil parser.
+
+ :param input_: An iterator over lines of input.
+ Typically a :class:`file` object or list of strings.
+ :param path: Path to input file, used for errors and @file imports.
+ :param encoding: Read strings using the given encoding.
+ All string values will be :class:`unicode` objects rather than
+ :class:`str`.
+ :param expand: Enables/disables expansion of the parsed tree.
+ :param defaults: See :meth:`Struct.expanditem <coil.struct.Struct.expanditem>`
+ :param ignore_missing: See :meth:`Struct.expanditem <coil.struct.Struct.expanditem>`
+ :param ignore_types: See :meth:`Struct.expanditem <coil.struct.Struct.expanditem>`
+ :param permissive: Disable some validation checks. Currently this
+ just includes checking for double-setting atrributes.
+ """
+
+ def __init__(self, input_, path=None, encoding=None, expand=True,
+ defaults=(), ignore_missing=(), ignore_types=(), permissive=False):
+ if path:
+ self._path = os.path.abspath(path)
+ else:
+ self._path = None
+
+ self._encoding = encoding
+ self._permissive = permissive
+ self._tokenizer = tokenizer.Tokenizer(input_, self._path, encoding)
+
+ # Create the root Struct and parse!
+ self._prototype = StructPrototype(permissive=permissive)
+
+ while self._tokenizer.peek('~', 'PATH', 'EOF').type != 'EOF':
+ self._parse_attribute(self._prototype)
+
+ self._tokenizer.next('EOF')
+ self._root = struct.Struct(self._prototype)
+ if expand:
+ self._root.expand(defaults=defaults,
+ ignore_missing=ignore_missing,
+ ignore_types=ignore_types)
+
+ def root(self):
+ """Get the root Struct.
+
+ :rtype: :class:`~coil.struct.Struct`
+ """
+ return self._root
+
+ def prototype(self):
+ """Get the raw unexpanded prototype, you probably don't want this.
+
+ :rtype: :class:`~coil.parser.StructPrototype`
+ """
+ return self._prototype
+
+ def __str__(self):
+ return "<%s %s>" % (self.__class__.__name__, self._root)
+
+ def __repr__(self):
+ return "<%s %s>" % (self.__class__.__name__, self._root)
+
+ def _parse_attribute(self, container):
+ """name: value"""
+
+ token = self._tokenizer.next('~', 'PATH')
+
+ if token.type == '~':
+ token = self._tokenizer.next('PATH')
+
+ try:
+ del container[token.value]
+ except errors.StructError, ex:
+ ex.location(token)
+ raise ex
+ else:
+ self._tokenizer.next(':')
+
+ if token.value[0] == '@':
+ special = getattr(self, "_special_%s" % token.value[1:], None)
+ if special is None:
+ raise errors.CoilParseError(token,
+ "Unknown special attribute: %s" % token.value)
+ else:
+ special(container, token)
+ elif '.' in token.value:
+ # ensure parents are created for flattened paths
+ parts = token.value.split('.')
+ for key in parts[:-1]:
+ if not container.get(key, False):
+ new = StructPrototype(container=container, name=key,
+ permissive=self._permissive)
+ container[key] = new
+ container = container[key]
+ token.value = parts[-1]
+ self._parse_value(container, token.value)
+ else:
+ self._parse_value(container, token.value)
+
+ def _parse_value(self, container, name):
+ """path, number, or string"""
+
+ token = self._tokenizer.peek('{', '[', '=', 'PATH', 'VALUE')
+
+ if token.type == '{':
+ # Got a struct, will be added inside _parse_struct
+ self._parse_struct(container, name)
+ elif token.type == '[':
+ # Got a list, will be added inside _parse_list
+ self._parse_list(container, name)
+ elif token.type == '=':
+ # Got a reference, chomp the =, save the link
+ self._tokenizer.next('=')
+ self._parse_link(container, name)
+ elif token.type == 'PATH':
+ # Got a reference, save the link
+ self._parse_link(container, name)
+ else:
+ # Plain old boring values
+ self._parse_plain(container, name)
+
+ def _parse_struct(self, container, name):
+ """{ attrbute... }"""
+
+ token = self._tokenizer.next('{')
+
+ try:
+ new = StructPrototype(container=container, name=name,
+ permissive=self._permissive)
+ container[name] = new
+ except errors.StructError, ex:
+ ex.location(token)
+ raise ex
+
+ while self._tokenizer.peek('~', 'PATH', '}').type != '}':
+ self._parse_attribute(new)
+
+ self._tokenizer.next('}')
+
+ def _parse_list(self, container, name):
+ """[ number or string or list ... ]"""
+
+ new = list()
+ container[name] = new
+ self._parse_list_values(new)
+
+ def _parse_list_values(self, container):
+ """[ number or string or list ... ]"""
+
+ self._tokenizer.next('[')
+ token = self._tokenizer.peek('[', ']', 'VALUE')
+
+ while token.type != ']':
+ if token.type == '[':
+ new = list()
+ container.append(new)
+ self._parse_list_values(new)
+ else:
+ container.append(self._tokenizer.next('VALUE').value)
+
+ token = self._tokenizer.peek('[', ']', 'VALUE')
+
+ self._tokenizer.next(']')
+
+ def _parse_link(self, container, name):
+ """some.path"""
+
+ token = self._tokenizer.next('PATH')
+ link = struct.Link(token.value)
+ container.set(name, link, location=token)
+
+ def _parse_plain(self, container, name):
+ """number, string, bool, or None"""
+
+ token = self._tokenizer.next('VALUE')
+ container.set(name, token.value, location=token)
+
+ def _special_extends(self, container, token):
+ """Handle @extends: some.struct"""
+
+ token = self._tokenizer.next('=', 'PATH')
+ if token.value == '=':
+ token = self._tokenizer.next('PATH')
+
+ if container.container is None:
+ raise errors.StructError(self,
+ "@root cannot extend other structs.")
+
+ path = token.value
+
+ try:
+ parent = container.get(path)
+ except errors.StructError, ex:
+ ex.location(token)
+ raise
+
+ if not isinstance(parent, struct.Struct):
+ raise errors.StructError(container,
+ "@extends target must be of type Struct")
+
+ if parent is container:
+ raise errors.StructError(container,
+ "@extends target cannot be self")
+
+ if not (path.startswith("@") or path.startswith("..")):
+ raise errors.StructError(container,
+ "@extends target cannot be children of container")
+
+ _container = container
+ while _container is not None:
+ if _container is parent:
+ raise errors.StructError(container,
+ "@extends target cannot be parents of container")
+ _container = _container.container
+
+ container.extends(parent)
+
+ def _extend_with_file(self, container, file_path, struct_path):
+ """Parse another coil file and merge it into the tree"""
+
+ coil_file = open(file_path)
+ parent = self.__class__(coil_file, path=file_path,
+ encoding=self._encoding, expand=False).prototype()
+
+ if struct_path:
+ parent = parent.get(struct_path)
+ if not isinstance(parent, struct.Struct):
+ raise errors.StructError(container,
+ "@file specification sub-import type must be Struct.")
+
+ container.extends(parent, True)
+
+ def _special_file(self, container, token):
+ """Handle @file"""
+
+ token = self._tokenizer.next('[', 'VALUE')
+
+ if token.type == '[':
+ # @file: [ "file_name" "substruct_name" ]
+ file_path = self._tokenizer.next('VALUE').value
+ struct_path = self._tokenizer.next('VALUE').value
+ self._tokenizer.next(']')
+ else:
+ # @file: "file_name"
+ file_path = token.value
+ struct_path = ""
+
+ file_path = container.expandvalue(file_path)
+ struct_path = container.expandvalue(struct_path)
+
+ if (not isinstance(file_path, basestring) or
+ not isinstance(struct_path, basestring)):
+ raise errors.CoilParseError(token, "@file value must be a string")
+
+ if self._path and not os.path.isabs(file_path):
+ file_path = os.path.join(os.path.dirname(self._path), file_path)
+
+ if not os.path.isabs(file_path):
+ raise errors.CoilParseError(token,
+ "Unable to find absolute path: %s" % file_path)
+
+ try:
+ self._extend_with_file(container, file_path, struct_path)
+ except IOError, ex:
+ raise errors.CoilParseError(token, str(ex))
+
+ def _special_package(self, container, token):
+ """Handle @package"""
+
+ token = self._tokenizer.next('VALUE')
+
+ value = container.expandvalue(token.value)
+
+ if not isinstance(value, basestring):
+ raise errors.CoilParseError(token,
+ "@package value must be a string")
+
+ try:
+ package, path = value.split(":", 1)
+ except ValueError:
+ errors.CoilParseError(token,
+ '@package value must be "package:path"')
+
+ parts = package.split(".")
+ parts.append("__init__.py")
+
+ fullpath = None
+ for directory in sys.path:
+ if not isinstance(directory, basestring):
+ continue
+ if os.path.exists(os.path.join(directory, *parts)):
+ fullpath = os.path.join(directory, *(parts[:-1] + [path]))
+ break
+
+ if not fullpath:
+ raise errors.CoilParseError(token,
+ "Unable to find package: %s" % package)
+
+ try:
+ self._extend_with_file(container, fullpath, "")
+ except IOError, ex:
+ raise errors.CoilParseError(token, str(ex))
+
+ def _special_map(self, container, token):
+ if container._map is not None:
+ raise errors.CoilParseError(token,
+ "Found multiple @map lists, only one is allowed")
+ container._map = []
+ self._parse_list_values(container._map)
diff --git a/coil/struct.py b/coil/struct.py
new file mode 100644
index 0000000..b2a311b
--- /dev/null
+++ b/coil/struct.py
@@ -0,0 +1,840 @@
+# Copyright (c) 2005-2006 Itamar Shtull-Trauring.
+# Copyright (c) 2008-2009 ITA Software, Inc.
+# See LICENSE.txt for details.
+
+"""Struct is the core object in Coil.
+
+Struct objects are similar to dicts except they are intended to be used
+as a tree and can handle relative references between them.
+"""
+
+from __future__ import generators
+
+import re
+from UserDict import DictMixin
+
+from coil import tokenizer, errors
+
+
+# Used by _expand_str()
+_EXPAND_BRACES = re.compile("^(.*){([^}]+)}(.*)$")
+_EXPAND_RANGE = re.compile("^(0*(\d+))\.\.(\d+)$")
+
+def _expand_str(string):
+ """Helper function for _expand_list to operate on individual strings"""
+
+ if not isinstance(string, basestring):
+ return [string]
+
+ match = _EXPAND_BRACES.search(string)
+ if not match:
+ return [string]
+ else:
+ new = []
+ prefix = match.group(1)
+ postfix = match.group(3)
+
+ for item in match.group(2).split(','):
+ range = _EXPAND_RANGE.match(item)
+ if range:
+ fmt = "%%s%%0%dd%%s" % len(range.group(1))
+ for i in xrange(int(range.group(2)), int(range.group(3))+1):
+ new.extend(_expand_str(fmt % (prefix, i, postfix)))
+ else:
+ new.extend(_expand_str("%s%s%s" % (prefix, item, postfix)))
+
+ return new
+
+def _expand_list(seq):
+ """Expand {1..2} and {1,2} constructs in a list of strings.
+ Although this would be a useful public function it is private for
+ now since in the future I think I will provide a list subclass.
+ """
+
+ new = []
+ for item in seq:
+ new.extend(_expand_str(item))
+
+ return new
+
+
+class Link(object):
+ """A temporary symbolic link to another item."""
+
+ def __init__(self, path):
+ """
+ :param path: A path to point at.
+ :type path: str
+ """
+ self.path = path
+
+ def __repr__(self):
+ return "%s(%s)" % (self.__class__.__name__, repr(self.path))
+
+class Struct(tokenizer.Location, DictMixin):
+ """A dict-like object for use in trees."""
+
+ KEY = re.compile(r'^%s$' % tokenizer.Tokenizer.KEY_REGEX)
+ PATH = re.compile(r'^%s$' % tokenizer.Tokenizer.PATH_REGEX)
+ EXPAND = re.compile(r'\$\{(%s)\}' % tokenizer.Tokenizer.PATH_REGEX)
+
+ #: Signal :meth:`get` to raise an error if key is not found
+ _raise = object()
+ #: Signal :meth:`set` to preserve location data for key
+ keep = object()
+ #: All possible types of expansion
+ _expansion_types = frozenset(['links', 'strings'])
+
+ # These first methods likely would need to be overridden by subclasses
+
+ def __init__(self, base=(), container=None, name=None, location=None):
+ """
+ :param base: A :class:`dict`, :class:`Struct`, or a sequence of
+ (key, value) pairs to initialize with. Any child :class:`dict`
+ or :class:`Struct` will be recursively copied.
+ :param container: the parent :class:`Struct` if there is one.
+ :param name: The name of this :class:`Struct` in its container.
+ :param location: The where this :class:`Struct` is defined.
+ This is normally only used by the :class:`Parser
+ <coil.parser.Parser>`.
+ """
+ assert getattr(base, '__iter__', False)
+
+ tokenizer.Location.__init__(self, location)
+ self.container = container
+ self.name = name
+ self._values = {}
+ self._order = []
+
+ # the list of child structs if this is a map, this map
+ # copy kludge probably can go away when StructPrototype does.
+ self._map = getattr(base, '_map', None)
+
+ # this has to be compared to none,
+ # because Struct overrides len
+ if name and container is not None:
+ self._path = "%s.%s" % (container._path, name)
+ else:
+ self._path = "@root"
+
+ if getattr(base, 'iteritems', False):
+ base_iter = base.iteritems()
+ else:
+ base_iter = iter(base)
+
+ for key, value in base_iter:
+ if isinstance(value, (Struct, dict)):
+ value = self.__class__(value, self, key)
+ elif isinstance(value, list):
+ value = list(value)
+ self[key] = value
+
+ def _get(self, key):
+ return self._values[key]
+
+ def _set(self, key, value):
+ self._values[key] = value
+ if key not in self._order:
+ self._order.append(key)
+
+ def _del(self, key):
+ del self._values[key]
+ try:
+ self._order.remove(key)
+ except ValueError:
+ raise KeyError
+
+ def __contains__(self, key):
+ return key in self._values
+
+ def __iter__(self):
+ """Iterate over the ordered list of keys."""
+ for key in self._order:
+ yield key
+
+ def __len__(self):
+ return len(self._values)
+
+ # The remaining methods likely do not need to be overridden in subclasses
+
+ def get(self, path, default=_raise):
+ """Get a value from any :class:`Struct` in the tree.
+
+ :param path: key or arbitrary path to fetch.
+ :param default: return this value if item is missing.
+ Note that the behavior here differs from a :class:`dict`.
+ If *default* is unspecified and missing a
+ :exc:`~errors.KeyMissingError` will be raised as
+ __getitem__ does, not return *None*.
+
+ :return: The fetched item or the value of *default*.
+ """
+
+ try:
+ parent, key = self._get_next_parent(path)
+ except KeyError:
+ if default is self._raise:
+ raise
+ else:
+ return default
+
+ if parent is self:
+ if not key:
+ value = self
+ else:
+ try:
+ value = self._get(key)
+ except KeyError:
+ if default is self._raise:
+ raise errors.KeyMissingError(self, key)
+ else:
+ value = default
+ else:
+ value = parent.get(key, default)
+
+ return value
+
+ __getitem__ = get
+
+ def _update_path(self, path):
+ self._path = path
+ for key, node in self.iteritems():
+ if isinstance(node, Struct):
+ node._update_path("%s.%s" % (path, key))
+
+ def set(self, path, value, location=None):
+ """Set a value in any :class:`Struct` in the tree.
+
+ :param path: key or arbitrary path to set.
+ :param value: value to save.
+ :param location: defines where this value was defined.
+ Set to :data:`Struct.keep` to not modify the location if it
+ is already set, this is used by :meth:`expanditem`.
+ """
+
+ parent, key = self._get_next_parent(path, True)
+
+ if parent is self:
+ if not key or not self.KEY.match(key):
+ raise errors.KeyValueError(self, key)
+
+ if isinstance(value, Struct) and value.container is None:
+ value.container = self
+ value.name = key
+ value._update_path("%s.%s" % (parent._path, key))
+
+ self._set(key, value)
+ else:
+ parent.set(key, value, location)
+
+ __setitem__ = set
+
+ def __delitem__(self, path):
+ parent, key = self._get_next_parent(path)
+
+ if parent is self:
+ if not key:
+ raise errors.KeyValueError(path)
+
+ try:
+ self._del(path)
+ except KeyError:
+ raise errors.KeyMissingError(self, key)
+ else:
+ del parent[key]
+
+ def merge(self, other):
+ """Recursively merge a coil :class:`Struct` tree.
+
+ This is similar to :meth:`update` except that it will update
+ the entire subtree rather than just this object.
+ """
+
+ for key, value in other.iteritems():
+ if isinstance(value, Struct):
+ dest = self.get(key, None)
+ if not isinstance(dest, Struct):
+ dest = Struct(container=self, name=key)
+ dest.merge(value)
+ self._set(key, dest)
+ else:
+ self._set(key, value)
+
+ def keys(self):
+ """Get an ordered list of keys."""
+ return list(iter(self))
+
+ def attributes(self):
+ """Alias for :meth:`keys`.
+
+ Only for compatibility with Coil <= 0.2.2.
+ """
+ return self.keys()
+
+ def has_key(self, key):
+ """True if key is in this :class:`Struct`"""
+ return key in self
+
+ def iteritems(self):
+ """Iterate over the ordered list of (key, value) pairs."""
+ for key in self:
+ yield key, self[key]
+
+ def sort(self, cmp=None, reverse=False):
+ """Sort in place. By default items are sorted by key.
+
+ Alternate sort methods may be provided by giving a custom
+ cmp method. The arguments to cmp will be (key, value) pairs.
+ """
+
+ def cmp_wrap(a, b):
+ return cmp((a, self[a]), (b, self[b]))
+
+ if not cmp:
+ self._order.sort(reverse=reverse)
+ else:
+ self._order.sort(cmp=cmp_wrap, reverse=reverse)
+
+ def expand(self, defaults=(), ignore_missing=(), recursive=True,
+ ignore_types=(), _block=()):
+ """Expand all :class:`Link` and sub-string variables in this
+ and, if recursion is enabled, all child :class:`Struct`
+ objects. This is normally called during parsing but may be
+ useful if more control is required.
+
+ This method modifies the tree!
+
+ :param defaults: See :meth:`expandvalue`
+ :param ignore_missing: :meth:`expandvalue`
+ :param recursive: recursively expand sub-structs
+ :type recursive: :class:`bool`
+ :param ignore_types: :meth:`expandvalue`
+ :param _block: See :meth:`expandvalue`
+ """
+
+ abspath = self.path()
+ if abspath in _block:
+ raise errors.StructError(self, "Circular reference to %s" % abspath)
+
+ _block = list(_block)
+ _block.append(abspath)
+
+ if self._map is not None:
+ map = _expand_list(self._map)
+ self._map = None
+ structs = []
+ lists = []
+
+ # We don't use iter because this loop delete stuff
+ for key, value in self.items():
+ value = self.expanditem(key,
+ defaults=defaults,
+ ignore_missing=ignore_missing,
+ ignore_types=ignore_types,
+ _block=_block)
+ if isinstance(value, Struct):
+ structs.append((key, value))
+ del self[key]
+ elif isinstance(value, list):
+ value = _expand_list(value)
+ if len(value) != len(map):
+ raise errors.StructError(self, "Invalid @map list: "
+ "expected length is %s, %s has length of %s" %
+ (len(map), key, len(value)))
+ lists.append((key, value))
+ del self[key]
+ else:
+ self.set(key, value, self.keep)
+
+ for key, orig in structs:
+ for i, suffix in enumerate(map):
+ name = "%s%s" % (key, suffix)
+ if not self.validate_key(name):
+ raise errors.StructError(self, "Invalid @map list: "
+ "key contains invalid characters: %r" % suffix)
+ new = orig.__class__(orig, name=name, container=self)
+ self[name] = new
+
+ for item_key, item_values in lists:
+ new[item_key] = item_values[i]
+
+ if recursive:
+ new.expand(defaults=defaults,
+ ignore_missing=ignore_missing,
+ ignore_types=ignore_types,
+ recursive=True,
+ _block=_block)
+
+ else:
+ for key in self:
+ value = self.expanditem(key,
+ defaults=defaults,
+ ignore_missing=ignore_missing,
+ ignore_types=ignore_types,
+ _block=_block)
+ self.set(key, value, self.keep)
+ if recursive and isinstance(value, Struct):
+ value.expand(defaults=defaults,
+ ignore_missing=ignore_missing,
+ ignore_types=ignore_types,
+ recursive=True,
+ _block=_block)
+
+ def expanditem(self, path, defaults=(), ignore_missing=(),
+ ignore_types=(), _block=()):
+ """Fetch and expand an item at the given path. All :class:`Link`
+ and sub-string variables will be followed in the process. This
+ method is a no-op if value is a :class:`Struct`, use the
+ :meth:`Struct.expand` method instead.
+
+ This method does not make any changes to the tree.
+
+ :param path: A key or arbitrary path to get.
+ :param defaults: See :meth:`expandvalue`
+ :param ignore_missing: See :meth:`expandvalue`
+ :param ignore_types: :meth:`expandvalue`
+ :param _block: See :meth:`expandvalue`
+ """
+
+ parent, key = self._get_next_parent(path)
+
+ if parent is self:
+ abspath = self.path(key)
+ if abspath in _block:
+ raise errors.StructError(self,
+ "Circular reference to %s" % abspath)
+
+ _block = list(_block)
+ _block.append(abspath)
+
+ try:
+ value = self[key]
+ except errors.KeyMissingError:
+ if key in defaults:
+ return defaults[key]
+ else:
+ raise
+
+ return self.expandvalue(value,
+ defaults=defaults,
+ ignore_missing=ignore_missing,
+ ignore_types=ignore_types,
+ _block=_block)
+ else:
+ return parent.expanditem(key,
+ defaults=defaults,
+ ignore_missing=ignore_missing,
+ ignore_types=ignore_types,
+ _block=_block)
+
+ def expandvalue(self, value, defaults=(), ignore_missing=(),
+ ignore_types=(), _block=()):
+ """Use this :class:`Struct` to expand the given value. All
+ :class:`Link` and sub-string variables will be followed in
+ the process. This method is a no-op if value is a
+ :class:`Struct`, use the :meth:`expand` method instead.
+
+ This method does not make any changes to the tree.
+
+ :param value: Any value to expand, typically a
+ :class:`Link` or string.
+ :param defaults: default values to use if undefined.
+ :type defaults: :class:`dict`
+ :param ignore_missing: a set of keys that are ignored if
+ undefined and not in defaults. If simply set to True
+ then all are ignored. Otherwise raise
+ :exc:`~errors.KeyMissingError`.
+ :type ignore_missing: *True* or any container
+ :param ignore_types: a set of types that should be ignored
+ during expansion. Possible values in the set are:
+
+ - 'strings': Don't expand ${foo} references in strings.
+ - 'links': Don't follow links, leaves the Link object.
+
+ More values will be added in the future.
+ :type ignore_types: container of strings
+ :param _block: a set of absolute paths that cannot be expanded.
+ This is only for use internally to avoid circular references.
+ :type block: any container
+ """
+
+ def expand_substr(match):
+ subkey = match.group(1)
+ try:
+ subval = self.expanditem(subkey,
+ defaults=defaults,
+ ignore_missing=ignore_missing,
+ ignore_types=ignore_types,
+ _block=_block)
+ except errors.KeyMissingError, ex:
+ if ignore_missing is True or ex.key in ignore_missing:
+ return match.group(0)
+ else:
+ raise
+
+ return str(subval)
+
+ def expand_link(link):
+ try:
+ subval = self.expanditem(link.path,
+ defaults=defaults,
+ ignore_missing=ignore_missing,
+ ignore_types=ignore_types,
+ _block=_block)
+ except errors.KeyMissingError, ex:
+ if ignore_missing is True or ex.key in ignore_missing:
+ return link
+ else:
+ raise
+
+ # Structs and lists must be copied
+ if isinstance(subval, Struct):
+ subval = subval.copy()
+ if isinstance(subval, list):
+ subval = list(subval)
+
+ return subval
+
+ def expand_list(list_):
+ for i in xrange(len(list_)):
+ if ("strings" not in ignore_types and
+ isinstance(list_[i], basestring)):
+ list_[i] = self.EXPAND.sub(expand_substr, list_[i])
+ elif isinstance(list_[i], list):
+ expand_list(list_[i])
+
+ # defaults should only contain simple keys, not paths.
+ for key in defaults:
+ assert "." not in key
+
+ # allow ignore_missing=False
+ if ignore_missing is False:
+ ignore_missing = ()
+
+ # catch invalid values of ignore_types
+ ignore_types = frozenset(ignore_types)
+ if not ignore_types.issubset(self._expansion_types):
+ raise ValueError("Invalid ignore_types: %s" % ", ".join(
+ ignore_types.difference(self._expansion_types)))
+
+ if isinstance(value, Struct):
+ pass
+ elif "strings" not in ignore_types and isinstance(value, basestring):
+ value = self.EXPAND.sub(expand_substr, value)
+ elif "links" not in ignore_types and isinstance(value, Link):
+ value = expand_link(value)
+ elif isinstance(value, list):
+ expand_list(value)
+
+ return value
+
+ def unexpanded(self, absolute=False, recursive=True):
+ """Find a set of all keys that have not been expanded.
+ This is generally only useful if :meth:`expand` was
+ run with the ignore_missing parameter was set to see got
+ missed.
+
+ Normally only the short key name is given as it would be
+ provided in defaults or ignore_missing parameters for the
+ various expansion methods. Set absolute=True to return the
+ full path for each key instead.
+
+ :param absolute: Enables absolute paths.
+ :type absolute: :class:`bool`
+ :param recursive: recursively search sub-structs
+ :type recursive: :class:`bool`
+
+ :return: unexpanded keys
+ :rtype: set
+ """
+
+ def normalize_key(key):
+ if absolute:
+ return self.path(key)
+ else:
+ return key.rsplit('.', 1).pop()
+
+ def unexpanded_list(list_):
+ keys = set()
+
+ for item in list_:
+ if isinstance(item, basestring):
+ for match in self.EXPAND.finditer(item):
+ keys.add(normalize_key(match.group(1)))
+ elif isinstance(item, Link):
+ keys.add(normalize_key(item.path))
+ elif isinstance(item, (list, tuple)):
+ keys.update(unexpanded_list(item))
+ elif recursive and isinstance(item, Struct):
+ keys.update(item.unexpanded(absolute))
+
+ return keys
+
+ return unexpanded_list(self.values())
+
+ def copy(self):
+ """Recursively copy this :class:`Struct`"""
+
+ return self.__class__(self)
+
+ def dict(self, strict=False, flat=False):
+ """Recursively copy this :class:`Struct` into :class:`dict` objects
+
+ :param strict: If True then fail if the tree contains any
+ values that cannot be represented in the coil text format.
+ :type strict: :class:`bool`
+
+ :param flat: If True this :class:`Struct` is copied into a single
+ :class:`dict` object whose keys are the full paths to the values
+ in the :class:`Struct`.
+ :type flat: :class:`bool`
+
+ :return: :class:`dict` representation of this :class:`Struct`
+ :rtype: :class:`dict`
+ """
+
+ new = {}
+ for key, value in self.iteritems():
+ if strict:
+ assert self.KEY.match(key)
+
+ if isinstance(value, Struct):
+ value = value.dict(strict, flat)
+ if flat:
+ new.update(dict([('%s.%s' % (key, k), value[k])
+ for k in value]))
+ else:
+ new[key] = value
+ elif isinstance(value, dict):
+ new[key] = value.copy()
+ elif isinstance(value, list):
+ new[key] = list(value)
+ else:
+ new[key] = value
+
+ return new
+
+
+
+ def path(self, path=None):
+ """Get the absolute path of this :class:`Struct` if path is
+ *None*, otherwise the relative path from this :class:`Struct`
+ to the given path."""
+
+ if path:
+ parent, key = self._get_next_parent(path)
+
+ if parent is self:
+ return "%s.%s" % (self._path, key)
+ else:
+ return parent.path(key)
+ else:
+ return self._path
+
+ def _stritem(self, item):
+ # FIXME: unicode breaks this, we need to handle encodings
+ # explicitly in Structs rather than just in Parser
+ if isinstance(item, basestring):
+ # Should we use """ for multi-line strings?
+ item = item.replace('\\', '\\\\')
+ item = item.replace('\n', '\\n')
+ item = item.replace('\r', '\\r')
+ item = item.replace('"', '\\"')
+ return '"%s"' % item
+ elif isinstance(item, (list, tuple)):
+ return "[%s]" % " ".join([self._stritem(x) for x in item])
+ elif (isinstance(item, (int, long, float)) or
+ item in (True, False, None)):
+ return str(item)
+ else:
+ raise errors.StructError(self,
+ "%s cannot be represented in the coil text format" % item)
+
+ def string(self, strict=True, prefix=''):
+ """Convert this :class:`Struct` tree to the coil text format.
+
+ Note that if any value is a unicode string then this
+ will return a unicode object rather than a str.
+
+ :param strict: If True then fail if the tree contains any
+ values that cannot be represented in the coil text format.
+ :type strict: *bool*
+ :param prefix: Start each line with the given prefix.
+ Used internally to properly intend sub-structs.
+ :type prefix: string
+ """
+
+ result = ""
+
+ for key, val in self.iteritems():
+ # This should never happen, but might as well be safe
+ assert self.KEY.match(key)
+
+ if isinstance(val, Struct):
+ child = val.string(strict, "%s " % prefix)
+ if child:
+ result = "%s%s%s: {\n%s\n%s}\n" % (result,
+ prefix, key, child, prefix)
+ else:
+ result = "%s%s%s: {}\n" % (result, prefix, key)
+ else:
+ result = "%s%s%s: %s\n" % (result, prefix,
+ key, self._stritem(val))
+
+ return result.rstrip()
+
+ def flatten(self, strict=True, prefix='', path=''):
+ """Alternate version of :meth:`Struct.string` that generates a
+ flat key: value list rather than a Struct tree. This is equally
+ as valid as the standard format and sometimes is easier to read.
+
+ :param strict: If True then fail if the tree contains any
+ values that cannot be represented in the coil text format.
+ :type strict: *bool*
+ :param prefix: Start each line with the given prefix.
+ :type prefix: string
+ :param path: path to prefix each key with. Used for recursion.
+ :type: string
+ """
+
+ result = ""
+
+ for key, val in self.iteritems():
+ assert self.KEY.match(key)
+
+ if path:
+ child_path = "%s.%s" % (path, key)
+ else:
+ child_path = key
+
+ if isinstance(val, Struct):
+ child = val.flatten(strict, prefix, child_path)
+ if child:
+ result = "%s%s\n" % (result, child)
+ else:
+ result = "%s%s%s: {}\n" % (result, prefix, child_path)
+ else:
+ result = "%s%s%s: %s\n" % (result, prefix,
+ child_path, self._stritem(val))
+
+ return result.rstrip()
+
+ def __str__(self):
+ return self.string()
+
+ def __repr__(self):
+ attrs = ["%s: %s" % (repr(key), repr(val))
+ for key, val in self.iteritems()]
+ return "%s({%s})" % (self.__class__.__name__, ", ".join(attrs))
+
+ @classmethod
+ def validate_key(cls, key):
+ """Check if the given key is valid.
+
+ :rtype: :class:`bool`
+ """
+ return bool(cls.KEY.match(key))
+
+ @classmethod
+ def validate_path(cls, path):
+ """Check if the given path is valid.
+
+ :rtype: :class:`bool`
+ """
+ return bool(cls.PATH.match(path))
+
+ def _get_next_parent(self, path, add_parents=False):
+ """Returns the next Struct in a path and the remaining path.
+
+ If the path is a single key just return self and the key.
+ If add_parents is true then create parent Structs as needed.
+ """
+
+ if not isinstance(path, basestring):
+ raise errors.KeyTypeError(self, path)
+
+ if path.startswith("@root"):
+ if self.container:
+ parent = self.container
+ else:
+ parent = self
+ path = path[5:]
+ elif "." not in path:
+ # Quick exit for the simple case...
+ return self, path
+ elif path.startswith(".."):
+ if self.container:
+ parent = self.container
+ else:
+ raise errors.StructError(self, "Reference past root")
+ path = path[1:]
+ elif path.startswith("."):
+ parent = self
+ path = path[1:]
+ else:
+ # check for mid-path parent references
+ if ".." in path:
+ raise errors.KeyValueError(self, path)
+
+ split = path.split(".", 1)
+ key = split.pop(0)
+ if split:
+ path = split[0]
+ else:
+ path = ""
+
+ try:
+ parent = self.get(key)
+ except errors.KeyMissingError:
+ if add_parents:
+ parent = self.__class__(container=self, name=key)
+ self.set(key, parent)
+ else:
+ raise
+
+ if not isinstance(parent, Struct):
+ raise errors.ValueTypeError(self, key, type(parent), Struct)
+
+ if parent is self and "." in path:
+ # Great, we went nowhere but there is still somewhere to go
+ parent, path = self._get_next_parent(path, add_parents)
+
+ return parent, path
+
+
+#: For compatibility with Coil <= 0.2.2, use KeyError or KeyMissingError
+StructAttributeError = errors.KeyMissingError
+
+class StructNode(object):
+ """For compatibility with Coil <= 0.2.2, use :class:`Struct` instead."""
+
+ def __init__(self, struct, container=None):
+ assert isinstance(struct, Struct)
+ assert (container is None or
+ (isinstance(container, StructNode) and
+ container._struct is struct.container))
+ self._struct = struct
+ self._struct.prototype = None
+ self._struct._attrsDict = struct
+ self._struct._attrsOrder = struct.keys()
+ self._struct._deletedAttrs = {}
+ self._container = struct.container
+
+ def has_key(self, attr):
+ return self._struct.has_key(attr)
+
+ def get(self, attr, default=Struct._raise):
+ val = self._struct.get(attr, default)
+ if isinstance(val, Struct):
+ val = self.__class__(val, self)
+ return val
+
+ def attributes(self):
+ return self._struct.keys()
+
+ def iteritems(self):
+ for key in self._struct:
+ yield key, self.get(key)
+
+ def __getattr__(self, attr):
+ return self.get(attr)
diff --git a/coil/test/__init__.py b/coil/test/__init__.py
new file mode 100644
index 0000000..f59031d
--- /dev/null
+++ b/coil/test/__init__.py
@@ -0,0 +1 @@
+"""Tests for coil."""
diff --git a/coil/test/test_parser.py b/coil/test/test_parser.py
new file mode 100644
index 0000000..dd73c8a
--- /dev/null
+++ b/coil/test/test_parser.py
@@ -0,0 +1,310 @@
+"""Tests for coil.parser."""
+
+import os
+import unittest
+from coil import parser, struct, parse_file, errors
+
+class BasicTestCase(unittest.TestCase):
+
+ def testEmpty(self):
+ root = parser.Parser([""]).root()
+ self.assert_(isinstance(root, struct.Struct))
+ self.assertEquals(len(root), 0)
+
+ def testSingle(self):
+ root = parser.Parser(["this: 'that'"]).root()
+ self.assertEquals(len(root), 1)
+ self.assertEquals(root['this'], "that")
+
+ def testMany(self):
+ root = parser.Parser(["this: 'that' int: 1 float: 2.0"]).root()
+ self.assertEquals(len(root), 3)
+ self.assertEquals(root['this'], "that")
+ self.assert_(isinstance(root['int'], int))
+ self.assertEquals(root['int'], 1)
+ self.assert_(isinstance(root['float'], float))
+ self.assertEquals(root['float'], 2.0)
+
+ def testStruct(self):
+ root = parser.Parser(["foo: { bar: 'baz' } -moo: 'cow'"]).root()
+ self.assert_(isinstance(root['foo'], struct.Struct))
+ self.assertEquals(root['foo']['bar'], "baz")
+ self.assertEquals(root.get('foo.bar'), "baz")
+ self.assertEquals(root.get('@root.foo.bar'), "baz")
+ self.assertEquals(root['-moo'], "cow")
+
+ def testExtends(self):
+ root = parser.Parser(["a: {x: 'x'} b: { @extends: ..a }"]).root()
+ self.assertEquals(root['b']['x'], "x")
+
+ def testRefrences(self):
+ root = parser.Parser(["a: 'a' b: a x: { c: ..a d: =..a }"]).root()
+ self.assertEquals(root['a'], 'a')
+ self.assertEquals(root['b'], 'a')
+ self.assertEquals(root.get('x.c'), 'a')
+ self.assertEquals(root.get('x.d'), 'a')
+
+ def testDelete(self):
+ root = parser.Parser(["a: {x: 'x' y: 'y'}"
+ "b: { @extends: ..a ~y}"]).root()
+ self.assertEquals(root['b']['x'], "x")
+ self.assertRaises(KeyError, lambda: root['b']['y'])
+
+ def testFile(self):
+ path = os.path.join(os.path.dirname(__file__), "simple.coil")
+ root = parser.Parser(["@file: %s" % repr(path)]).root()
+ self.assertEquals(root.get('x'), "x value")
+ self.assertEquals(root.get('y.z'), "z value")
+
+ def testFileSub(self):
+ path = os.path.join(os.path.dirname(__file__), "simple.coil")
+ root = parser.Parser(["sub: { @file: [%s 'y']}" % repr(path)]).root()
+ self.assertEquals(root.get('sub.z'), "z value")
+
+ self.assertRaises(errors.StructError, parser.Parser,
+ ["sub: { @file: [%s 'a']}" % repr(path)])
+
+ self.assertRaises(errors.StructError, parser.Parser,
+ ["sub: { @file: [%s 'x']}" % repr(path)])
+
+ def testFileDelete(self):
+ path = os.path.join(os.path.dirname(__file__), "simple.coil")
+ root = parser.Parser(["sub: { @file: %s ~y.z}" % repr(path)]).root()
+ self.assertEquals(root.get('sub.x'), "x value")
+ self.assert_(root.get('sub.y', None) is not None)
+ self.assertRaises(KeyError, lambda: root.get('sub.y.z'))
+
+ def testFileExpansion(self):
+ path = os.path.join(os.path.dirname(__file__), "simple.coil")
+ text = "path: %s sub: { @file: '${@root.path}' }" % repr(path)
+ root = parser.Parser([text]).root()
+ self.assertEquals(root.get('sub.x'), "x value")
+ self.assertEquals(root.get('sub.y.z'), "z value")
+
+ def testPackage(self):
+ root = parser.Parser(["@package: 'coil.test:simple.coil'"]).root()
+ self.assertEquals(root.get('x'), "x value")
+ self.assertEquals(root.get('y.z'), "z value")
+
+ def testComments(self):
+ root = parser.Parser(["y: [12 #hello\n]"]).root()
+ self.assertEquals(root.get("y"), [12])
+
+ def testParseError(self):
+ for coil in (
+ "struct: {",
+ "struct: }",
+ "a: b:",
+ ":",
+ "[]",
+ "a: ~b",
+ "@x: 2",
+ "x: 12c",
+ "x: 12.c3",
+ "x: @root",
+ "x: ..a",
+ 'x: {@package: "coil.test:nosuchfile"}',
+ # should get internal parse error
+ 'x: {@package: "coil.test:test_parser.py"}',
+ 'z: [{x: 2}]', # can't have struct in list
+ r'z: "lalalal \"', # string is not closed
+ 'a: 1 z: [ =@root.a ]',
+ 'a: {@extends: @root.b}', # b doesn't exist
+ 'a: {@extends: ..b}', # b doesn't exist
+ 'a: {@extends: x}',
+ 'a: {@extends: .}',
+ 'a: 1 b: { @extends: ..a }', # extend struct only
+ 'a: { @extends: ..a }', # extend self
+ 'a: { b: {} @extends: b }', # extend children
+ 'a: { b: { @extends: ...a } }', # extend parents
+ 'a: [1 2 3]]',
+ ):
+ self.assertRaises(errors.CoilError, parser.Parser, [coil])
+
+ def testOrder(self):
+ self.assertEqual(parser.Parser(["x: =y y: 'foo'"]).root()['x'], "foo")
+ self.assertEqual(parser.Parser(["y: 'foo' x: =y"]).root()['x'], "foo")
+
+ def testList(self):
+ root = parser.Parser(["x: ['a' 1 2.0 True False None]"]).root()
+ self.assertEqual(root['x'], ['a', 1, 2.0, True, False, None])
+
+ def testNestedList(self):
+ root = parser.Parser(["x: ['a' ['b' 'c']]"]).root()
+ self.assertEqual(root['x'], ['a', ['b', 'c']])
+
+class ExtendsTestCase(unittest.TestCase):
+
+ def setUp(self):
+ self.text = """
+ A: {
+ a: "a"
+ b: "b"
+ c: "c"
+ l: [ "${a}" [ "${a}" ] ]
+ }
+ B: {
+ @extends: ..A
+ e: [ "one" 2 "omg three" ]
+ ~c
+ }
+ C: {
+ a: ..A.a
+ b: @root.B.b
+ }
+ D: {
+ @extends: @root.B
+ }
+ E: {
+ F.G.H: {
+ a: 1 b: 2 c: 3
+ }
+
+ F.G.I: {
+ @extends: ..H
+ }
+ }
+ L: {
+ @extends: ..A
+ a: "l"
+ }
+ """
+ self.tree = parser.Parser(self.text.splitlines()).root()
+
+ def testBasic(self):
+ self.assertEquals(self.tree['A']['a'], "a")
+ self.assertEquals(self.tree['A']['b'], "b")
+ self.assertEquals(self.tree['A']['c'], "c")
+ self.assertEquals(len(self.tree['A']), 4)
+
+ def testExtendsAndDelete(self):
+ self.assertEquals(self.tree['B']['a'], "a")
+ self.assertEquals(self.tree['B']['b'], "b")
+ self.assertRaises(KeyError, lambda: self.tree['B']['c'])
+ self.assertEquals(self.tree['B']['e'], [ "one", 2, "omg three" ])
+ self.assertEquals(len(self.tree['B']), 4)
+
+ def testRefrences(self):
+ self.assertEquals(self.tree['C']['a'], "a")
+ self.assertEquals(self.tree['C']['b'], "b")
+ self.assertEquals(len(self.tree['C']), 2)
+
+ def testExtends(self):
+ self.assertEquals(self.tree['D']['a'], "a")
+ self.assertEquals(self.tree['D']['b'], "b")
+ self.assertRaises(KeyError, lambda: self.tree['D']['c'])
+ self.assertEquals(self.tree['D']['e'], [ "one", 2, "omg three" ])
+ self.assertEquals(len(self.tree['D']), 4)
+
+ def testRelativePaths(self):
+ self.assertEquals(self.tree['E']['F']['G']['H']['a'], 1)
+ self.assertEquals(self.tree['E']['F']['G']['I']['a'], 1)
+ self.assertEquals(self.tree['E']['F']['G']['H'], self.tree['E']['F']['G']['I'])
+
+ def testLists(self):
+ self.assertEquals(self.tree['L']['l'][0], "l")
+ self.assertEquals(self.tree['L']['l'][1][0], "l")
+
+class ParseFileTestCase(unittest.TestCase):
+
+ def setUp(self):
+ self.path = os.path.dirname(__file__)
+
+ def testExample(self):
+ root = parse_file(os.path.join(self.path, "example.coil"))
+ self.assertEquals(root['x'], 1)
+ self.assertEquals(root.get('y.a'), 2)
+ self.assertEquals(root.get('y.x'), 1)
+ self.assertEquals(root.get('y.a2'), 2)
+
+ def testExample2(self):
+ root = parse_file(os.path.join(self.path, "example2.coil"))
+ self.assertEquals(root.get('sub.x'), "foo")
+ self.assertEquals(root.get('sub.y.a'), "bar")
+ self.assertEquals(root.get('sub.y.x'), "foo")
+ self.assertEquals(root.get('sub.y.a2'), "bar")
+ self.assertEquals(root.get('sub2.y.a'), 2)
+ self.assertEquals(root.get('sub2.y.x'), 1)
+ self.assertEquals(root.get('sub2.y.a2'), 2)
+
+ def testExample3(self):
+ root = parse_file(os.path.join(self.path, "example3.coil"))
+ self.assertEquals(root['x'], 1)
+ self.assertEquals(root.get('y.a'), 2)
+ self.assertEquals(root.get('y.x'), 1)
+ self.assertEquals(root.get('y.a2'), 2)
+ self.assertEquals(root.get('y.b'), 3)
+
+class MapTestCase(unittest.TestCase):
+
+ def setUp(self):
+ self.text = """
+ expanded: {
+ a1: {
+ x: 1
+ y: 1
+ z: 1
+ }
+ a2: {
+ x: 2
+ y: 3
+ z: 1
+ }
+ a3: {
+ x: 3
+ y: 5
+ z: 1
+ }
+ b1: {
+ x: 1
+ y: 1
+ z: 2
+ }
+ b2: {
+ x: 2
+ y: 3
+ z: 2
+ }
+ b3: {
+ x: 3
+ y: 5
+ z: 2
+ }
+ }
+ map: {
+ @map: [1 2 3]
+ x: [1 2 3]
+ y: [1 3 5]
+ a: { z: 1 }
+ b: { z: 2 }
+ }
+ map1: {
+ @extends: ..map
+ }
+ map2: {
+ @extends: ..map
+ a: { z: 3 }
+ j: [7 8 9]
+ }
+ """
+ self.tree = parser.Parser(self.text.splitlines()).root()
+
+ def testMap(self):
+ self.assertEquals(self.tree['map'], self.tree['expanded'])
+
+ def testExtends(self):
+ self.assertEquals(self.tree['map1'], self.tree['expanded'])
+ self.assertEquals(self.tree['map2.a1.z'], 3)
+ self.assertEquals(self.tree['map2.a1.j'], 7)
+ self.assertEquals(self.tree['map2.a2.z'], 3)
+ self.assertEquals(self.tree['map2.a2.j'], 8)
+ self.assertEquals(self.tree['map2.a3.z'], 3)
+ self.assertEquals(self.tree['map2.a3.j'], 9)
+
+class ReparseTestCase(unittest.TestCase):
+
+ def testStringWhitespace(self):
+ text = """a: 'this\nis\r\na\tstring\n\r\n\t'"""
+ orig = parser.Parser([text]).root()
+ new = parser.Parser([str(orig)]).root()
+ self.assertEquals(orig, new)
diff --git a/coil/test/test_struct.py b/coil/test/test_struct.py
new file mode 100644
index 0000000..53b0cda
--- /dev/null
+++ b/coil/test/test_struct.py
@@ -0,0 +1,301 @@
+"""Tests for coil.struct."""
+
+import unittest
+from coil import struct, errors
+
+class BasicTestCase(unittest.TestCase):
+
+ def setUp(self):
+ # Use a tuple to preserve order
+ self.data = (('first', {
+ 'string': "something",
+ 'float': 2.5,
+ 'int': 1,
+ 'dict': {
+ 'x': 1,
+ 'y': 2,
+ 'z': "another thing"}
+ }),
+ ('second', "something else"),
+ ('last', [ "list", "of", "strings" ]))
+ self.struct = struct.Struct(self.data)
+
+ def testFirstLevelContains(self):
+ for key in ('first', 'second', 'last'):
+ self.assert_(key in self.struct)
+
+ def testSecondLevelContains(self):
+ for key in ('string', 'float', 'int'):
+ self.assert_(key in self.struct['first'])
+
+ def testKeyOrder(self):
+ self.assertEquals(self.struct.keys(), ['first', 'second', 'last'])
+
+ def testGetItem(self):
+ self.assertEquals(self.struct['second'], "something else")
+
+ def testGetSimple(self):
+ self.assertEquals(self.struct.get('second'), "something else")
+
+ def testGetDefault(self):
+ self.assertEquals(self.struct.get('bogus', "awesome"), "awesome")
+ self.assertEquals(self.struct.get('bogus.sub', "awesome"), "awesome")
+
+ def testGetPath(self):
+ self.assertEquals(self.struct.path(), "@root");
+ self.assertEquals(self.struct.get('first.int'), 1)
+
+ def testGetRelativePath(self):
+ self.assertEquals(self.struct.path(''), "@root")
+ self.assertEquals(self.struct.path('first'), "@root.first")
+ self.assertEquals(self.struct.path('last'), "@root.last")
+ self.assertEquals(self.struct.get('first').path('string'), "@root.first.string")
+ self.assertEquals(self.struct.get('first').path('dict.x'), "@root.first.dict.x")
+ self.assertEquals(self.struct.get('first').path('dict.y'), "@root.first.dict.y")
+
+ def testGetParent(self):
+ child = self.struct['first']
+ self.assertEquals(child.get('..second'), "something else")
+
+ def testGetRoot(self):
+ child = self.struct['first']
+ self.assertEquals(child.get('@root.second'), "something else")
+
+ def testIterItems(self):
+ itemlist = [("one", 1), ("two", 2), ("three", 3)]
+ self.assertEquals(list(struct.Struct(itemlist).iteritems()), itemlist)
+
+ def testKeyMissing(self):
+ self.assertRaises(errors.KeyMissingError, lambda: self.struct['bogus'])
+ self.assertRaises(errors.KeyMissingError, self.struct.get, 'bad')
+
+ def testKeyType(self):
+ self.assertRaises(errors.KeyTypeError, lambda: self.struct[None])
+ self.assertRaises(errors.KeyTypeError, self.struct.get, None)
+
+ def testKeyValue(self):
+ self.assertRaises(errors.KeyValueError,
+ self.struct.set, 'first#', '')
+ self.assertRaises(errors.KeyValueError,
+ self.struct.set, 'first..second', '')
+
+ def testDict(self):
+ self.assertEquals(self.struct['first'].dict(), dict(self.data[0][1]))
+
+ def testFlatDict(self):
+ fd = self.struct.dict(flat=True)
+ s = struct.Struct()
+ for k in fd:
+ s[k] = fd[k]
+ self.assertEquals(self.struct, s)
+
+ def testSetShort(self):
+ s = struct.Struct()
+ s['new'] = True
+ self.assertEquals(s['new'], True)
+
+ def testSetLong(self):
+ s = struct.Struct()
+ s['new.sub'] = True
+ self.assertEquals(s['new.sub'], True)
+ self.assertEquals(s['new']['sub'], True)
+
+ def testSetSubStruct(self):
+ s = struct.Struct({'sub': {'x': '${y}'}})
+ self.assertRaises(KeyError, s.expand)
+ s['sub.y'] = "zap"
+ s.expand()
+ self.assertEquals(s['sub.x'], "zap")
+ self.assertEquals(s['sub.y'], "zap")
+ self.assertEquals(s['sub']['x'], "zap")
+ self.assertEquals(s['sub']['y'], "zap")
+
+ def testCopy(self):
+ a = self.struct['first'].copy()
+ b = self.struct['first'].copy()
+ a['string'] = "this is a"
+ b['string'] = "this is b"
+ self.assertEquals(a['string'], "this is a")
+ self.assertEquals(b['string'], "this is b")
+ self.assertEquals(a['@root.string'], "this is a")
+ self.assertEquals(b['@root.string'], "this is b")
+ self.assertEquals(self.struct['first.string'], "something")
+
+ def testValidate(self):
+ self.assertEquals(struct.Struct.validate_key("foo"), True)
+ self.assertEquals(struct.Struct.validate_key("foo.bar"), False)
+ self.assertEquals(struct.Struct.validate_key("@root"), False)
+ self.assertEquals(struct.Struct.validate_key("#blah"), False)
+ self.assertEquals(struct.Struct.validate_path("foo"), True)
+ self.assertEquals(struct.Struct.validate_path("foo.bar"), True)
+ self.assertEquals(struct.Struct.validate_path("@root"), True)
+ self.assertEquals(struct.Struct.validate_path("#blah"), False)
+
+ def testMerge(self):
+ s1 = self.struct.copy()
+ s2 = struct.Struct()
+ s2['first.new'] = "whee"
+ s2['other.new'] = "woot"
+ s2['new'] = "zomg"
+ s1.merge(s2)
+ self.assertEquals(s1['first.string'], "something")
+ self.assertEquals(s1['first.new'], "whee")
+ self.assertEquals(s1['other'], struct.Struct({'new': "woot"}))
+ self.assertEquals(s1['new'], "zomg")
+
+class ExpansionTestCase(unittest.TestCase):
+
+ def testExpand(self):
+ root = struct.Struct()
+ root["foo"] = "bbq"
+ root["bar"] = "omgwtf${foo}"
+ root.expand()
+ self.assertEquals(root.get('bar'), "omgwtfbbq")
+
+ def testExpandItem(self):
+ root = struct.Struct()
+ root["foo"] = "bbq"
+ root["bar"] = "omgwtf${foo}"
+ self.assertEquals(root.get('bar'), "omgwtf${foo}")
+ self.assertEquals(root.expanditem('bar'), "omgwtfbbq")
+
+ def testExpandDefault(self):
+ root = struct.Struct()
+ root["foo"] = "bbq"
+ root["bar"] = "omgwtf${foo}${baz}"
+ root.expand({'foo':"123",'baz':"456"})
+ self.assertEquals(root.get('bar'), "omgwtfbbq456")
+
+ def testExpandItemDefault(self):
+ root = struct.Struct()
+ root["foo"] = "bbq"
+ root["bar"] = "omgwtf${foo}${baz}"
+ self.assertEquals(root.get('bar'), "omgwtf${foo}${baz}")
+ self.assertEquals(root.expanditem('bar',
+ defaults={'foo':"123",'baz':"456"}), "omgwtfbbq456")
+
+ def testExpandIgnore(self):
+ root = struct.Struct()
+ root["foo"] = "bbq"
+ root["bar"] = "omgwtf${foo}${baz}"
+ root.expand(ignore_missing=True)
+ self.assertEquals(root.get('bar'), "omgwtfbbq${baz}")
+ root.expand(ignore_missing=('baz',))
+ self.assertEquals(root.get('bar'), "omgwtfbbq${baz}")
+
+ def testExpandIgnoreType(self):
+ root = struct.Struct()
+ root["foo"] = "bbq"
+ root["bar"] = "omgwtf${foo}"
+ root.expand(ignore_types=('strings',))
+ self.assertEquals(root.get('bar'), "omgwtf${foo}")
+ root["lfoo"] = struct.Link("foo")
+ root.expand(ignore_types=('links',))
+ self.assertEquals(root.get('bar'), "omgwtfbbq")
+ self.assert_(isinstance(root.get('lfoo'), struct.Link))
+ root.expand()
+ self.assert_(isinstance(root.get('lfoo'), basestring))
+ self.assertEquals(root.get('lfoo'), "bbq")
+
+ def testUnexpanded(self):
+ root = struct.Struct()
+ root["foo"] = "bbq"
+ root["bar"] = "omgwtf${foo}${baz}"
+ root.expand(ignore_missing=True)
+ self.assertEquals(root.unexpanded(), set(["baz"]))
+ self.assertEquals(root.unexpanded(True), set(["@root.baz"]))
+
+ def testExpandItemIgnore(self):
+ root = struct.Struct()
+ root["foo"] = "bbq"
+ root["bar"] = "omgwtf${foo}${baz}"
+ self.assertEquals(root.get('bar'), "omgwtf${foo}${baz}")
+ self.assertEquals(root.expanditem('bar', ignore_missing=('baz',)),
+ "omgwtfbbq${baz}")
+
+ def testExpandError(self):
+ root = struct.Struct()
+ root["bar"] = "omgwtf${foo}"
+ self.assertRaises(KeyError, root.expand)
+ self.assertEquals(root.get('bar'), "omgwtf${foo}")
+
+ def testExpandItemError(self):
+ root = struct.Struct()
+ root["bar"] = "omgwtf${foo}"
+ self.assertEquals(root.get('bar'), "omgwtf${foo}")
+ self.assertRaises(KeyError, root.expanditem, 'bar')
+ self.assertEquals(root.get('bar'), "omgwtf${foo}")
+
+ def testExpandInList(self):
+ root = struct.Struct()
+ root["foo"] = "bbq"
+ root["bar"] = [ "omgwtf${foo}" ]
+ self.assertEquals(root['bar'][0], "omgwtf${foo}")
+ root.expand()
+ self.assertEquals(root['bar'][0], "omgwtfbbq")
+
+ def testExpandInSubList(self):
+ root = struct.Struct()
+ root["foo"] = "bbq"
+ root["bar"] = [ [ "omgwtf${foo}" ] ]
+ self.assertEquals(root['bar'][0][0], "omgwtf${foo}")
+ root.expand()
+ self.assertEquals(root['bar'][0][0], "omgwtfbbq")
+
+ def testExpandMixed(self):
+ root = struct.Struct()
+ root["foo"] = "${bar}"
+ self.assertEquals(root.expanditem("foo", {'bar': "a"}), "a")
+ root["bar"] = "b"
+ self.assertEquals(root.expanditem("foo", {'bar': "a"}), "b")
+
+ def testCopy(self):
+ a = struct.Struct()
+ a["foo"] = [ "omgwtf${bar}" ]
+ a["bar"] = "a"
+ b = a.copy()
+ b["bar"] = "b"
+ self.assertEquals(a.expanditem("foo"), [ "omgwtfa" ])
+ self.assertEquals(b.expanditem("foo"), [ "omgwtfb" ])
+ a.expand()
+ b.expand()
+ self.assertEquals(a.get("foo"), [ "omgwtfa" ])
+ self.assertEquals(b.get("foo"), [ "omgwtfb" ])
+
+ def testSortKeys(self):
+ ukeys = ['z','r','a','c']
+ skeys = sorted(ukeys)
+ a = struct.Struct([(k,None) for k in ukeys])
+ self.assertEquals(a.keys(), ukeys)
+ self.assertNotEqual(a.keys(), skeys)
+ a.sort()
+ self.assertEquals(a.keys(), skeys)
+ self.assertNotEqual(a.keys(), ukeys)
+
+ def testSortValues(self):
+ def keycmp(a,b):
+ return cmp(a[1], b[1])
+ uvalues = ['z','r','a','c']
+ svalues = sorted(uvalues)
+ a = struct.Struct()
+ for i,v in enumerate(uvalues):
+ a["_%s" % i] = v
+ self.assertEquals(a.values(), uvalues)
+ self.assertNotEqual(a.values(), svalues)
+ a.sort(cmp=keycmp)
+ self.assertEquals(a.values(), svalues)
+ self.assertNotEqual(a.values(), uvalues)
+
+
+class StringTestCase(unittest.TestCase):
+ def testNestedList(self):
+ root = struct.Struct({'x': ['a', ['b', 'c']]})
+ self.assertEquals(str(root), 'x: ["a" ["b" "c"]]')
+
+ def testNormal(self):
+ root = struct.Struct({'x': {'y': None}})
+ self.assertEquals(str(root), 'x: {\n y: None\n}')
+
+ def testFlat(self):
+ root = struct.Struct({'x': {'y': None}})
+ self.assertEquals(root.flatten(), 'x.y: None')
diff --git a/coil/test/test_text.py b/coil/test/test_text.py
new file mode 100644
index 0000000..bcbb1ba
--- /dev/null
+++ b/coil/test/test_text.py
@@ -0,0 +1,253 @@
+"""Test text format."""
+
+import os
+import unittest
+from twisted.python.util import sibpath
+from coil import text, struct
+
+
+class TextTestCase(unittest.TestCase):
+
+ def testStringParse(self):
+ for structStr, value in (
+ [r'x: "\n\r \t\""', u'\n\r \t\"'],
+ [r'x: "hello"', u"hello"],
+ [r'x: "\\n"', ur"\n"],
+ ['x: "' + u"\u3456".encode("utf-8") + '"', u'\u3456'],
+ [r'x: "\" \ x"', u'" \ x'],
+ ):
+ x = text.fromString(structStr).get("x")
+ self.assertEquals(x, value)
+ self.assert_(isinstance(x, unicode))
+
+ def testIntParse(self):
+ for structStr, value in [
+ ('x: 1', 1),
+ ('x: 20909', 20909),
+ ('x: -34324', -34324),
+ ('x: 0', 0)]:
+ x = text.fromString(structStr).get("x")
+ self.assertEquals(x, value)
+
+ def testListParse(self):
+ for s, l in [#('x: [None 1 2.3 ["hello \\"world"] [7]]',
+ # [None, 1, 2.3, [u'hello "world'], [7]]),
+ ('x: ["a" "b"]', [u"a", u"b"])]:
+ self.assertEquals(text.fromString(s).get("x"), l)
+
+ def testComments(self):
+ s = "y: [12 #hello\n]"
+ self.assertEquals(text.fromString(s).get("y"), [12])
+
+ def testStruct(self):
+ s = '''
+struct: {
+ x: 12 y: 14
+ substruct: {
+ a: "hello world"
+ b: False
+ }
+}
+a-number: 2
+-moo: 3
+'''
+ root = text.fromString(s)
+ self.assertEquals(list(root.attributes()), ["struct", "a-number", "-moo"])
+ self.assertEquals(root.get("a-number"), 2)
+ self.assertEquals(root.get("-moo"), 3)
+ struct_ = root.get("struct")
+ self.assertEquals(list(struct_.attributes()), ["x", "y", "substruct"])
+ self.assertEquals(struct_.get("x"), 12)
+ substruct = struct_.get("substruct")
+ self.assertEquals(list(substruct.attributes()), ["a", "b"])
+ self.assertEquals(substruct.get("b"), False)
+
+ def testAttributePath(self):
+ s = '''
+struct: {
+ sub: {a: 1}
+ sub.b: 2
+ sub.c: 3
+ sub.d-e: 5
+}'''
+ root = text.fromString(s).get("struct")
+ self.assertEquals(root.get("sub").get("a"), 1)
+ self.assertEquals(root.get("sub").get("b"), 2)
+ self.assertEquals(root.get("sub").get("c"), 3)
+ self.assertEquals(root.get("sub").get("d-e"), 5)
+
+ def testBad(self):
+ for s in [
+ "struct: {",
+ "struct: }",
+ "a: b:",
+ ":",
+ "[]",
+ "a: ~b",
+ "@x: 2",
+ "x: 12c",
+ "x: 12.c3",
+ "x: @root",
+ "x: ..a",
+ 'x: {@package: "coil.test:nosuchfile"}',
+ 'x: {@file: "%s"}' % (sibpath(__file__, "nosuchfile"),),
+ 'x: {@package: "coil.test:test_text.py"}', # should get internal parse error
+ 'z: [{x: 2}]', # can't have struct in list
+ r'z: "lalalal \"', # string is not closed
+ 'a: 1 z: [ =@root.a ]',
+ 'a: {@extends: @root.b}', # b doesn't exist
+ 'a: {@extends: ..b}', # b doesn't exist
+ 'a: {@extends: x}',
+ 'a: {@extends: .}',
+ 'a: [1 2 3]]',
+ ]:
+ self.assertRaises(text.ParseError, text.fromString, s)
+
+ try:
+ text.fromString("x: 1\n2\n")
+ except text.ParseError, e:
+ self.assertEquals(e.line, 2)
+ self.assertEquals(e.column, 1)
+ else:
+ raise RuntimeError
+
+ def testDeleted(self):
+ s = '''
+struct1: {
+ a: {b: 1 x: 3}
+ c: 2
+ d: {b: 2}
+}
+struct2: {
+ @extends: ..struct1
+ ~c
+ ~a.b
+}
+~struct2.d
+'''
+ root = text.fromString(s).get("struct2")
+ self.assertEquals(list(root.attributes()), ["a"])
+ self.assertEquals(list(root.get("a").attributes()), ["x"])
+ self.assert_(not root.has_key("d"))
+ self.assert_(not root.has_key("c"))
+
+ def testLink(self):
+ s = '''
+struct: {
+ sub: {a: =..b c2: =c c: 1}
+ b: 2
+ c: =@root.x
+}
+x: "hello"
+'''
+ root = struct.StructNode(text.fromString(s))
+ self.assertEquals(root.struct.c, "hello")
+ self.assertEquals(root.struct.sub.a, 2)
+ self.assertEquals(root.struct.sub.c2, 1)
+
+ def testSimpleExtends(self):
+ s = '''
+bar: {
+ a: 1
+ b: 2
+ c: {d: 7}
+}
+
+foo: {
+ @extends: ..bar
+ a: 3
+ c: {
+ @extends: @root.bar.c
+ b2: =...bar.b
+ e: 4
+ }
+ c.f: 9 # nicer way of doing it
+}
+'''
+ foo = struct.StructNode(text.fromString(s)).foo
+ self.assertEquals(foo.a, 3)
+ self.assertEquals(foo.b, 2)
+ self.assertEquals(foo.c.d, 7)
+ self.assertEquals(foo.c.b2, 2)
+ self.assertEquals(foo.c.e, 4)
+ self.assertEquals(foo.c.f, 9)
+ for n in ("a", "b", "c"):
+ self.assert_(foo.has_key(n))
+ for n in ("d", "b2", "e", "f"):
+ self.assert_(foo.c.has_key(n))
+
+ def testStupidExtensionSemantics(self):
+ s = '''
+ base: {x: 1}
+ sub: {
+ @extends: ..base
+ }
+ base.y: 2 # sub should NOT have y
+ '''
+ s = text.fromString(s)
+ self.assertEquals(s.get("base").get("y"), 2)
+ self.assertEquals(s.get("sub").get("x"), 1)
+ self.assertRaises(struct.StructAttributeError, lambda: s.get("sub").get("y"))
+ self.assert_(not s.get("sub").has_key("y"))
+
+ def _testFile(self, root):
+ self.assertEquals(root.get("x"), 1)
+ self.assertEquals(root.get("y").get("a"), 2)
+
+ def testPathImport(self):
+ path = os.path.abspath(sibpath(__file__, "example.coil"))
+ s = 'x: {@file: "%s"}' % path
+ self._testFile(text.fromString(s).get("x"))
+ s = 'x: {@file: "example.coil"}'
+ self._testFile(text.fromString(s, __file__).get("x"))
+
+ def testPackageImport(self):
+ s = 'x: {@package: "coil:test/example.coil"}'
+ self._testFile(text.fromString(s).get("x"))
+ s = 'x: {@package: "coil.test:example.coil"}'
+ self._testFile(text.fromString(s).get("x"))
+
+ def testFileImport(self):
+ path = sibpath(__file__, "example2.coil")
+ s = text.fromFile(path)
+ node = struct.StructNode(s)
+ # make sure relative and absolute paths work and are relative
+ # to sub-struct that did the @file.
+ self.assertEquals(node.sub.y.x, u"foo")
+ self.assertEquals(node.sub.y.a2, u"bar")
+ self.assertEquals(node.sub2.y.a2, 2) # 'a' didn't get overriden this time
+ # XXX TODO
+ #self.assertEquals(node.sub3.a2, 2) # 'a' didn't get overriden this time
+
+ def testFileSubImport(self):
+ # @file can reference a sub-struct of imported file.
+ s = text.fromFile(sibpath(__file__, "filesubimport.coil"))
+ node = struct.StructNode(s)
+ # 0. Top-level import:
+ self.assertEquals(node.imp.sub.x, "default")
+ self.assertEquals(node.imp.sub.y, 2)
+ self.assertEquals(node.imp.sub.two.parentx, "default")
+ # 1. Single level sub-struct:
+ self.assertEquals(node.sub.x, "foo")
+ self.assertEquals(node.sub.y, 2)
+ self.assertEquals(node.sub.two.parentx, "foo")
+ # 2. Two level sub-struct:
+ self.assertEquals(node.subsub.parentx, "bar")
+ self.assertEquals(node.subsub.value, "hello")
+
+ def testFileImportAtTopLevel(self):
+ path = sibpath(__file__, "example3.coil")
+ s = text.fromFile(path)
+ node = struct.StructNode(s)
+ self.assertEquals(node.y.a, 2)
+ self.assertEquals(node.y.b, 3)
+ self.assertEquals(node.x, 1)
+
+ def testRootLinks(self):
+ s = """x: 1
+ y: {x: =@root.x}
+ z: {y2: {@extends: ...y}}"""
+ node = struct.StructNode(text.fromString(s))
+ self.assertEquals(node.x, 1)
+ self.assertEquals(node.y.x, 1)
+ self.assertEquals(node.z.y2.x, 1)
diff --git a/coil/test/test_tokenizer.py b/coil/test/test_tokenizer.py
new file mode 100644
index 0000000..48c4460
--- /dev/null
+++ b/coil/test/test_tokenizer.py
@@ -0,0 +1,125 @@
+"""Tests for coil.tokenizer."""
+
+import unittest
+from coil import tokenizer
+
+class TokenizerTestCase(unittest.TestCase):
+
+ def testEmpty(self):
+ tok = tokenizer.Tokenizer([""])
+ self.assertEquals(tok.next().type, 'EOF')
+
+ def testPath(self):
+ tok = tokenizer.Tokenizer(["somekey"])
+ first = tok.next()
+ self.assert_(isinstance(first, tokenizer.Token))
+ self.assertEquals(first.type, 'PATH')
+ self.assertEquals(first.value, "somekey")
+ self.assertEquals(first.line, 1)
+ self.assertEquals(first.column, 1)
+ self.assertEquals(tok.next().type, 'EOF')
+
+ def testString(self):
+ tok = tokenizer.Tokenizer(["'string'"])
+ first = tok.next()
+ self.assertEquals(first.type, 'VALUE')
+ self.assert_(isinstance(first.value, str))
+ self.assertEquals(first.value, "string")
+ self.assertEquals(first.line, 1)
+ self.assertEquals(first.column, 1)
+ self.assertEquals(tok.next().type, 'EOF')
+
+ def testUnocide(self):
+ tok = tokenizer.Tokenizer(
+ [u"'\u3456'".encode("utf-8")],
+ encoding='utf-8')
+ first = tok.next()
+ self.assertEquals(first.type, 'VALUE')
+ self.assert_(isinstance(first.value, unicode))
+ self.assertEquals(first.value, u"\u3456")
+ self.assertEquals(first.line, 1)
+ self.assertEquals(first.column, 1)
+ self.assertEquals(tok.next().type, 'EOF')
+
+ def testNumbers(self):
+ tok = tokenizer.Tokenizer(["1 2.0 -3 -4.0 0"])
+ token = tok.next()
+ self.assertEquals(token.type, 'VALUE')
+ self.assertEquals(token.value, 1)
+ self.assert_(isinstance(token.value, int))
+ token = tok.next()
+ self.assertEquals(token.type, 'VALUE')
+ self.assertEquals(token.value, 2.0)
+ self.assert_(isinstance(token.value, float))
+ token = tok.next()
+ self.assertEquals(token.type, 'VALUE')
+ self.assertEquals(token.value, -3)
+ self.assert_(isinstance(token.value, int))
+ token = tok.next()
+ self.assertEquals(token.type, 'VALUE')
+ self.assertEquals(token.value, -4)
+ self.assert_(isinstance(token.value, float))
+ token = tok.next()
+ self.assertEquals(token.type, 'VALUE')
+ self.assertEquals(token.value, 0)
+ self.assert_(isinstance(token.value, int))
+ self.assertEquals(tok.next().type, 'EOF')
+
+ def testBoolean(self):
+ tok = tokenizer.Tokenizer(["True False"])
+ token = tok.next()
+ self.assertEquals(token.type, 'VALUE')
+ self.assertEquals(token.value, True)
+ self.assert_(isinstance(token.value, bool))
+ token = tok.next()
+ self.assertEquals(token.type, 'VALUE')
+ self.assertEquals(token.value, False)
+ self.assert_(isinstance(token.value, bool))
+ self.assertEquals(tok.next().type, 'EOF')
+
+ def testNone(self):
+ tok = tokenizer.Tokenizer(["None"])
+ token = tok.next()
+ self.assertEquals(token.type, 'VALUE')
+ self.assertEquals(token.value, None)
+ self.assertEquals(tok.next().type, 'EOF')
+
+ def testCounters(self):
+ tok = tokenizer.Tokenizer(["'string' '''foo''' '' '''''' other",
+ "'''multi line string",
+ "it is crazy''' hi",
+ " bye"])
+ tok.next()
+ token = tok.next()
+ self.assertEquals(token.line, 1)
+ self.assertEquals(token.column, 10)
+ token = tok.next()
+ self.assertEquals(token.line, 1)
+ self.assertEquals(token.column, 20)
+ token = tok.next()
+ self.assertEquals(token.line, 1)
+ self.assertEquals(token.column, 23)
+ token = tok.next() # other
+ self.assertEquals(token.line, 1)
+ self.assertEquals(token.column, 30)
+ token = tok.next()
+ self.assertEquals(token.line, 2)
+ self.assertEquals(token.column, 1)
+ token = tok.next() # hi
+ self.assertEquals(token.line, 3)
+ self.assertEquals(token.column, 16)
+ token = tok.next() # bye
+ self.assertEquals(token.line, 4)
+ self.assertEquals(token.column, 3)
+ self.assertEquals(tok.next().type, 'EOF')
+
+ def testSpecialChars(self):
+ tok = tokenizer.Tokenizer(["{}[]:~="])
+ self.assertEquals(tok.next().type, '{')
+ self.assertEquals(tok.next().type, '}')
+ self.assertEquals(tok.next().type, '[')
+ self.assertEquals(tok.next().type, ']')
+ self.assertEquals(tok.next().type, ':')
+ self.assertEquals(tok.next().type, '~')
+ self.assertEquals(tok.next().type, '=')
+ self.assertEquals(tok.next().type, 'EOF')
diff --git a/coil/text.py b/coil/text.py
new file mode 100644
index 0000000..51602d6
--- /dev/null
+++ b/coil/text.py
@@ -0,0 +1,39 @@
+# Copyright (c) 2005-2006 Itamar Shtull-Trauring.
+# Copyright (c) 2008-2009 ITA Software, Inc.
+# See LICENSE.txt for details.
+
+"""Compatibility with <= 0.2.2, do not use in new code!"""
+
+from coil import parser, parse_file, errors
+
+ParseError = errors.CoilError
+
+#: Compatibility options for Parser to match the behavior of 0.2.2
+_compat = {'encoding': 'UTF-8',
+ 'ignore_types': ['strings'],
+ 'permissive': True}
+
+def fromSequence(iterOfStrings, filePath=None):
+ """Load a Struct from a sequence of strings.
+
+ @param filePath: path the strings were loaded from. Required for
+ relative @file arguments to work.
+ """
+ # The strings in 0.2.2 were allowed to contain newlines. We now
+ # expect the iter to be of lines, not arbitrary strings.
+ lines = []
+ for line in iterOfStrings:
+ lines += line.splitlines()
+ return parser.Parser(lines, filePath, **_compat).root()
+
+def fromString(st, filePath=None):
+ """Load a Struct from a string.
+
+ @param filePath: path the string was loaded from. Required for
+ relative @file arguments to work.
+ """
+ return parser.Parser(st.splitlines(), filePath, **_compat).root()
+
+def fromFile(path):
+ """Load a struct from a file, given a path on the filesystem."""
+ return parse_file(path, **_compat)
diff --git a/coil/tokenizer.py b/coil/tokenizer.py
new file mode 100644
index 0000000..df57043
--- /dev/null
+++ b/coil/tokenizer.py
@@ -0,0 +1,289 @@
+# Copyright (c) 2008-2009 ITA Software, Inc.
+# See LICENSE.txt for details.
+
+"""Break the input into a sequence of small tokens."""
+
+import re
+
+from coil import errors
+
+class Location(object):
+ """Represents a location in a file"""
+
+ def __init__(self, location=None):
+ if location:
+ self.filePath = location.filePath
+ self.line = location.line
+ self.column = location.column
+ else:
+ self.filePath = None
+ self.line = None
+ self.column = None
+
+class Token(Location):
+ """Represents a single token"""
+
+ #: Valid Token types
+ TYPES = ('{', '}', '[', ']', ':', '~', '=', 'PATH', 'VALUE', 'EOF')
+
+ def __init__(self, location, type_, value=None):
+ """
+ @param location: A L{Location} object that defines where this
+ token was found, typically this is the L{Tokenizer}.
+ @param type_: A string defining the type of token.
+ Must be one of the types listed in L{Token.TYPES}.
+ @param value: The string value of this token.
+ """
+ assert type_ in self.TYPES
+
+ self.type = type_
+ self.value = value
+ Location.__init__(self, location)
+
+ def __str__(self):
+ return "<%s: %s>" % (self.type, self.value)
+
+class Tokenizer(Location):
+ """Split input into basic tokens"""
+
+ # Note: Keys may start with any number of -'s but must be followed by a
+ # letter
+ KEY_REGEX = r'-*[a-zA-Z_][\w-]*'
+ PATH_REGEX = r'(@|\.+)?%s(\.%s)*' % (KEY_REGEX, KEY_REGEX)
+
+ PATH = re.compile(PATH_REGEX)
+ FLOAT = re.compile(r'-?[0-9]+\.[0-9]+')
+ INTEGER = re.compile(r'-?[0-9]+')
+ KEYWORD = re.compile(r'(True|False|None)')
+ WHITESPACE = re.compile(r'(#.*|\s+)')
+
+ # Strings are a bit tricky...
+ # The terminating quotes are optional for ''' quotes because
+ # they may span multiple lines. The rest of the voodoo is an
+ # attempt to allow escaping of quotes and require \ characters
+ # to always be paired with another character.
+ _STR1 = re.compile(r"'''((\\.|[^\\']|''?(?!'))*)(''')?")
+ _STR2 = re.compile(r'"""((\\.|[^\\"]|""?(?!"))*)(""")?')
+ _STR3 = re.compile(r"'((\\.|[^\\'])*)(')")
+ _STR4 = re.compile(r'"((\\.|[^\\"])*)(")')
+ _STRESC = re.compile(r'\\.')
+
+ def __init__(self, input_, filePath=None, encoding=None):
+ """
+ @param input_: An iterator over lines of input.
+ Typically a C{file} object or list of strings.
+ @param filePath: Path to input file, used for errors.
+ @param encoding: Read strings using the given encoding. All
+ string values will be C{unicode} objects rather than C{str}.
+ """
+
+ self.filePath = filePath
+ self.line = 0
+ self.column = 0
+ self._input = input_
+ self._buffer = ""
+ self._encoding = encoding
+ self._stack = []
+
+ # We iterate over the input in both next and _parse_string
+ self._next_line = self._next_line_generator().next
+
+ def _expect(self, token, types):
+ """Check that token has the correct type"""
+
+ assert types
+ for type_ in types:
+ assert type_ in Token.TYPES
+
+ if token.type not in types:
+ if token.type == token.value:
+ unexpected = repr(token.type)
+ else:
+ unexpected = "%s: %s" % (token.type, repr(token.value))
+
+ raise errors.CoilParseError(token,
+ "Unexpected %s, looking for %s" %
+ (unexpected, " ".join(types)))
+
+ def _push(self, token):
+ """Push a token back into the tokenizer"""
+
+ assert isinstance(token, Token)
+ self._stack.append(token)
+
+ def peek(self, *types):
+ """Peek at the next token but keep it in the tokenizer"""
+
+ token = self.next(*types)
+ self._push(token)
+ return token
+
+ def next(self, *types):
+ """Read the input in search of the next token"""
+
+ token = self._next()
+ if types:
+ self._expect(token, types)
+ return token
+
+ def _next(self):
+ """Only used by self.next()"""
+
+ if self._stack:
+ return self._stack.pop()
+
+ # Eat whitespace and comments
+ while True:
+ if not self._buffer:
+ try:
+ self._buffer = self._next_line()
+ except StopIteration:
+ return Token(self, 'EOF')
+
+
+ # Buffer should at least have a newline
+ assert self._buffer
+
+ # Skip over all whitespace and comments
+ match = self.WHITESPACE.match(self._buffer)
+ if match:
+ self._buffer = self._buffer[match.end():]
+ self.column += match.end()
+ else:
+ break
+
+ # Special characters
+ for tok in ('{', '}', '[', ']', ':', '~', '='):
+ if self._buffer[0] == tok:
+ token = Token(self, tok, tok)
+ self._buffer = self._buffer[1:]
+ self.column += 1
+ return token
+
+ match = self.FLOAT.match(self._buffer)
+ if match:
+ token = Token(self, 'VALUE', float(match.group(0)))
+ self._buffer = self._buffer[match.end():]
+ self.column += match.end()
+ return token
+
+ match = self.INTEGER.match(self._buffer)
+ if match:
+ token = Token(self, 'VALUE', int(match.group(0)))
+ self._buffer = self._buffer[match.end():]
+ self.column += match.end()
+ return token
+
+ match = self.KEYWORD.match(self._buffer)
+ if match:
+ if match.group(0) == "None":
+ token = Token(self, 'VALUE', None)
+ else:
+ token = Token(self, 'VALUE', bool(match.group(0) == 'True'))
+ self._buffer = self._buffer[match.end():]
+ self.column += match.end()
+ return token
+
+ match = self.PATH.match(self._buffer)
+ if match:
+ token = Token(self, 'PATH', match.group(0))
+ self._buffer = self._buffer[match.end():]
+ self.column += match.end()
+ return token
+
+ # Strings are special because they may span multiple lines
+ if self._buffer[0] in ('"', "'"):
+ return self._parse_string()
+
+ # Unknown input :-(
+ raise errors.CoilParseError(self,
+ "Unrecognized input: %s" % self._buffer)
+
+ def _next_line_generator(self):
+ for line in self._input:
+ if not line or line[-1] != '\n':
+ line = "%s\n" % line
+ self.line += 1
+ self.column = 1
+ yield line
+
+ def _escape_string(self, token):
+ replace = {
+ "\\\\": "\\",
+ "\\n": "\n",
+ "\\r": "\r",
+ "\\t": "\t",
+ "\\'": "'",
+ '\\"': '"',
+ }
+
+ def do_replace(match):
+ val = match.group(0)
+ key = str(val)
+ if key in replace:
+ return replace[key]
+ else:
+ return val
+
+ token.value = self._STRESC.sub(do_replace, token.value)
+
+ def _parse_string(self):
+ def decode(buf):
+ # If _encoding is set all strings should
+ # be unicode instead of str
+ if self._encoding:
+ try:
+ return buf.decode(self._encoding)
+ except UnicodeDecodeError, ex:
+ raise errors.CoilUnicodeError(self, str(ex))
+ else:
+ return buf
+
+ token = Token(self, 'VALUE')
+ strbuf = decode(self._buffer)
+ pattern = None
+
+ # Loop until the string is terminated
+ while True:
+ if not pattern:
+ # Find the correct string type
+ for pat in (self._STR1, self._STR2, self._STR3, self._STR4):
+ match = pat.match(strbuf)
+ if match:
+ pattern = pat
+ break
+ else:
+ match = pattern.match(strbuf)
+
+ if not match:
+ raise errors.CoilParseError(token, "Invalid string")
+
+ if not match.group(3):
+ # Read another line if string has no ending ''' or """
+ try:
+ new = self._next_line()
+ except StopIteration:
+ raise errors.CoilParseError(token, "Unterminated string")
+
+ strbuf += decode(new)
+ else:
+ token.value = match.group(1)
+ break
+
+ # Convert any escaped characters
+ self._escape_string(token)
+
+ # Fix up the column counter
+ try:
+ col = match.group(0).rindex('\n')
+ self.column = match.end() - col
+ except ValueError:
+ self.column += match.end()
+
+ # _buffer needs to be converted back to str
+ self._buffer = strbuf[match.end():]
+ if isinstance(self._buffer, unicode):
+ self._buffer = str(self._buffer.encode(self._encoding))
+ assert isinstance(self._buffer, str)
+
+ return token