spamegg
") ## >>> serialize(p.close()) ## 'spamegg
' Modified: lxml/trunk/src/lxml/cstd.pxd ============================================================================== --- lxml/trunk/src/lxml/cstd.pxd (original) +++ lxml/trunk/src/lxml/cstd.pxd Mon Sep 10 16:23:22 2007 @@ -13,6 +13,7 @@ cdef int strcmp(char* s1, char* s2) cdef int strncmp(char* s1, char* s2, size_t len) cdef void* memcpy(void* dest, void* src, size_t len) + cdef void* memset(void* s, int c, size_t len) cdef extern from "stdarg.h": ctypedef void *va_list Modified: lxml/trunk/src/lxml/docloader.pxi ============================================================================== --- lxml/trunk/src/lxml/docloader.pxi (original) +++ lxml/trunk/src/lxml/docloader.pxi Mon Sep 10 16:23:22 2007 @@ -94,9 +94,12 @@ cdef class _ResolverContext(_ExceptionContext): cdef _ResolverRegistry _resolvers cdef _TempStore _storage - def __init__(self, _ResolverRegistry resolvers not None): + def __init__(self, _ResolverRegistry resolvers): _ExceptionContext.__init__(self) - self._resolvers = resolvers + if resolvers is None: + self._resolvers = _ResolverRegistry() + else: + self._resolvers = resolvers self._storage = _TempStore() cdef void clear(self): Modified: lxml/trunk/src/lxml/dtd.pxi ============================================================================== --- lxml/trunk/src/lxml/dtd.pxi (original) +++ lxml/trunk/src/lxml/dtd.pxi Mon Sep 10 16:23:22 2007 @@ -88,10 +88,10 @@ cdef tree.xmlDtd* _parseDtdFromFilelike(file) except NULL: cdef _ExceptionContext exc_context - cdef _FileParserContext dtd_parser + cdef _FileReaderContext dtd_parser cdef tree.xmlDtd* c_dtd exc_context = _ExceptionContext() - dtd_parser = _FileParserContext(file, exc_context) + dtd_parser = _FileReaderContext(file, exc_context) c_dtd = dtd_parser._readDtd() Modified: lxml/trunk/src/lxml/etree.pyx ============================================================================== --- lxml/trunk/src/lxml/etree.pyx (original) +++ lxml/trunk/src/lxml/etree.pyx Mon Sep 10 16:23:22 2007 @@ -2131,19 +2131,20 @@ ################################################################################ # Include submodules -include "proxy.pxi" # Proxy handling (element backpointers/memory/etc.) -include "apihelpers.pxi" # Private helper functions -include "xmlerror.pxi" # Error and log handling -include "classlookup.pxi"# Element class lookup mechanisms -include "nsclasses.pxi" # Namespace implementation and registry -include "docloader.pxi" # Support for custom document loaders -include "parser.pxi" # XML Parser -include "serializer.pxi" # XML output functions -include "iterparse.pxi" # incremental XML parsing -include "xmlid.pxi" # XMLID and IDDict -include "extensions.pxi" # XPath/XSLT extension functions -include "xpath.pxi" # XPath evaluation -include "xslt.pxi" # XSL transformations +include "proxy.pxi" # Proxy handling (element backpointers/memory/etc.) +include "apihelpers.pxi" # Private helper functions +include "xmlerror.pxi" # Error and log handling +include "classlookup.pxi" # Element class lookup mechanisms +include "nsclasses.pxi" # Namespace implementation and registry +include "docloader.pxi" # Support for custom document loaders +include "parser.pxi" # XML Parser +include "parsertarget.pxi" # ET Parser target +include "serializer.pxi" # XML output functions +include "iterparse.pxi" # incremental XML parsing +include "xmlid.pxi" # XMLID and IDDict +include "extensions.pxi" # XPath/XSLT extension functions +include "xpath.pxi" # XPath evaluation +include "xslt.pxi" # XSL transformations ################################################################################ Modified: lxml/trunk/src/lxml/htmlparser.pxd ============================================================================== --- lxml/trunk/src/lxml/htmlparser.pxd (original) +++ lxml/trunk/src/lxml/htmlparser.pxd Mon Sep 10 16:23:22 2007 @@ -17,7 +17,11 @@ cdef xmlParserCtxt* htmlCreateMemoryParserCtxt(char* buffer, int size) cdef xmlParserCtxt* htmlCreateFileParserCtxt(char* filename, char* encoding) cdef void htmlFreeParserCtxt(xmlParserCtxt* ctxt) + cdef void htmlCtxtReset(xmlParserCtxt* ctxt) + cdef int htmlCtxtUseOptions(xmlParserCtxt* ctxt, int options) cdef int htmlParseDocument(xmlParserCtxt* ctxt) + cdef int htmlParseChunk(xmlParserCtxt* ctxt, + char* chunk, int size, int terminate) cdef xmlDoc* htmlCtxtReadFile(xmlParserCtxt* ctxt, char* filename, char* encoding, Modified: lxml/trunk/src/lxml/iterparse.pxi ============================================================================== --- lxml/trunk/src/lxml/iterparse.pxi (original) +++ lxml/trunk/src/lxml/iterparse.pxi Mon Sep 10 16:23:22 2007 @@ -48,7 +48,7 @@ c_ns = c_ns.next return count -cdef class _IterparseContext(_ResolverContext): +cdef class _IterparseContext(_ParserContext): cdef xmlparser.startElementNsSAX2Func _origSaxStart cdef xmlparser.endElementNsSAX2Func _origSaxEnd cdef _Element _root @@ -64,8 +64,8 @@ cdef char* _tag_href cdef char* _tag_name - def __init__(self, _ResolverRegistry resolvers): - _ResolverContext.__init__(self, resolvers) + def __init__(self): + _ParserContext.__init__(self) self._ns_stack = [] self._pop_ns = self._ns_stack.pop self._node_stack = [] @@ -73,22 +73,25 @@ self._events = [] self._event_index = 0 - cdef void _wrapCallbacks(self, xmlparser.xmlSAXHandler* sax): + cdef void _initParserContext(self, xmlparser.xmlParserCtxt* c_ctxt): "wrap original SAX2 callbacks" + cdef xmlparser.xmlSAXHandler* sax + _ParserContext._initParserContext(self, c_ctxt) + sax = c_ctxt.sax self._origSaxStart = sax.startElementNs # only override start event handler if needed if self._event_filter == 0 or \ self._event_filter & (ITERPARSE_FILTER_START | \ ITERPARSE_FILTER_START_NS | \ ITERPARSE_FILTER_END_NS): - sax.startElementNs = _saxStart + sax.startElementNs = _iterparseSaxStart self._origSaxEnd = sax.endElementNs # only override end event handler if needed if self._event_filter == 0 or \ self._event_filter & (ITERPARSE_FILTER_END | \ ITERPARSE_FILTER_END_NS): - sax.endElementNs = _saxEnd + sax.endElementNs = _iterparseSaxEnd cdef _setEventFilter(self, events, tag): self._event_filter = _buildIterparseEventFilter(events) @@ -184,9 +187,10 @@ cdef xmlparser.endElementNsSAX2Func _getOrigEnd(xmlparser.xmlParserCtxt* c_ctxt): return (<_IterparseContext>c_ctxt._private)._origSaxEnd -cdef void _saxStart(void* ctxt, char* localname, char* prefix, char* URI, - int nb_namespaces, char** namespaces, - int nb_attributes, int nb_defaulted, char** attributes): +cdef void _iterparseSaxStart(void* ctxt, char* localname, char* prefix, + char* URI, int nb_namespaces, char** namespaces, + int nb_attributes, int nb_defaulted, + char** attributes): # no Python in here! cdef xmlparser.xmlParserCtxt* c_ctxt cdef xmlparser.startElementNsSAX2Func origStart @@ -196,7 +200,7 @@ nb_attributes, nb_defaulted, attributes) _pushSaxStartEvent(c_ctxt, c_ctxt.node) -cdef void _saxEnd(void* ctxt, char* localname, char* prefix, char* URI): +cdef void _iterparseSaxEnd(void* ctxt, char* localname, char* prefix, char* URI): # no Python in here! cdef xmlparser.xmlParserCtxt* c_ctxt cdef xmlparser.endElementNsSAX2Func origEnd @@ -276,15 +280,17 @@ parse_options = parse_options | xmlparser.XML_PARSE_NOBLANKS _BaseParser.__init__(self, parse_options, remove_comments, remove_pis, - _IterparseContext) + None) context = <_IterparseContext>self._context context._setEventFilter(events, tag) - context._wrapCallbacks(self._parser_ctxt.sax) xmlparser.xmlCtxtUseOptions(self._parser_ctxt, parse_options) xmlparser.xmlCtxtResetPush(self._parser_ctxt, NULL, 0, c_filename, NULL) self._lockParser() # will not be unlocked - no other methods supported + cdef _ParserContext _createContext(self, target): + return _IterparseContext() + def __iter__(self): return self @@ -318,7 +324,8 @@ break if error != 0: self._source = None - _raiseParseError(self._parser_ctxt, self._filename, None) + _raiseParseError(self._parser_ctxt, self._filename, + self._context._error_log) if python.PyList_GET_SIZE(context._events) == 0: self.root = context._root self._source = None Modified: lxml/trunk/src/lxml/parser.pxi ============================================================================== --- lxml/trunk/src/lxml/parser.pxi (original) +++ lxml/trunk/src/lxml/parser.pxi Mon Sep 10 16:23:22 2007 @@ -2,7 +2,6 @@ cimport xmlparser cimport htmlparser -from xmlparser cimport xmlParserCtxt, xmlDict class ParseError(LxmlSyntaxError): """Syntax error while parsing an XML document. @@ -26,17 +25,17 @@ LXML_HTML_PARSER LXML_ITERPARSE_PARSER -cdef class _ParserContext: +cdef class _ParserDictionaryContext: # Global parser context to share the string dictionary. # - # This class is a singleton! + # This class is a delegate singleton! # - # It creates _ParserContext objects for each thread to keep thread state, + # It creates _ParserDictionaryContext objects for each thread to keep thread state, # but those must never be used directly. Always stick to using the static # __GLOBAL_PARSER_CONTEXT as defined below the class. # - cdef xmlDict* _c_dict + cdef tree.xmlDict* _c_dict cdef _BaseParser _default_parser def __dealloc__(self): if self._c_dict is not NULL: @@ -49,33 +48,33 @@ cdef python.PyObject* result thread_dict = python.PyThreadState_GetDict() if thread_dict is not NULL: - python.PyDict_SetItem(this is a paragraph
-##reserved characters: <&>
-##detta är också ett stycke
-## -## """ +def xmlwriter(): + r""" + >>> file = StringIO.StringIO() + >>> w = SimpleXMLWriter.XMLWriter(file) + >>> html = w.start("html") + >>> x = w.start("head") + >>> w.element("title", "my document") + >>> w.data("\n") + >>> w.element("meta", name="hello", value="goodbye") + >>> w.data("\n") + >>> w.end() + >>> x = w.start("body") + >>> w.element("h1", "this is a heading") + >>> w.data("\n") + >>> w.element("p", u"this is a paragraph") + >>> w.data("\n") + >>> w.element("p", u"reserved characters: <&>") + >>> w.data("\n") + >>> w.element("p", u"detta ?r ocks? ett stycke") + >>> w.data("\n") + >>> w.close(html) + >>> print file.getvalue() +this is a paragraph
+reserved characters: <&>
+detta är också ett stycke
+ + """ + +# doesn't work with lxml.etree +del xmlwriter # -------------------------------------------------------------------- # reported bugs -## def bug_xmltoolkit21(): -## """ -## marshaller gives obscure errors for non-string values +def bug_xmltoolkit21(): + """ + marshaller gives obscure errors for non-string values -## >>> elem = ElementTree.Element(123) -## >>> serialize(elem) # tag -## Traceback (most recent call last): -## TypeError: cannot serialize 123 (type int) -## >>> elem = ElementTree.Element("elem") -## >>> elem.text = 123 -## >>> serialize(elem) # text -## Traceback (most recent call last): -## TypeError: cannot serialize 123 (type int) -## >>> elem = ElementTree.Element("elem") -## >>> elem.tail = 123 -## >>> serialize(elem) # tail -## Traceback (most recent call last): -## TypeError: cannot serialize 123 (type int) -## >>> elem = ElementTree.Element("elem") -## >>> elem.set(123, "123") -## >>> serialize(elem) # attribute key -## Traceback (most recent call last): -## TypeError: cannot serialize 123 (type int) -## >>> elem = ElementTree.Element("elem") -## >>> elem.set("123", 123) -## >>> serialize(elem) # attribute value -## Traceback (most recent call last): -## TypeError: cannot serialize 123 (type int) + >>> elem = ElementTree.Element(123) + >>> serialize(elem) # tag + Traceback (most recent call last): + TypeError: cannot serialize 123 (type int) + >>> elem = ElementTree.Element("elem") + >>> elem.text = 123 + >>> serialize(elem) # text + Traceback (most recent call last): + TypeError: cannot serialize 123 (type int) + >>> elem = ElementTree.Element("elem") + >>> elem.tail = 123 + >>> serialize(elem) # tail + Traceback (most recent call last): + TypeError: cannot serialize 123 (type int) + >>> elem = ElementTree.Element("elem") + >>> elem.set(123, "123") + >>> serialize(elem) # attribute key + Traceback (most recent call last): + TypeError: cannot serialize 123 (type int) + >>> elem = ElementTree.Element("elem") + >>> elem.set("123", 123) + >>> serialize(elem) # attribute value + Traceback (most recent call last): + TypeError: cannot serialize 123 (type int) -## """ + """ + +# doesn't work with lxml.etree +del bug_xmltoolkit21 def bug_xmltoolkit25(): """ @@ -898,92 +993,199 @@ ['tbody'] """ -## def bug_xmltoolkitX1(): -## """ -## dump() doesn't flush the output buffer +def bug_xmltoolkitX1(): + """ + dump() doesn't flush the output buffer -## >>> tree = ElementTree.XML("v?lue
") -## >>> serialize(p.close()) -## 'välue
' + >>> tree = ElementTree.XML("v\xc3\xa4lue
") -## >>> serialize(p.close()) -## 'välue
' + >>> tree = ElementTree.Element(u"t?g") + >>> ElementTree.tostring(tree, "utf-8") + 'v\xc3\xa4lue
") -## >>> serialize(p.close().find("p")) -## 'välue
' - -## latin-1 character references -## >>> p = HTMLTreeBuilder.TreeBuilder() -## >>> p.feed("välue
") -## >>> serialize(p.close()) -## 'välue
' + >>> tree = ElementTree.Element("tag") + >>> tree.set(u"?ttr", u"v?lue") + >>> ElementTree.tostring(tree, "utf-8") + 'välue
") -## >>> serialize(p.close()) -## 'välue
' + """ -## mixed latin-1 text and unicode entities -## >>> p = HTMLTreeBuilder.TreeBuilder() -## >>> p.feed("”v?lue”
") -## >>> serialize(p.close()) -## '”välue”
' +# doesn't work with lxml.etree +del bug_xmltoolkit39 -## mixed unicode and latin-1 entities -## >>> p = HTMLTreeBuilder.TreeBuilder() -## >>> p.feed("”välue”
") -## >>> serialize(p.close()) -## '”välue”
' +def bug_xmltoolkit45(): + """ + problems parsing mixed unicode/non-ascii html documents -## """ + latin-1 text + >>> p = HTMLTreeBuilder.TreeBuilder() + >>> p.feed("v?lue
") + >>> serialize(p.close()) + 'välue
' + + utf-8 text + >>> p = HTMLTreeBuilder.TreeBuilder(encoding="utf-8") + >>> p.feed("v\xc3\xa4lue
") + >>> serialize(p.close()) + 'välue
' + + utf-8 text using meta tag + >>> p = HTMLTreeBuilder.TreeBuilder() + >>> p.feed("v\xc3\xa4lue
") + >>> serialize(p.close().find("p")) + 'välue
' + + latin-1 character references + >>> p = HTMLTreeBuilder.TreeBuilder() + >>> p.feed("välue
") + >>> serialize(p.close()) + 'välue
' + + latin-1 character entities + >>> p = HTMLTreeBuilder.TreeBuilder() + >>> p.feed("välue
") + >>> serialize(p.close()) + 'välue
' + + mixed latin-1 text and unicode entities + >>> p = HTMLTreeBuilder.TreeBuilder() + >>> p.feed("”v?lue”
") + >>> serialize(p.close()) + '”välue”
' + + mixed unicode and latin-1 entities + >>> p = HTMLTreeBuilder.TreeBuilder() + >>> p.feed("”välue”
") + >>> serialize(p.close()) + '”välue”
' + + """ + +# doesn't work with lxml.etree +del bug_xmltoolkit45 + +def bug_xmltoolkit46(): + """ + problems parsing open BR tags + + >>> p = HTMLTreeBuilder.TreeBuilder() + >>> p.feed("key
value
key
value
spamegg
") -## >>> serialize(p.close()) -## 'spamegg
' -## """ +def parsehtml(): + """ + Test HTML parsing. + + >>> # p = HTMLTreeBuilder.TreeBuilder() + >>> p = ElementTree.HTMLParser() + >>> p.feed("spamegg
") + >>> serialize(p.close()) + 'spamegg
' + """ + +# doesn't work with lxml.etree +del parsehtml def parseliteral(): r""" @@ -331,21 +361,24 @@ 'body' """ -## def simpleparsefile(): -## """ -## Test the xmllib-based parser. - -## >>> from elementtree import SimpleXMLTreeBuilder -## >>> parser = SimpleXMLTreeBuilder.TreeBuilder() -## >>> tree = ElementTree.parse("samples/simple.xml", parser) -## >>> normalize_crlf(tree) -## >>> tree.write(sys.stdout) -##html
test
html
test
- ... hi there - ... guy
- ...%s
' % utext + bom = '\xEF\xBB\xBF' + xml = bom + uxml.encode("utf-8") + tree = etree.XML(xml) + self.assertEquals(utext, tree.text) + def test_encoding_8bit_parse_stringio(self): utext = u'S?k p? nettet' uxml = u'%s
' % utext From scoder at codespeak.net Tue Sep 18 12:31:21 2007 From: scoder at codespeak.net (scoder at codespeak.net) Date: Tue, 18 Sep 2007 12:31:21 +0200 (CEST) Subject: [Lxml-checkins] r46717 - lxml/trunk/src/lxml Message-ID: <20070918103121.4D279813A@code0.codespeak.net> Author: scoder Date: Tue Sep 18 12:31:20 2007 New Revision: 46717 Modified: lxml/trunk/src/lxml/objectify.pyx Log: iterfind() in objectify Modified: lxml/trunk/src/lxml/objectify.pyx ============================================================================== --- lxml/trunk/src/lxml/objectify.pyx (original) +++ lxml/trunk/src/lxml/objectify.pyx Tue Sep 18 12:31:20 2007 @@ -363,6 +363,12 @@ sibling = self.__getitem__(key) parent.remove(sibling) + def iterfind(self, path): + # Reimplementation of Element.iterfind() to make it work without child + # iteration. + xpath = etree.ETXPath(path) + return iter(xpath(self)) + def findall(self, path): # Reimplementation of Element.findall() to make it work without child # iteration. From scoder at codespeak.net Tue Sep 18 12:48:00 2007 From: scoder at codespeak.net (scoder at codespeak.net) Date: Tue, 18 Sep 2007 12:48:00 +0200 (CEST) Subject: [Lxml-checkins] r46718 - lxml/trunk/src/lxml Message-ID: <20070918104800.DFECC8132@code0.codespeak.net> Author: scoder Date: Tue Sep 18 12:48:00 2007 New Revision: 46718 Modified: lxml/trunk/src/lxml/xslt.pxi Log: avoid XPath instantiation on startup Modified: lxml/trunk/src/lxml/xslt.pxi ============================================================================== --- lxml/trunk/src/lxml/xslt.pxi (original) +++ lxml/trunk/src/lxml/xslt.pxi Tue Sep 18 12:48:00 2007 @@ -598,10 +598,16 @@ cdef object _REPLACE_PI_HREF _REPLACE_PI_HREF = _RE_PI_HREF.sub -cdef XPath _findStylesheetByID -_findStylesheetByID = XPath( - "//xsl:stylesheet[@xml:id = $id]", - {"xsl":"http://www.w3.org/1999/XSL/Transform"}) +cdef XPath __findStylesheetByID +__findStylesheetByID = None + +cdef _findStylesheetByID(_Document doc, id): + global __findStylesheetByID + if __findStylesheetByID is None: + __findStylesheetByID = XPath( + "//xsl:stylesheet[@xml:id = $id]", + {"xsl" : "http://www.w3.org/1999/XSL/Transform"}) + return __findStylesheetByID(doc, id=id) cdef class _XSLTProcessingInstruction(PIBase): def parseXSL(self, parser=None): @@ -647,7 +653,7 @@ return _elementTreeFactory(result_node._doc, result_node) # try XPath search - root = _findStylesheetByID(self._doc, id=funicode(c_href)) + root = _findStylesheetByID(self._doc, funicode(c_href)) if not root: raise ValueError, "reference to non-existing embedded stylesheet" elif len(root) > 1: From scoder at codespeak.net Tue Sep 18 12:50:07 2007 From: scoder at codespeak.net (scoder at codespeak.net) Date: Tue, 18 Sep 2007 12:50:07 +0200 (CEST) Subject: [Lxml-checkins] r46719 - lxml/trunk/src/lxml Message-ID: <20070918105007.187EB8078@code0.codespeak.net> Author: scoder Date: Tue Sep 18 12:50:06 2007 New Revision: 46719 Modified: lxml/trunk/src/lxml/xpath.pxi Log: warn about libxml2 2.6.27 XPath bug Modified: lxml/trunk/src/lxml/xpath.pxi ============================================================================== --- lxml/trunk/src/lxml/xpath.pxi (original) +++ lxml/trunk/src/lxml/xpath.pxi Tue Sep 18 12:50:06 2007 @@ -91,6 +91,12 @@ cdef void _setupDict(self, xpath.xmlXPathContext* xpathCtxt): __GLOBAL_PARSER_CONTEXT.initXPathParserDict(xpathCtxt) +cdef int _XPATH_VERSION_WARNING_REQUIRED +if _LIBXML_VERSION_INT == 20627: + _XPATH_VERSION_WARNING_REQUIRED = 1 +else: + _XPATH_VERSION_WARNING_REQUIRED = 0 + cdef class _XPathEvaluatorBase: cdef xpath.xmlXPathContext* _xpathCtxt cdef _XPathContext _context @@ -98,6 +104,12 @@ cdef _ErrorLog _error_log def __init__(self, namespaces, extensions, enable_regexp): + global _XPATH_VERSION_WARNING_REQUIRED + if _XPATH_VERSION_WARNING_REQUIRED: + _XPATH_VERSION_WARNING_REQUIRED = 0 + import warnings + warnings.warn("This version of libxml2 has a known XPath bug. " + \ + "Use it at your own risk.") self._error_log = _ErrorLog() self._context = _XPathContext(namespaces, extensions, enable_regexp, None) From lxml-checkins at codespeak.net Tue Sep 18 21:34:14 2007 From: lxml-checkins at codespeak.net (Viagra.com Inc) Date: Tue, 18 Sep 2007 21:34:14 +0200 (CEST) Subject: [Lxml-checkins] September 70% OFF Message-ID: <20070918103555.4293.qmail@xdsl-355.elblag.dialog.net.pl> An HTML attachment was scrubbed... URL: http://codespeak.net/pipermail/lxml-checkins/attachments/20070918/44868afa/attachment.htm From scoder at codespeak.net Wed Sep 19 14:48:26 2007 From: scoder at codespeak.net (scoder at codespeak.net) Date: Wed, 19 Sep 2007 14:48:26 +0200 (CEST) Subject: [Lxml-checkins] r46730 - lxml/trunk/doc Message-ID: <20070919124826.82E0D816D@code0.codespeak.net> Author: scoder Date: Wed Sep 19 14:48:25 2007 New Revision: 46730 Modified: lxml/trunk/doc/parsing.txt lxml/trunk/doc/tutorial.txt Log: getiterator() -> iter() Modified: lxml/trunk/doc/parsing.txt ============================================================================== --- lxml/trunk/doc/parsing.txt (original) +++ lxml/trunk/doc/parsing.txt Wed Sep 19 14:48:25 2007 @@ -269,7 +269,7 @@ -------------------- As an extension over ElementTree, lxml.etree accepts a ``tag`` keyword -argument just like ``element.getiterator(tag)``. This restricts events to a +argument just like ``element.iter(tag)``. This restricts events to a specific tag or namespace:: >>> context = etree.iterparse(StringIO(xml), tag="element") Modified: lxml/trunk/doc/tutorial.txt ============================================================================== --- lxml/trunk/doc/tutorial.txt (original) +++ lxml/trunk/doc/tutorial.txt Wed Sep 19 14:48:25 2007 @@ -310,7 +310,7 @@%s
' % text).encode('iso-8859-1') + + tree = self.etree.parse( + StringIO(html_latin1), + self.etree.HTMLParser(encoding="iso-8859-1")) + p = tree.find("//p") + self.assertEquals(p.text, text) + + def test_parse_encoding_8bit_override(self): + text = u'S?k p? nettet' + wrong_head = ''' + + + ''' + html_latin1 = (u'%s%s
' % (wrong_head, + text) + ).encode('iso-8859-1') + + self.assertRaises(self.etree.ParseError, + self.etree.parse, + StringIO(html_latin1)) + + tree = self.etree.parse( + StringIO(html_latin1), + self.etree.HTMLParser(encoding="iso-8859-1")) + p = tree.find("//p") + self.assertEquals(p.text, text) + def test_module_HTML_broken(self): element = self.etree.HTML(self.broken_html_str) self.assertEqual(self.etree.tostring(element), Modified: lxml/trunk/src/lxml/tree.pxd ============================================================================== --- lxml/trunk/src/lxml/tree.pxd (original) +++ lxml/trunk/src/lxml/tree.pxd Thu Sep 20 14:13:48 2007 @@ -40,6 +40,7 @@ cdef int xmlCharEncCloseFunc(xmlCharEncodingHandler* handler) cdef xmlCharEncoding xmlDetectCharEncoding(char* text, int len) cdef char* xmlGetCharEncodingName(xmlCharEncoding enc) + cdef xmlCharEncoding xmlParseCharEncoding(char* name) cdef extern from "libxml/chvalid.h": cdef int xmlIsChar_ch(char c) From lxml-checkins at codespeak.net Thu Sep 20 22:27:59 2007 From: lxml-checkins at codespeak.net (Viagra.com Inc) Date: Thu, 20 Sep 2007 22:27:59 +0200 (CEST) Subject: [Lxml-checkins] Lovers package at discount price! Message-ID: <20070920112931.3344.qmail@host30-221-dynamic.11-79-r.retail.telecomitalia.it> An HTML attachment was scrubbed... URL: http://codespeak.net/pipermail/lxml-checkins/attachments/20070920/69e10e78/attachment.htm From scoder at codespeak.net Fri Sep 21 08:44:23 2007 From: scoder at codespeak.net (scoder at codespeak.net) Date: Fri, 21 Sep 2007 08:44:23 +0200 (CEST) Subject: [Lxml-checkins] r46774 - in lxml/trunk: . src/lxml src/lxml/tests Message-ID: <20070921064423.C8E058111@code0.codespeak.net> Author: scoder Date: Fri Sep 21 08:44:22 2007 New Revision: 46774 Modified: lxml/trunk/CHANGES.txt lxml/trunk/src/lxml/htmlparser.pxd lxml/trunk/src/lxml/iterparse.pxi lxml/trunk/src/lxml/parser.pxi lxml/trunk/src/lxml/tests/test_htmlparser.py lxml/trunk/src/lxml/xmlparser.pxd Log: preliminary HTML support for iterparse Modified: lxml/trunk/CHANGES.txt ============================================================================== --- lxml/trunk/CHANGES.txt (original) +++ lxml/trunk/CHANGES.txt Fri Sep 21 08:44:22 2007 @@ -8,6 +8,10 @@ Features added -------------- +* ``iterparse()`` accepts an ``html`` boolean keyword argument for + parsing with the HTML parser (note that this interface may be + subject to change) + * Parsers accept an ``encoding`` keyword argument that overrides the encoding of the parsed documents. Modified: lxml/trunk/src/lxml/htmlparser.pxd ============================================================================== --- lxml/trunk/src/lxml/htmlparser.pxd (original) +++ lxml/trunk/src/lxml/htmlparser.pxd Fri Sep 21 08:44:22 2007 @@ -1,6 +1,6 @@ from tree cimport xmlDoc, xmlDict from tree cimport xmlInputReadCallback, xmlInputCloseCallback -from xmlparser cimport xmlParserCtxt +from xmlparser cimport xmlParserCtxt, xmlSAXHandler from xmlerror cimport xmlError cdef extern from "libxml/HTMLparser.h": @@ -16,6 +16,10 @@ cdef xmlParserCtxt* htmlCreateMemoryParserCtxt(char* buffer, int size) cdef xmlParserCtxt* htmlCreateFileParserCtxt(char* filename, char* encoding) + cdef xmlParserCtxt* htmlCreatePushParserCtxt(xmlSAXHandler* sax, + void* user_data, + char* chunk, int size, + char* filename, int enc) cdef void htmlFreeParserCtxt(xmlParserCtxt* ctxt) cdef void htmlCtxtReset(xmlParserCtxt* ctxt) cdef int htmlCtxtUseOptions(xmlParserCtxt* ctxt, int options) Modified: lxml/trunk/src/lxml/iterparse.pxi ============================================================================== --- lxml/trunk/src/lxml/iterparse.pxi (original) +++ lxml/trunk/src/lxml/iterparse.pxi Fri Sep 21 08:44:22 2007 @@ -51,6 +51,8 @@ cdef class _IterparseContext(_ParserContext): cdef xmlparser.startElementNsSAX2Func _origSaxStart cdef xmlparser.endElementNsSAX2Func _origSaxEnd + cdef xmlparser.startElementSAXFunc _origSaxStartNoNs + cdef xmlparser.endElementSAXFunc _origSaxEndNoNs cdef _Element _root cdef _Document _doc cdef int _event_filter @@ -78,19 +80,23 @@ _ParserContext._initParserContext(self, c_ctxt) sax = c_ctxt.sax self._origSaxStart = sax.startElementNs + self._origSaxStartNoNs = sax.startElement # only override start event handler if needed if self._event_filter == 0 or \ self._event_filter & (ITERPARSE_FILTER_START | \ ITERPARSE_FILTER_START_NS | \ ITERPARSE_FILTER_END_NS): - sax.startElementNs = _iterparseSaxStart + sax.startElementNs = _iterparseSaxStart + sax.startElement = _iterparseSaxStartNoNs self._origSaxEnd = sax.endElementNs + self._origSaxEndNoNs = sax.endElement # only override end event handler if needed if self._event_filter == 0 or \ self._event_filter & (ITERPARSE_FILTER_END | \ ITERPARSE_FILTER_END_NS): sax.endElementNs = _iterparseSaxEnd + sax.endElement = _iterparseSaxEndNoNs cdef _setEventFilter(self, events, tag): self._event_filter = _buildIterparseEventFilter(events) @@ -183,9 +189,15 @@ cdef xmlparser.startElementNsSAX2Func _getOrigStart(xmlparser.xmlParserCtxt* c_ctxt): return (<_IterparseContext>c_ctxt._private)._origSaxStart +cdef xmlparser.startElementSAXFunc _getOrigStartNoNs(xmlparser.xmlParserCtxt* c_ctxt): + return (<_IterparseContext>c_ctxt._private)._origSaxStartNoNs + cdef xmlparser.endElementNsSAX2Func _getOrigEnd(xmlparser.xmlParserCtxt* c_ctxt): return (<_IterparseContext>c_ctxt._private)._origSaxEnd +cdef xmlparser.endElementSAXFunc _getOrigEndNoNs(xmlparser.xmlParserCtxt* c_ctxt): + return (<_IterparseContext>c_ctxt._private)._origSaxEndNoNs + cdef void _iterparseSaxStart(void* ctxt, char* localname, char* prefix, char* URI, int nb_namespaces, char** namespaces, int nb_attributes, int nb_defaulted, @@ -208,6 +220,24 @@ origEnd = _getOrigEnd(c_ctxt) origEnd(ctxt, localname, prefix, URI) +cdef void _iterparseSaxStartNoNs(void* ctxt, char* name, char** attributes): + # no Python in here! + cdef xmlparser.xmlParserCtxt* c_ctxt + cdef xmlparser.startElementSAXFunc origStart + c_ctxt =P
') + + iterator = iterparse(f, html=True) + self.assertEquals(None, iterator.root) + + events = list(iterator) + root = iterator.root + self.assert_(root is not None) + self.assertEquals( + [('end', root[0][0]), ('end', root[0]), ('end', root[1][0]), + ('end', root[1]), ('end', root)], + events) + + def test_html_iterparse_file(self): + iterparse = self.etree.iterparse + iterator = iterparse(fileInTestDir("css_shakespear.html"), + html=True) + + self.assertEquals(None, iterator.root) + events = list(iterator) + root = iterator.root + self.assert_(root is not None) + self.assertEquals(249, len(events)) + self.assertEquals( + [], + [ event for (event, element) in events if event != 'end' ]) + def test_suite(): suite = unittest.TestSuite() suite.addTests([unittest.makeSuite(HtmlParserTestCaseBase)]) Modified: lxml/trunk/src/lxml/xmlparser.pxd ============================================================================== --- lxml/trunk/src/lxml/xmlparser.pxd (original) +++ lxml/trunk/src/lxml/xmlparser.pxd Fri Sep 21 08:44:22 2007 @@ -19,6 +19,10 @@ char* prefix, char* URI) + ctypedef void (*startElementSAXFunc)(void* ctx, char* name, char** atts) + + ctypedef void (*endElementSAXFunc)(void* ctx, char* name) + ctypedef void (*charactersSAXFunc)(void* ctx, char* ch, int len) ctypedef void (*cdataBlockSAXFunc)(void* ctx, char* value, int len) @@ -49,6 +53,8 @@ internalSubsetSAXFunc internalSubset startElementNsSAX2Func startElementNs endElementNsSAX2Func endElementNs + startElementSAXFunc startElement + endElementSAXFunc endElement charactersSAXFunc characters cdataBlockSAXFunc cdataBlock commentSAXFunc comment @@ -170,3 +176,4 @@ cdef xmlParserInput* xmlNewInputFromFile(xmlParserCtxt* ctxt, char* filename) cdef void xmlFreeInputStream(xmlParserInput* input) + cdef int xmlSwitchEncoding(xmlParserCtxt* ctxt, int enc) From scoder at codespeak.net Fri Sep 21 09:34:01 2007 From: scoder at codespeak.net (scoder at codespeak.net) Date: Fri, 21 Sep 2007 09:34:01 +0200 (CEST) Subject: [Lxml-checkins] r46775 - in lxml/trunk: . doc Message-ID: <20070921073401.D8DE48150@code0.codespeak.net> Author: scoder Date: Fri Sep 21 09:34:00 2007 New Revision: 46775 Modified: lxml/trunk/doc/build.txt lxml/trunk/setup.py Log: require Cython 0.9.6.6 Modified: lxml/trunk/doc/build.txt ============================================================================== --- lxml/trunk/doc/build.txt (original) +++ lxml/trunk/doc/build.txt Fri Sep 21 09:34:00 2007 @@ -33,11 +33,11 @@ be an lxml developer, you do need a working Cython installation. You can use EasyInstall_ to install it:: - easy_install Cython==0.9.6.5 + easy_install Cython==0.9.6.6 .. _EasyInstall: http://peak.telecommunity.com/DevCenter/EasyInstall -lxml currently requires Cython 0.9.6.5, but it should work with later +lxml currently requires Cython 0.9.6.6, but it should work with later versions. Modified: lxml/trunk/setup.py ============================================================================== --- lxml/trunk/setup.py (original) +++ lxml/trunk/setup.py Fri Sep 21 09:34:00 2007 @@ -16,7 +16,7 @@ except pkg_resources.VersionConflict: from ez_setup import use_setuptools use_setuptools(version="0.6c5") - #pkg_resources.require("Cython==0.9.6.5") + #pkg_resources.require("Cython==0.9.6.6") from setuptools import setup extra_options["zip_safe"] = False except ImportError: From scoder at codespeak.net Fri Sep 21 10:16:44 2007 From: scoder at codespeak.net (scoder at codespeak.net) Date: Fri, 21 Sep 2007 10:16:44 +0200 (CEST) Subject: [Lxml-checkins] r46776 - lxml/trunk/src/lxml Message-ID: <20070921081644.2FD948097@code0.codespeak.net> Author: scoder Date: Fri Sep 21 10:16:42 2007 New Revision: 46776 Modified: lxml/trunk/src/lxml/objectify.pyx Log: public function objectify.pytypename() Modified: lxml/trunk/src/lxml/objectify.pyx ============================================================================== --- lxml/trunk/src/lxml/objectify.pyx (original) +++ lxml/trunk/src/lxml/objectify.pyx Fri Sep 21 10:16:42 2007 @@ -926,11 +926,16 @@ return _lower_bool(b) cdef _pytypename(obj): - if python.PyUnicode_Check(obj): + if python._isString(obj): return "str" else: return _typename(obj) +def pytypename(obj): + """Find the name of the corresponding PyType for a Python object. + """ + return _pytypename(obj) + cdef _registerPyTypes(): pytype = PyType('int', int, IntElement) pytype.xmlSchemaTypes = ("int", "short", "byte", "unsignedShort", From ianb at codespeak.net Fri Sep 21 19:31:55 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 21 Sep 2007 19:31:55 +0200 (CEST) Subject: [Lxml-checkins] r46801 - lxml/trunk/src/lxml/html Message-ID: <20070921173155.30C468162@code0.codespeak.net> Author: ianb Date: Fri Sep 21 19:31:53 2007 New Revision: 46801 Modified: lxml/trunk/src/lxml/html/setmixin.py Log: Because list() calls len(obj), implementing __len__ in terms of list() causes infinite recursion Modified: lxml/trunk/src/lxml/html/setmixin.py ============================================================================== --- lxml/trunk/src/lxml/html/setmixin.py (original) +++ lxml/trunk/src/lxml/html/setmixin.py Fri Sep 21 19:31:53 2007 @@ -5,7 +5,10 @@ """ def __len__(self): - return len(list(self)) + length = 0 + for item in self: + length += 1 + return length def __contains__(self, item): for has_item in self: From ianb at codespeak.net Fri Sep 21 19:33:15 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 21 Sep 2007 19:33:15 +0200 (CEST) Subject: [Lxml-checkins] r46802 - lxml/trunk/src/lxml/html Message-ID: <20070921173315.0CF8D8162@code0.codespeak.net> Author: ianb Date: Fri Sep 21 19:33:14 2007 New Revision: 46802 Modified: lxml/trunk/src/lxml/html/__init__.py Log: Some fixes to forms: ignore unnamed inputs when setting form.fields; don't try to delete keys when setting form.fields; be more careful about updating CheckboxGroup.value. Also, export more symbols in lxml.html.__all__ Modified: lxml/trunk/src/lxml/html/__init__.py ============================================================================== --- lxml/trunk/src/lxml/html/__init__.py (original) +++ lxml/trunk/src/lxml/html/__init__.py Fri Sep 21 19:33:14 2007 @@ -13,9 +13,11 @@ from lxml.html._dictmixin import DictMixin import sets -__all__ = ['document_fromstring', 'tostring', 'Element', 'defs', - 'find_rel_links', 'find_class', 'make_links_absolute', - 'resolve_base_href', 'iterlinks', 'rewrite_links', 'open_in_browser'] +__all__ = [ + 'document_fromstring', 'fragment_fromstring', 'fragments_fromstring', 'fromstring', + 'tostring', 'Element', 'defs', 'open_in_browser', 'submit_form', + 'find_rel_links', 'find_class', 'make_links_absolute', + 'resolve_base_href', 'iterlinks', 'rewrite_links', 'open_in_browser'] _rel_links_xpath = etree.XPath("descendant-or-self::a[@rel]") #_class_xpath = etree.XPath(r"descendant-or-self::*[regexp:match(@class, concat('\b', $class_name, '\b'))]", {'regexp': 'http://exslt.org/regular-expressions'}) @@ -583,10 +585,11 @@ prev_keys.remove(key) self.fields[key] = value for key in prev_keys: - # FIXME: but right now I don't even allow - # deleting, and I'm not sure what it would - # mean if I did. - del self.fields[key] + if key is None: + # Case of an unnamed input; these aren't really + # expressed in form_values() anyway. + continue + self.fields[key] = None fields = property(fields__get, fields__set, doc=fields__get.__doc__) @@ -1040,7 +1043,11 @@ return CheckboxValues(self) def value__set(self, value): self.value.clear() - self.value |= value + if not hasattr(value, '__iter__'): + raise ValueError( + "A CheckboxGroup (name=%r) must be set to a sequence (not %r)" + % (self[0].name, value)) + self.value.update(value) def value__del(self): self.value.clear() value = property(value__get, value__set, value__del, doc=value__get.__doc__) From ianb at codespeak.net Fri Sep 21 19:33:50 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 21 Sep 2007 19:33:50 +0200 (CEST) Subject: [Lxml-checkins] r46803 - lxml/trunk/doc Message-ID: <20070921173350.2C2CD8167@code0.codespeak.net> Author: ianb Date: Fri Sep 21 19:33:49 2007 New Revision: 46803 Modified: lxml/trunk/doc/lxmlhtml.txt Log: Fixed all the parsing references. Fix builder import. Added section on parsing. Added docs on forms. Modified: lxml/trunk/doc/lxmlhtml.txt ============================================================================== --- lxml/trunk/doc/lxmlhtml.txt (original) +++ lxml/trunk/doc/lxmlhtml.txt Fri Sep 21 19:33:49 2007 @@ -29,6 +29,35 @@ Parsing HTML fragments ---------------------- +There are several functions available to parse HTML: + +``parse(filename_url_or_file)``: + Parses the named file or url, or if the object has a ``.read()`` + method, parses from that. + + If you give a URL, or if the object has a ``.geturl()`` method (as + file-like objects from ``urllib.urlopen()`` have), then that URL + is used as the base URL. + +``document_fromstring(string)``: + Parses a document from the given string. This always creates a + correct HTML document, which means the parent node is ````, + and there is a body and possibly a head. + +``fragment_fromstring(string, create_parent=False)``: + Returns an HTML fragment from a string. The fragment must contain + just a single element, unless ``create_parent`` is given; + e.g,. ``fragment_fromstring(string, create_parent='div')`` will + wrap the element in a ``