1 | """Beautiful Soup
|
---|
2 | Elixir and Tonic
|
---|
3 | "The Screen-Scraper's Friend"
|
---|
4 | v2.1.1
|
---|
5 | http://www.crummy.com/software/BeautifulSoup/
|
---|
6 |
|
---|
7 | Beautiful Soup parses arbitrarily invalid XML- or HTML-like substance
|
---|
8 | into a tree representation. It provides methods and Pythonic idioms
|
---|
9 | that make it easy to search and modify the tree.
|
---|
10 |
|
---|
11 | A well-formed XML/HTML document will yield a well-formed data
|
---|
12 | structure. An ill-formed XML/HTML document will yield a
|
---|
13 | correspondingly ill-formed data structure. If your document is only
|
---|
14 | locally well-formed, you can use this library to find and process the
|
---|
15 | well-formed part of it. The BeautifulSoup class has heuristics for
|
---|
16 | obtaining a sensible parse tree in the face of common HTML errors.
|
---|
17 |
|
---|
18 | Beautiful Soup has no external dependencies. It works with Python 2.2
|
---|
19 | and up.
|
---|
20 |
|
---|
21 | Beautiful Soup defines classes for four different parsing strategies:
|
---|
22 |
|
---|
23 | * BeautifulStoneSoup, for parsing XML, SGML, or your domain-specific
|
---|
24 | language that kind of looks like XML.
|
---|
25 |
|
---|
26 | * BeautifulSoup, for parsing run-of-the-mill HTML code, be it valid
|
---|
27 | or invalid.
|
---|
28 |
|
---|
29 | * ICantBelieveItsBeautifulSoup, for parsing valid but bizarre HTML
|
---|
30 | that trips up BeautifulSoup.
|
---|
31 |
|
---|
32 | * BeautifulSOAP, for making it easier to parse XML documents that use
|
---|
33 | lots of subelements containing a single string, where you'd prefer
|
---|
34 | they put that string into an attribute (such as SOAP messages).
|
---|
35 |
|
---|
36 | You can subclass BeautifulStoneSoup or BeautifulSoup to create a
|
---|
37 | parsing strategy specific to an XML schema or a particular bizarre
|
---|
38 | HTML document. Typically your subclass would just override
|
---|
39 | SELF_CLOSING_TAGS and/or NESTABLE_TAGS.
|
---|
40 | """ #"
|
---|
41 | from __future__ import generators
|
---|
42 |
|
---|
43 | __author__ = "Leonard Richardson (leonardr@segfault.org)"
|
---|
44 | __version__ = "2.1.1"
|
---|
45 | __date__ = "$Date: 2004/10/18 00:14:20 $"
|
---|
46 | __copyright__ = "Copyright (c) 2004-2005 Leonard Richardson"
|
---|
47 | __license__ = "PSF"
|
---|
48 |
|
---|
49 | from sgmllib import SGMLParser, SGMLParseError
|
---|
50 | import types
|
---|
51 | import re
|
---|
52 | import sgmllib
|
---|
53 |
|
---|
54 | #This code makes Beautiful Soup able to parse XML with namespaces
|
---|
55 | sgmllib.tagfind = re.compile('[a-zA-Z][-_.:a-zA-Z0-9]*')
|
---|
56 |
|
---|
57 | class NullType(object):
|
---|
58 |
|
---|
59 | """Similar to NoneType with a corresponding singleton instance
|
---|
60 | 'Null' that, unlike None, accepts any message and returns itself.
|
---|
61 |
|
---|
62 | Examples:
|
---|
63 | >>> Null("send", "a", "message")("and one more",
|
---|
64 | ... "and what you get still") is Null
|
---|
65 | True
|
---|
66 | """
|
---|
67 |
|
---|
68 | def __new__(cls): return Null
|
---|
69 | def __call__(self, *args, **kwargs): return Null
|
---|
70 | ## def __getstate__(self, *args): return Null
|
---|
71 | def __getattr__(self, attr): return Null
|
---|
72 | def __getitem__(self, item): return Null
|
---|
73 | def __setattr__(self, attr, value): pass
|
---|
74 | def __setitem__(self, item, value): pass
|
---|
75 | def __len__(self): return 0
|
---|
76 | # FIXME: is this a python bug? otherwise ``for x in Null: pass``
|
---|
77 | # never terminates...
|
---|
78 | def __iter__(self): return iter([])
|
---|
79 | def __contains__(self, item): return False
|
---|
80 | def __repr__(self): return "Null"
|
---|
81 | Null = object.__new__(NullType)
|
---|
82 |
|
---|
83 | class PageElement:
|
---|
84 | """Contains the navigational information for some part of the page
|
---|
85 | (either a tag or a piece of text)"""
|
---|
86 |
|
---|
87 | def setup(self, parent=Null, previous=Null):
|
---|
88 | """Sets up the initial relations between this element and
|
---|
89 | other elements."""
|
---|
90 | self.parent = parent
|
---|
91 | self.previous = previous
|
---|
92 | self.next = Null
|
---|
93 | self.previousSibling = Null
|
---|
94 | self.nextSibling = Null
|
---|
95 | if self.parent and self.parent.contents:
|
---|
96 | self.previousSibling = self.parent.contents[-1]
|
---|
97 | self.previousSibling.nextSibling = self
|
---|
98 |
|
---|
99 | def findNext(self, name=None, attrs={}, text=None):
|
---|
100 | """Returns the first item that matches the given criteria and
|
---|
101 | appears after this Tag in the document."""
|
---|
102 | return self._first(self.fetchNext, name, attrs, text)
|
---|
103 | firstNext = findNext
|
---|
104 |
|
---|
105 | def fetchNext(self, name=None, attrs={}, text=None, limit=None):
|
---|
106 | """Returns all items that match the given criteria and appear
|
---|
107 | before after Tag in the document."""
|
---|
108 | return self._fetch(name, attrs, text, limit, self.nextGenerator)
|
---|
109 |
|
---|
110 | def findNextSibling(self, name=None, attrs={}, text=None):
|
---|
111 | """Returns the closest sibling to this Tag that matches the
|
---|
112 | given criteria and appears after this Tag in the document."""
|
---|
113 | return self._first(self.fetchNextSiblings, name, attrs, text)
|
---|
114 | firstNextSibling = findNextSibling
|
---|
115 |
|
---|
116 | def fetchNextSiblings(self, name=None, attrs={}, text=None, limit=None):
|
---|
117 | """Returns the siblings of this Tag that match the given
|
---|
118 | criteria and appear after this Tag in the document."""
|
---|
119 | return self._fetch(name, attrs, text, limit, self.nextSiblingGenerator)
|
---|
120 |
|
---|
121 | def findPrevious(self, name=None, attrs={}, text=None):
|
---|
122 | """Returns the first item that matches the given criteria and
|
---|
123 | appears before this Tag in the document."""
|
---|
124 | return self._first(self.fetchPrevious, name, attrs, text)
|
---|
125 |
|
---|
126 | def fetchPrevious(self, name=None, attrs={}, text=None, limit=None):
|
---|
127 | """Returns all items that match the given criteria and appear
|
---|
128 | before this Tag in the document."""
|
---|
129 | return self._fetch(name, attrs, text, limit, self.previousGenerator)
|
---|
130 | firstPrevious = findPrevious
|
---|
131 |
|
---|
132 | def findPreviousSibling(self, name=None, attrs={}, text=None):
|
---|
133 | """Returns the closest sibling to this Tag that matches the
|
---|
134 | given criteria and appears before this Tag in the document."""
|
---|
135 | return self._first(self.fetchPreviousSiblings, name, attrs, text)
|
---|
136 | firstPreviousSibling = findPreviousSibling
|
---|
137 |
|
---|
138 | def fetchPreviousSiblings(self, name=None, attrs={}, text=None,
|
---|
139 | limit=None):
|
---|
140 | """Returns the siblings of this Tag that match the given
|
---|
141 | criteria and appear before this Tag in the document."""
|
---|
142 | return self._fetch(name, attrs, text, limit,
|
---|
143 | self.previousSiblingGenerator)
|
---|
144 |
|
---|
145 | def findParent(self, name=None, attrs={}):
|
---|
146 | """Returns the closest parent of this Tag that matches the given
|
---|
147 | criteria."""
|
---|
148 | r = Null
|
---|
149 | l = self.fetchParents(name, attrs, 1)
|
---|
150 | if l:
|
---|
151 | r = l[0]
|
---|
152 | return r
|
---|
153 | firstParent = findParent
|
---|
154 |
|
---|
155 | def fetchParents(self, name=None, attrs={}, limit=None):
|
---|
156 | """Returns the parents of this Tag that match the given
|
---|
157 | criteria."""
|
---|
158 | return self._fetch(name, attrs, None, limit, self.parentGenerator)
|
---|
159 |
|
---|
160 | #These methods do the real heavy lifting.
|
---|
161 |
|
---|
162 | def _first(self, method, name, attrs, text):
|
---|
163 | r = Null
|
---|
164 | l = method(name, attrs, text, 1)
|
---|
165 | if l:
|
---|
166 | r = l[0]
|
---|
167 | return r
|
---|
168 |
|
---|
169 | def _fetch(self, name, attrs, text, limit, generator):
|
---|
170 | "Iterates over a generator looking for things that match."
|
---|
171 | if not hasattr(attrs, 'items'):
|
---|
172 | attrs = {'class' : attrs}
|
---|
173 |
|
---|
174 | results = []
|
---|
175 | g = generator()
|
---|
176 | while True:
|
---|
177 | try:
|
---|
178 | i = g.next()
|
---|
179 | except StopIteration:
|
---|
180 | break
|
---|
181 | found = None
|
---|
182 | if isinstance(i, Tag):
|
---|
183 | if not text:
|
---|
184 | if not name or self._matches(i, name):
|
---|
185 | match = True
|
---|
186 | for attr, matchAgainst in attrs.items():
|
---|
187 | check = i.get(attr)
|
---|
188 | if not self._matches(check, matchAgainst):
|
---|
189 | match = False
|
---|
190 | break
|
---|
191 | if match:
|
---|
192 | found = i
|
---|
193 | elif text:
|
---|
194 | if self._matches(i, text):
|
---|
195 | found = i
|
---|
196 | if found:
|
---|
197 | results.append(found)
|
---|
198 | if limit and len(results) >= limit:
|
---|
199 | break
|
---|
200 | return results
|
---|
201 |
|
---|
202 | #Generators that can be used to navigate starting from both
|
---|
203 | #NavigableTexts and Tags.
|
---|
204 | def nextGenerator(self):
|
---|
205 | i = self
|
---|
206 | while i:
|
---|
207 | i = i.next
|
---|
208 | yield i
|
---|
209 |
|
---|
210 | def nextSiblingGenerator(self):
|
---|
211 | i = self
|
---|
212 | while i:
|
---|
213 | i = i.nextSibling
|
---|
214 | yield i
|
---|
215 |
|
---|
216 | def previousGenerator(self):
|
---|
217 | i = self
|
---|
218 | while i:
|
---|
219 | i = i.previous
|
---|
220 | yield i
|
---|
221 |
|
---|
222 | def previousSiblingGenerator(self):
|
---|
223 | i = self
|
---|
224 | while i:
|
---|
225 | i = i.previousSibling
|
---|
226 | yield i
|
---|
227 |
|
---|
228 | def parentGenerator(self):
|
---|
229 | i = self
|
---|
230 | while i:
|
---|
231 | i = i.parent
|
---|
232 | yield i
|
---|
233 |
|
---|
234 | def _matches(self, chunk, howToMatch):
|
---|
235 | #print 'looking for %s in %s' % (howToMatch, chunk)
|
---|
236 | #
|
---|
237 | # If given a list of items, return true if the list contains a
|
---|
238 | # text element that matches.
|
---|
239 | if isList(chunk) and not isinstance(chunk, Tag):
|
---|
240 | for tag in chunk:
|
---|
241 | if isinstance(tag, NavigableText) and self._matches(tag, howToMatch):
|
---|
242 | return True
|
---|
243 | return False
|
---|
244 | if callable(howToMatch):
|
---|
245 | return howToMatch(chunk)
|
---|
246 | if isinstance(chunk, Tag):
|
---|
247 | #Custom match methods take the tag as an argument, but all other
|
---|
248 | #ways of matching match the tag name as a string
|
---|
249 | chunk = chunk.name
|
---|
250 | #Now we know that chunk is a string
|
---|
251 | if not isinstance(chunk, basestring):
|
---|
252 | chunk = str(chunk)
|
---|
253 | if hasattr(howToMatch, 'match'):
|
---|
254 | # It's a regexp object.
|
---|
255 | return howToMatch.search(chunk)
|
---|
256 | if isList(howToMatch):
|
---|
257 | return chunk in howToMatch
|
---|
258 | if hasattr(howToMatch, 'items'):
|
---|
259 | return howToMatch.has_key(chunk)
|
---|
260 | #It's just a string
|
---|
261 | return str(howToMatch) == chunk
|
---|
262 |
|
---|
263 | class NavigableText(PageElement):
|
---|
264 |
|
---|
265 | def __getattr__(self, attr):
|
---|
266 | "For backwards compatibility, text.string gives you text"
|
---|
267 | if attr == 'string':
|
---|
268 | return self
|
---|
269 | else:
|
---|
270 | raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__.__name__, attr)
|
---|
271 |
|
---|
272 | class NavigableString(str, NavigableText):
|
---|
273 | pass
|
---|
274 |
|
---|
275 | class NavigableUnicodeString(unicode, NavigableText):
|
---|
276 | pass
|
---|
277 |
|
---|
278 | class Tag(PageElement):
|
---|
279 |
|
---|
280 | """Represents a found HTML tag with its attributes and contents."""
|
---|
281 |
|
---|
282 | def __init__(self, name, attrs=None, parent=Null, previous=Null):
|
---|
283 | "Basic constructor."
|
---|
284 | self.name = name
|
---|
285 | if attrs == None:
|
---|
286 | attrs = []
|
---|
287 | self.attrs = attrs
|
---|
288 | self.contents = []
|
---|
289 | self.setup(parent, previous)
|
---|
290 | self.hidden = False
|
---|
291 |
|
---|
292 | def get(self, key, default=None):
|
---|
293 | """Returns the value of the 'key' attribute for the tag, or
|
---|
294 | the value given for 'default' if it doesn't have that
|
---|
295 | attribute."""
|
---|
296 | return self._getAttrMap().get(key, default)
|
---|
297 |
|
---|
298 | def __getitem__(self, key):
|
---|
299 | """tag[key] returns the value of the 'key' attribute for the tag,
|
---|
300 | and throws an exception if it's not there."""
|
---|
301 | return self._getAttrMap()[key]
|
---|
302 |
|
---|
303 | def __iter__(self):
|
---|
304 | "Iterating over a tag iterates over its contents."
|
---|
305 | return iter(self.contents)
|
---|
306 |
|
---|
307 | def __len__(self):
|
---|
308 | "The length of a tag is the length of its list of contents."
|
---|
309 | return len(self.contents)
|
---|
310 |
|
---|
311 | def __contains__(self, x):
|
---|
312 | return x in self.contents
|
---|
313 |
|
---|
314 | def __nonzero__(self):
|
---|
315 | "A tag is non-None even if it has no contents."
|
---|
316 | return True
|
---|
317 |
|
---|
318 | def __setitem__(self, key, value):
|
---|
319 | """Setting tag[key] sets the value of the 'key' attribute for the
|
---|
320 | tag."""
|
---|
321 | self._getAttrMap()
|
---|
322 | self.attrMap[key] = value
|
---|
323 | found = False
|
---|
324 | for i in range(0, len(self.attrs)):
|
---|
325 | if self.attrs[i][0] == key:
|
---|
326 | self.attrs[i] = (key, value)
|
---|
327 | found = True
|
---|
328 | if not found:
|
---|
329 | self.attrs.append((key, value))
|
---|
330 | self._getAttrMap()[key] = value
|
---|
331 |
|
---|
332 | def __delitem__(self, key):
|
---|
333 | "Deleting tag[key] deletes all 'key' attributes for the tag."
|
---|
334 | for item in self.attrs:
|
---|
335 | if item[0] == key:
|
---|
336 | self.attrs.remove(item)
|
---|
337 | #We don't break because bad HTML can define the same
|
---|
338 | #attribute multiple times.
|
---|
339 | self._getAttrMap()
|
---|
340 | if self.attrMap.has_key(key):
|
---|
341 | del self.attrMap[key]
|
---|
342 |
|
---|
343 | def __call__(self, *args, **kwargs):
|
---|
344 | """Calling a tag like a function is the same as calling its
|
---|
345 | fetch() method. Eg. tag('a') returns a list of all the A tags
|
---|
346 | found within this tag."""
|
---|
347 | return apply(self.fetch, args, kwargs)
|
---|
348 |
|
---|
349 | def __getattr__(self, tag):
|
---|
350 | if len(tag) > 3 and tag.rfind('Tag') == len(tag) - 3:
|
---|
351 | return self.first(tag[:-3])
|
---|
352 | elif tag.find('__') != 0:
|
---|
353 | return self.first(tag)
|
---|
354 |
|
---|
355 | def __eq__(self, other):
|
---|
356 | """Returns true iff this tag has the same name, the same attributes,
|
---|
357 | and the same contents (recursively) as the given tag.
|
---|
358 |
|
---|
359 | NOTE: right now this will return false if two tags have the
|
---|
360 | same attributes in a different order. Should this be fixed?"""
|
---|
361 | if not hasattr(other, 'name') or not hasattr(other, 'attrs') or not hasattr(other, 'contents') or self.name != other.name or self.attrs != other.attrs or len(self) != len(other):
|
---|
362 | return False
|
---|
363 | for i in range(0, len(self.contents)):
|
---|
364 | if self.contents[i] != other.contents[i]:
|
---|
365 | return False
|
---|
366 | return True
|
---|
367 |
|
---|
368 | def __ne__(self, other):
|
---|
369 | """Returns true iff this tag is not identical to the other tag,
|
---|
370 | as defined in __eq__."""
|
---|
371 | return not self == other
|
---|
372 |
|
---|
373 | def __repr__(self):
|
---|
374 | """Renders this tag as a string."""
|
---|
375 | return str(self)
|
---|
376 |
|
---|
377 | def __unicode__(self):
|
---|
378 | return self.__str__(1)
|
---|
379 |
|
---|
380 | def __str__(self, needUnicode=None, showStructureIndent=None):
|
---|
381 | """Returns a string or Unicode representation of this tag and
|
---|
382 | its contents.
|
---|
383 |
|
---|
384 | NOTE: since Python's HTML parser consumes whitespace, this
|
---|
385 | method is not certain to reproduce the whitespace present in
|
---|
386 | the original string."""
|
---|
387 |
|
---|
388 | attrs = []
|
---|
389 | if self.attrs:
|
---|
390 | for key, val in self.attrs:
|
---|
391 | attrs.append('%s="%s"' % (key, val))
|
---|
392 | close = ''
|
---|
393 | closeTag = ''
|
---|
394 | if self.isSelfClosing():
|
---|
395 | close = ' /'
|
---|
396 | else:
|
---|
397 | closeTag = '</%s>' % self.name
|
---|
398 | indentIncrement = None
|
---|
399 | if showStructureIndent != None:
|
---|
400 | indentIncrement = showStructureIndent
|
---|
401 | if not self.hidden:
|
---|
402 | indentIncrement += 1
|
---|
403 | contents = self.renderContents(indentIncrement, needUnicode=needUnicode)
|
---|
404 | if showStructureIndent:
|
---|
405 | space = '\n%s' % (' ' * showStructureIndent)
|
---|
406 | if self.hidden:
|
---|
407 | s = contents
|
---|
408 | else:
|
---|
409 | s = []
|
---|
410 | attributeString = ''
|
---|
411 | if attrs:
|
---|
412 | attributeString = ' ' + ' '.join(attrs)
|
---|
413 | if showStructureIndent:
|
---|
414 | s.append(space)
|
---|
415 | s.append('<%s%s%s>' % (self.name, attributeString, close))
|
---|
416 | s.append(contents)
|
---|
417 | if closeTag and showStructureIndent != None:
|
---|
418 | s.append(space)
|
---|
419 | s.append(closeTag)
|
---|
420 | s = ''.join(s)
|
---|
421 | isUnicode = type(s) == types.UnicodeType
|
---|
422 | if needUnicode and not isUnicode:
|
---|
423 | s = unicode(s)
|
---|
424 | elif isUnicode and needUnicode == False:
|
---|
425 | s = str(s)
|
---|
426 | return s
|
---|
427 |
|
---|
428 | def prettify(self, needUnicode=None):
|
---|
429 | return self.__str__(needUnicode, showStructureIndent=True)
|
---|
430 |
|
---|
431 | def renderContents(self, showStructureIndent=None, needUnicode=None):
|
---|
432 | """Renders the contents of this tag as a (possibly Unicode)
|
---|
433 | string."""
|
---|
434 | s = []
|
---|
435 | for c in self:
|
---|
436 | text = None
|
---|
437 | if isinstance(c, NavigableUnicodeString) or type(c) == types.UnicodeType:
|
---|
438 | text = unicode(c)
|
---|
439 | elif isinstance(c, Tag):
|
---|
440 | s.append(c.__str__(needUnicode, showStructureIndent))
|
---|
441 | elif needUnicode:
|
---|
442 | text = unicode(c)
|
---|
443 | else:
|
---|
444 | text = str(c)
|
---|
445 | if text:
|
---|
446 | if showStructureIndent != None:
|
---|
447 | if text[-1] == '\n':
|
---|
448 | text = text[:-1]
|
---|
449 | s.append(text)
|
---|
450 | return ''.join(s)
|
---|
451 |
|
---|
452 | #Soup methods
|
---|
453 |
|
---|
454 | def firstText(self, text, recursive=True):
|
---|
455 | """Convenience method to retrieve the first piece of text matching the
|
---|
456 | given criteria. 'text' can be a string, a regular expression object,
|
---|
457 | a callable that takes a string and returns whether or not the
|
---|
458 | string 'matches', etc."""
|
---|
459 | return self.first(recursive=recursive, text=text)
|
---|
460 |
|
---|
461 | def fetchText(self, text, recursive=True, limit=None):
|
---|
462 | """Convenience method to retrieve all pieces of text matching the
|
---|
463 | given criteria. 'text' can be a string, a regular expression object,
|
---|
464 | a callable that takes a string and returns whether or not the
|
---|
465 | string 'matches', etc."""
|
---|
466 | return self.fetch(recursive=recursive, text=text, limit=limit)
|
---|
467 |
|
---|
468 | def first(self, name=None, attrs={}, recursive=True, text=None):
|
---|
469 | """Return only the first child of this
|
---|
470 | Tag matching the given criteria."""
|
---|
471 | r = Null
|
---|
472 | l = self.fetch(name, attrs, recursive, text, 1)
|
---|
473 | if l:
|
---|
474 | r = l[0]
|
---|
475 | return r
|
---|
476 | findChild = first
|
---|
477 |
|
---|
478 | def fetch(self, name=None, attrs={}, recursive=True, text=None,
|
---|
479 | limit=None):
|
---|
480 | """Extracts a list of Tag objects that match the given
|
---|
481 | criteria. You can specify the name of the Tag and any
|
---|
482 | attributes you want the Tag to have.
|
---|
483 |
|
---|
484 | The value of a key-value pair in the 'attrs' map can be a
|
---|
485 | string, a list of strings, a regular expression object, or a
|
---|
486 | callable that takes a string and returns whether or not the
|
---|
487 | string matches for some custom definition of 'matches'. The
|
---|
488 | same is true of the tag name."""
|
---|
489 | generator = self.recursiveChildGenerator
|
---|
490 | if not recursive:
|
---|
491 | generator = self.childGenerator
|
---|
492 | return self._fetch(name, attrs, text, limit, generator)
|
---|
493 | fetchChildren = fetch
|
---|
494 |
|
---|
495 | #Utility methods
|
---|
496 |
|
---|
497 | def isSelfClosing(self):
|
---|
498 | """Returns true iff this is a self-closing tag as defined in the HTML
|
---|
499 | standard.
|
---|
500 |
|
---|
501 | TODO: This is specific to BeautifulSoup and its subclasses, but it's
|
---|
502 | used by __str__"""
|
---|
503 | return self.name in BeautifulSoup.SELF_CLOSING_TAGS
|
---|
504 |
|
---|
505 | def append(self, tag):
|
---|
506 | """Appends the given tag to the contents of this tag."""
|
---|
507 | self.contents.append(tag)
|
---|
508 |
|
---|
509 | #Private methods
|
---|
510 |
|
---|
511 | def _getAttrMap(self):
|
---|
512 | """Initializes a map representation of this tag's attributes,
|
---|
513 | if not already initialized."""
|
---|
514 | if not getattr(self, 'attrMap'):
|
---|
515 | self.attrMap = {}
|
---|
516 | for (key, value) in self.attrs:
|
---|
517 | self.attrMap[key] = value
|
---|
518 | return self.attrMap
|
---|
519 |
|
---|
520 | #Generator methods
|
---|
521 | def childGenerator(self):
|
---|
522 | for i in range(0, len(self.contents)):
|
---|
523 | yield self.contents[i]
|
---|
524 | raise StopIteration
|
---|
525 |
|
---|
526 | def recursiveChildGenerator(self):
|
---|
527 | stack = [(self, 0)]
|
---|
528 | while stack:
|
---|
529 | tag, start = stack.pop()
|
---|
530 | if isinstance(tag, Tag):
|
---|
531 | for i in range(start, len(tag.contents)):
|
---|
532 | a = tag.contents[i]
|
---|
533 | yield a
|
---|
534 | if isinstance(a, Tag) and tag.contents:
|
---|
535 | if i < len(tag.contents) - 1:
|
---|
536 | stack.append((tag, i + 1))
|
---|
537 | stack.append((a, 0))
|
---|
538 | break
|
---|
539 | raise StopIteration
|
---|
540 |
|
---|
541 |
|
---|
542 | def isList(l):
|
---|
543 | """Convenience method that works with all 2.x versions of Python
|
---|
544 | to determine whether or not something is listlike."""
|
---|
545 | return hasattr(l, '__iter__') \
|
---|
546 | or (type(l) in (types.ListType, types.TupleType))
|
---|
547 |
|
---|
548 | def buildTagMap(default, *args):
|
---|
549 | """Turns a list of maps, lists, or scalars into a single map.
|
---|
550 | Used to build the SELF_CLOSING_TAGS and NESTABLE_TAGS maps out
|
---|
551 | of lists and partial maps."""
|
---|
552 | built = {}
|
---|
553 | for portion in args:
|
---|
554 | if hasattr(portion, 'items'):
|
---|
555 | #It's a map. Merge it.
|
---|
556 | for k, v in portion.items():
|
---|
557 | built[k] = v
|
---|
558 | elif isList(portion):
|
---|
559 | #It's a list. Map each item to the default.
|
---|
560 | for k in portion:
|
---|
561 | built[k] = default
|
---|
562 | else:
|
---|
563 | #It's a scalar. Map it to the default.
|
---|
564 | built[portion] = default
|
---|
565 | return built
|
---|
566 |
|
---|
567 | class BeautifulStoneSoup(Tag, SGMLParser):
|
---|
568 |
|
---|
569 | """This class contains the basic parser and fetch code. It defines
|
---|
570 | a parser that knows nothing about tag behavior except for the
|
---|
571 | following:
|
---|
572 |
|
---|
573 | You can't close a tag without closing all the tags it encloses.
|
---|
574 | That is, "<foo><bar></foo>" actually means
|
---|
575 | "<foo><bar></bar></foo>".
|
---|
576 |
|
---|
577 | [Another possible explanation is "<foo><bar /></foo>", but since
|
---|
578 | this class defines no SELF_CLOSING_TAGS, it will never use that
|
---|
579 | explanation.]
|
---|
580 |
|
---|
581 | This class is useful for parsing XML or made-up markup languages,
|
---|
582 | or when BeautifulSoup makes an assumption counter to what you were
|
---|
583 | expecting."""
|
---|
584 |
|
---|
585 | SELF_CLOSING_TAGS = {}
|
---|
586 | NESTABLE_TAGS = {}
|
---|
587 | RESET_NESTING_TAGS = {}
|
---|
588 | QUOTE_TAGS = {}
|
---|
589 |
|
---|
590 | #As a public service we will by default silently replace MS smart quotes
|
---|
591 | #and similar characters with their HTML or ASCII equivalents.
|
---|
592 | MS_CHARS = { '\x80' : '€',
|
---|
593 | '\x81' : ' ',
|
---|
594 | '\x82' : '‚',
|
---|
595 | '\x83' : 'ƒ',
|
---|
596 | '\x84' : '„',
|
---|
597 | '\x85' : '…',
|
---|
598 | '\x86' : '†',
|
---|
599 | '\x87' : '‡',
|
---|
600 | '\x88' : '⁁',
|
---|
601 | '\x89' : '%',
|
---|
602 | '\x8A' : 'Š',
|
---|
603 | '\x8B' : '<',
|
---|
604 | '\x8C' : 'Œ',
|
---|
605 | '\x8D' : '?',
|
---|
606 | '\x8E' : 'Z',
|
---|
607 | '\x8F' : '?',
|
---|
608 | '\x90' : '?',
|
---|
609 | '\x91' : '‘',
|
---|
610 | '\x92' : '’',
|
---|
611 | '\x93' : '“',
|
---|
612 | '\x94' : '”',
|
---|
613 | '\x95' : '•',
|
---|
614 | '\x96' : '–',
|
---|
615 | '\x97' : '—',
|
---|
616 | '\x98' : '˜',
|
---|
617 | '\x99' : '™',
|
---|
618 | '\x9a' : 'š',
|
---|
619 | '\x9b' : '>',
|
---|
620 | '\x9c' : 'œ',
|
---|
621 | '\x9d' : '?',
|
---|
622 | '\x9e' : 'z',
|
---|
623 | '\x9f' : 'Ÿ', }
|
---|
624 |
|
---|
625 | PARSER_MASSAGE = [(re.compile('(<[^<>]*)/>'),
|
---|
626 | lambda(x):x.group(1) + ' />'),
|
---|
627 | (re.compile('<!\s+([^<>]*)>'),
|
---|
628 | lambda(x):'<!' + x.group(1) + '>'),
|
---|
629 | (re.compile("([\x80-\x9f])"),
|
---|
630 | lambda(x): BeautifulStoneSoup.MS_CHARS.get(x.group(1)))
|
---|
631 | ]
|
---|
632 |
|
---|
633 | ROOT_TAG_NAME = '[document]'
|
---|
634 |
|
---|
635 | def __init__(self, text=None, avoidParserProblems=True,
|
---|
636 | initialTextIsEverything=True):
|
---|
637 | """Initialize this as the 'root tag' and feed in any text to
|
---|
638 | the parser.
|
---|
639 |
|
---|
640 | NOTE about avoidParserProblems: sgmllib will process most bad
|
---|
641 | HTML, and BeautifulSoup has tricks for dealing with some HTML
|
---|
642 | that kills sgmllib, but Beautiful Soup can nonetheless choke
|
---|
643 | or lose data if your data uses self-closing tags or
|
---|
644 | declarations incorrectly. By default, Beautiful Soup sanitizes
|
---|
645 | its input to avoid the vast majority of these problems. The
|
---|
646 | problems are relatively rare, even in bad HTML, so feel free
|
---|
647 | to pass in False to avoidParserProblems if they don't apply to
|
---|
648 | you, and you'll get better performance. The only reason I have
|
---|
649 | this turned on by default is so I don't get so many tech
|
---|
650 | support questions.
|
---|
651 |
|
---|
652 | The two most common instances of invalid HTML that will choke
|
---|
653 | sgmllib are fixed by the default parser massage techniques:
|
---|
654 |
|
---|
655 | <br/> (No space between name of closing tag and tag close)
|
---|
656 | <! --Comment--> (Extraneous whitespace in declaration)
|
---|
657 |
|
---|
658 | You can pass in a custom list of (RE object, replace method)
|
---|
659 | tuples to get Beautiful Soup to scrub your input the way you
|
---|
660 | want."""
|
---|
661 | Tag.__init__(self, self.ROOT_TAG_NAME)
|
---|
662 | if avoidParserProblems \
|
---|
663 | and not isList(avoidParserProblems):
|
---|
664 | avoidParserProblems = self.PARSER_MASSAGE
|
---|
665 | self.avoidParserProblems = avoidParserProblems
|
---|
666 | SGMLParser.__init__(self)
|
---|
667 | self.quoteStack = []
|
---|
668 | self.hidden = 1
|
---|
669 | self.reset()
|
---|
670 | if hasattr(text, 'read'):
|
---|
671 | #It's a file-type object.
|
---|
672 | text = text.read()
|
---|
673 | if text:
|
---|
674 | self.feed(text)
|
---|
675 | if initialTextIsEverything:
|
---|
676 | self.done()
|
---|
677 |
|
---|
678 | def __getattr__(self, methodName):
|
---|
679 | """This method routes method call requests to either the SGMLParser
|
---|
680 | superclass or the Tag superclass, depending on the method name."""
|
---|
681 | if methodName.find('start_') == 0 or methodName.find('end_') == 0 \
|
---|
682 | or methodName.find('do_') == 0:
|
---|
683 | return SGMLParser.__getattr__(self, methodName)
|
---|
684 | elif methodName.find('__') != 0:
|
---|
685 | return Tag.__getattr__(self, methodName)
|
---|
686 | else:
|
---|
687 | raise AttributeError
|
---|
688 |
|
---|
689 | def feed(self, text):
|
---|
690 | if self.avoidParserProblems:
|
---|
691 | for fix, m in self.avoidParserProblems:
|
---|
692 | text = fix.sub(m, text)
|
---|
693 | SGMLParser.feed(self, text)
|
---|
694 |
|
---|
695 | def done(self):
|
---|
696 | """Called when you're done parsing, so that the unclosed tags can be
|
---|
697 | correctly processed."""
|
---|
698 | self.endData() #NEW
|
---|
699 | while self.currentTag.name != self.ROOT_TAG_NAME:
|
---|
700 | self.popTag()
|
---|
701 |
|
---|
702 | def reset(self):
|
---|
703 | SGMLParser.reset(self)
|
---|
704 | self.currentData = []
|
---|
705 | self.currentTag = None
|
---|
706 | self.tagStack = []
|
---|
707 | self.pushTag(self)
|
---|
708 |
|
---|
709 | def popTag(self):
|
---|
710 | tag = self.tagStack.pop()
|
---|
711 | # Tags with just one string-owning child get the child as a
|
---|
712 | # 'string' property, so that soup.tag.string is shorthand for
|
---|
713 | # soup.tag.contents[0]
|
---|
714 | if len(self.currentTag.contents) == 1 and \
|
---|
715 | isinstance(self.currentTag.contents[0], NavigableText):
|
---|
716 | self.currentTag.string = self.currentTag.contents[0]
|
---|
717 |
|
---|
718 | #print "Pop", tag.name
|
---|
719 | if self.tagStack:
|
---|
720 | self.currentTag = self.tagStack[-1]
|
---|
721 | return self.currentTag
|
---|
722 |
|
---|
723 | def pushTag(self, tag):
|
---|
724 | #print "Push", tag.name
|
---|
725 | if self.currentTag:
|
---|
726 | self.currentTag.append(tag)
|
---|
727 | self.tagStack.append(tag)
|
---|
728 | self.currentTag = self.tagStack[-1]
|
---|
729 |
|
---|
730 | def endData(self):
|
---|
731 | currentData = ''.join(self.currentData)
|
---|
732 | if currentData:
|
---|
733 | if not currentData.strip():
|
---|
734 | if '\n' in currentData:
|
---|
735 | currentData = '\n'
|
---|
736 | else:
|
---|
737 | currentData = ' '
|
---|
738 | c = NavigableString
|
---|
739 | if type(currentData) == types.UnicodeType:
|
---|
740 | c = NavigableUnicodeString
|
---|
741 | o = c(currentData)
|
---|
742 | o.setup(self.currentTag, self.previous)
|
---|
743 | if self.previous:
|
---|
744 | self.previous.next = o
|
---|
745 | self.previous = o
|
---|
746 | self.currentTag.contents.append(o)
|
---|
747 | self.currentData = []
|
---|
748 |
|
---|
749 | def _popToTag(self, name, inclusivePop=True):
|
---|
750 | """Pops the tag stack up to and including the most recent
|
---|
751 | instance of the given tag. If inclusivePop is false, pops the tag
|
---|
752 | stack up to but *not* including the most recent instqance of
|
---|
753 | the given tag."""
|
---|
754 | if name == self.ROOT_TAG_NAME:
|
---|
755 | return
|
---|
756 |
|
---|
757 | numPops = 0
|
---|
758 | mostRecentTag = None
|
---|
759 | for i in range(len(self.tagStack) - 1, 0, -1):
|
---|
760 | if name == self.tagStack[i].name:
|
---|
761 | numPops = len(self.tagStack) - i
|
---|
762 | break
|
---|
763 | if not inclusivePop:
|
---|
764 | numPops = numPops - 1
|
---|
765 |
|
---|
766 | for i in range(0, numPops):
|
---|
767 | mostRecentTag = self.popTag()
|
---|
768 | return mostRecentTag
|
---|
769 |
|
---|
770 | def _smartPop(self, name):
|
---|
771 |
|
---|
772 | """We need to pop up to the previous tag of this type, unless
|
---|
773 | one of this tag's nesting reset triggers comes between this
|
---|
774 | tag and the previous tag of this type, OR unless this tag is a
|
---|
775 | generic nesting trigger and another generic nesting trigger
|
---|
776 | comes between this tag and the previous tag of this type.
|
---|
777 |
|
---|
778 | Examples:
|
---|
779 | <p>Foo<b>Bar<p> should pop to 'p', not 'b'.
|
---|
780 | <p>Foo<table>Bar<p> should pop to 'table', not 'p'.
|
---|
781 | <p>Foo<table><tr>Bar<p> should pop to 'tr', not 'p'.
|
---|
782 | <p>Foo<b>Bar<p> should pop to 'p', not 'b'.
|
---|
783 |
|
---|
784 | <li><ul><li> *<li>* should pop to 'ul', not the first 'li'.
|
---|
785 | <tr><table><tr> *<tr>* should pop to 'table', not the first 'tr'
|
---|
786 | <td><tr><td> *<td>* should pop to 'tr', not the first 'td'
|
---|
787 | """
|
---|
788 |
|
---|
789 | nestingResetTriggers = self.NESTABLE_TAGS.get(name)
|
---|
790 | isNestable = nestingResetTriggers != None
|
---|
791 | isResetNesting = self.RESET_NESTING_TAGS.has_key(name)
|
---|
792 | popTo = None
|
---|
793 | inclusive = True
|
---|
794 | for i in range(len(self.tagStack) - 1, 0, -1):
|
---|
795 | p = self.tagStack[i]
|
---|
796 | if (not p or p.name == name) and not isNestable:
|
---|
797 | #Non-nestable tags get popped to the top or to their
|
---|
798 | #last occurance.
|
---|
799 | popTo = name
|
---|
800 | break
|
---|
801 | if (nestingResetTriggers != None
|
---|
802 | and p.name in nestingResetTriggers) \
|
---|
803 | or (nestingResetTriggers == None and isResetNesting
|
---|
804 | and self.RESET_NESTING_TAGS.has_key(p.name)):
|
---|
805 |
|
---|
806 | #If we encounter one of the nesting reset triggers
|
---|
807 | #peculiar to this tag, or we encounter another tag
|
---|
808 | #that causes nesting to reset, pop up to but not
|
---|
809 | #including that tag.
|
---|
810 |
|
---|
811 | popTo = p.name
|
---|
812 | inclusive = False
|
---|
813 | break
|
---|
814 | p = p.parent
|
---|
815 | if popTo:
|
---|
816 | self._popToTag(popTo, inclusive)
|
---|
817 |
|
---|
818 | def unknown_starttag(self, name, attrs, selfClosing=0):
|
---|
819 | #print "Start tag %s" % name
|
---|
820 | if self.quoteStack:
|
---|
821 | #This is not a real tag.
|
---|
822 | #print "<%s> is not real!" % name
|
---|
823 | attrs = ''.join(map(lambda(x, y): ' %s="%s"' % (x, y), attrs))
|
---|
824 | self.handle_data('<%s%s>' % (name, attrs))
|
---|
825 | return
|
---|
826 | self.endData()
|
---|
827 | if not name in self.SELF_CLOSING_TAGS and not selfClosing:
|
---|
828 | self._smartPop(name)
|
---|
829 | tag = Tag(name, attrs, self.currentTag, self.previous)
|
---|
830 | if self.previous:
|
---|
831 | self.previous.next = tag
|
---|
832 | self.previous = tag
|
---|
833 | self.pushTag(tag)
|
---|
834 | if selfClosing or name in self.SELF_CLOSING_TAGS:
|
---|
835 | self.popTag()
|
---|
836 | if name in self.QUOTE_TAGS:
|
---|
837 | #print "Beginning quote (%s)" % name
|
---|
838 | self.quoteStack.append(name)
|
---|
839 | self.literal = 1
|
---|
840 |
|
---|
841 | def unknown_endtag(self, name):
|
---|
842 | if self.quoteStack and self.quoteStack[-1] != name:
|
---|
843 | #This is not a real end tag.
|
---|
844 | #print "</%s> is not real!" % name
|
---|
845 | self.handle_data('</%s>' % name)
|
---|
846 | return
|
---|
847 | self.endData()
|
---|
848 | self._popToTag(name)
|
---|
849 | if self.quoteStack and self.quoteStack[-1] == name:
|
---|
850 | self.quoteStack.pop()
|
---|
851 | self.literal = (len(self.quoteStack) > 0)
|
---|
852 |
|
---|
853 | def handle_data(self, data):
|
---|
854 | self.currentData.append(data)
|
---|
855 |
|
---|
856 | def handle_pi(self, text):
|
---|
857 | "Propagate processing instructions right through."
|
---|
858 | self.handle_data("<?%s>" % text)
|
---|
859 |
|
---|
860 | def handle_comment(self, text):
|
---|
861 | "Propagate comments right through."
|
---|
862 | self.handle_data("<!--%s-->" % text)
|
---|
863 |
|
---|
864 | def handle_charref(self, ref):
|
---|
865 | "Propagate char refs right through."
|
---|
866 | self.handle_data('&#%s;' % ref)
|
---|
867 |
|
---|
868 | def handle_entityref(self, ref):
|
---|
869 | "Propagate entity refs right through."
|
---|
870 | self.handle_data('&%s;' % ref)
|
---|
871 |
|
---|
872 | def handle_decl(self, data):
|
---|
873 | "Propagate DOCTYPEs and the like right through."
|
---|
874 | self.handle_data('<!%s>' % data)
|
---|
875 |
|
---|
876 | def parse_declaration(self, i):
|
---|
877 | """Treat a bogus SGML declaration as raw data. Treat a CDATA
|
---|
878 | declaration as regular data."""
|
---|
879 | j = None
|
---|
880 | if self.rawdata[i:i + 9] == '<![CDATA[':
|
---|
881 | k = self.rawdata.find(']]>', i)
|
---|
882 | if k == -1:
|
---|
883 | k = len(self.rawdata)
|
---|
884 | self.handle_data(self.rawdata[i + 9:k])
|
---|
885 | j = k + 3
|
---|
886 | else:
|
---|
887 | try:
|
---|
888 | j = SGMLParser.parse_declaration(self, i)
|
---|
889 | except SGMLParseError:
|
---|
890 | toHandle = self.rawdata[i:]
|
---|
891 | self.handle_data(toHandle)
|
---|
892 | j = i + len(toHandle)
|
---|
893 | return j
|
---|
894 |
|
---|
895 | class BeautifulSoup(BeautifulStoneSoup):
|
---|
896 |
|
---|
897 | """This parser knows the following facts about HTML:
|
---|
898 |
|
---|
899 | * Some tags have no closing tag and should be interpreted as being
|
---|
900 | closed as soon as they are encountered.
|
---|
901 |
|
---|
902 | * The text inside some tags (ie. 'script') may contain tags which
|
---|
903 | are not really part of the document and which should be parsed
|
---|
904 | as text, not tags. If you want to parse the text as tags, you can
|
---|
905 | always fetch it and parse it explicitly.
|
---|
906 |
|
---|
907 | * Tag nesting rules:
|
---|
908 |
|
---|
909 | Most tags can't be nested at all. For instance, the occurance of
|
---|
910 | a <p> tag should implicitly close the previous <p> tag.
|
---|
911 |
|
---|
912 | <p>Para1<p>Para2
|
---|
913 | should be transformed into:
|
---|
914 | <p>Para1</p><p>Para2
|
---|
915 |
|
---|
916 | Some tags can be nested arbitrarily. For instance, the occurance
|
---|
917 | of a <blockquote> tag should _not_ implicitly close the previous
|
---|
918 | <blockquote> tag.
|
---|
919 |
|
---|
920 | Alice said: <blockquote>Bob said: <blockquote>Blah
|
---|
921 | should NOT be transformed into:
|
---|
922 | Alice said: <blockquote>Bob said: </blockquote><blockquote>Blah
|
---|
923 |
|
---|
924 | Some tags can be nested, but the nesting is reset by the
|
---|
925 | interposition of other tags. For instance, a <tr> tag should
|
---|
926 | implicitly close the previous <tr> tag within the same <table>,
|
---|
927 | but not close a <tr> tag in another table.
|
---|
928 |
|
---|
929 | <table><tr>Blah<tr>Blah
|
---|
930 | should be transformed into:
|
---|
931 | <table><tr>Blah</tr><tr>Blah
|
---|
932 | but,
|
---|
933 | <tr>Blah<table><tr>Blah
|
---|
934 | should NOT be transformed into
|
---|
935 | <tr>Blah<table></tr><tr>Blah
|
---|
936 |
|
---|
937 | Differing assumptions about tag nesting rules are a major source
|
---|
938 | of problems with the BeautifulSoup class. If BeautifulSoup is not
|
---|
939 | treating as nestable a tag your page author treats as nestable,
|
---|
940 | try ICantBelieveItsBeautifulSoup before writing your own
|
---|
941 | subclass."""
|
---|
942 |
|
---|
943 | SELF_CLOSING_TAGS = buildTagMap(None, ['br' , 'hr', 'input', 'img', 'meta',
|
---|
944 | 'spacer', 'link', 'frame', 'base'])
|
---|
945 |
|
---|
946 | QUOTE_TAGS = {'script': None}
|
---|
947 |
|
---|
948 | #According to the HTML standard, each of these inline tags can
|
---|
949 | #contain another tag of the same type. Furthermore, it's common
|
---|
950 | #to actually use these tags this way.
|
---|
951 | NESTABLE_INLINE_TAGS = ['span', 'font', 'q', 'object', 'bdo', 'sub', 'sup',
|
---|
952 | 'center']
|
---|
953 |
|
---|
954 | #According to the HTML standard, these block tags can contain
|
---|
955 | #another tag of the same type. Furthermore, it's common
|
---|
956 | #to actually use these tags this way.
|
---|
957 | NESTABLE_BLOCK_TAGS = ['blockquote', 'div', 'fieldset', 'ins', 'del']
|
---|
958 |
|
---|
959 | #Lists can contain other lists, but there are restrictions.
|
---|
960 | NESTABLE_LIST_TAGS = { 'ol' : [],
|
---|
961 | 'ul' : [],
|
---|
962 | 'li' : ['ul', 'ol'],
|
---|
963 | 'dl' : [],
|
---|
964 | 'dd' : ['dl'],
|
---|
965 | 'dt' : ['dl'] }
|
---|
966 |
|
---|
967 | #Tables can contain other tables, but there are restrictions.
|
---|
968 | NESTABLE_TABLE_TAGS = {'table' : [],
|
---|
969 | 'tr' : ['table', 'tbody', 'tfoot', 'thead'],
|
---|
970 | 'td' : ['tr'],
|
---|
971 | 'th' : ['tr'],
|
---|
972 | }
|
---|
973 |
|
---|
974 | NON_NESTABLE_BLOCK_TAGS = ['address', 'form', 'p', 'pre']
|
---|
975 |
|
---|
976 | #If one of these tags is encountered, all tags up to the next tag of
|
---|
977 | #this type are popped.
|
---|
978 | RESET_NESTING_TAGS = buildTagMap(None, NESTABLE_BLOCK_TAGS, 'noscript',
|
---|
979 | NON_NESTABLE_BLOCK_TAGS,
|
---|
980 | NESTABLE_LIST_TAGS,
|
---|
981 | NESTABLE_TABLE_TAGS)
|
---|
982 |
|
---|
983 | NESTABLE_TAGS = buildTagMap([], NESTABLE_INLINE_TAGS, NESTABLE_BLOCK_TAGS,
|
---|
984 | NESTABLE_LIST_TAGS, NESTABLE_TABLE_TAGS)
|
---|
985 |
|
---|
986 | class ICantBelieveItsBeautifulSoup(BeautifulSoup):
|
---|
987 |
|
---|
988 | """The BeautifulSoup class is oriented towards skipping over
|
---|
989 | common HTML errors like unclosed tags. However, sometimes it makes
|
---|
990 | errors of its own. For instance, consider this fragment:
|
---|
991 |
|
---|
992 | <b>Foo<b>Bar</b></b>
|
---|
993 |
|
---|
994 | This is perfectly valid (if bizarre) HTML. However, the
|
---|
995 | BeautifulSoup class will implicitly close the first b tag when it
|
---|
996 | encounters the second 'b'. It will think the author wrote
|
---|
997 | "<b>Foo<b>Bar", and didn't close the first 'b' tag, because
|
---|
998 | there's no real-world reason to bold something that's already
|
---|
999 | bold. When it encounters '</b></b>' it will close two more 'b'
|
---|
1000 | tags, for a grand total of three tags closed instead of two. This
|
---|
1001 | can throw off the rest of your document structure. The same is
|
---|
1002 | true of a number of other tags, listed below.
|
---|
1003 |
|
---|
1004 | It's much more common for someone to forget to close (eg.) a 'b'
|
---|
1005 | tag than to actually use nested 'b' tags, and the BeautifulSoup
|
---|
1006 | class handles the common case. This class handles the
|
---|
1007 | not-co-common case: where you can't believe someone wrote what
|
---|
1008 | they did, but it's valid HTML and BeautifulSoup screwed up by
|
---|
1009 | assuming it wouldn't be.
|
---|
1010 |
|
---|
1011 | If this doesn't do what you need, try subclassing this class or
|
---|
1012 | BeautifulSoup, and providing your own list of NESTABLE_TAGS."""
|
---|
1013 |
|
---|
1014 | I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS = \
|
---|
1015 | ['em', 'big', 'i', 'small', 'tt', 'abbr', 'acronym', 'strong',
|
---|
1016 | 'cite', 'code', 'dfn', 'kbd', 'samp', 'strong', 'var', 'b',
|
---|
1017 | 'big']
|
---|
1018 |
|
---|
1019 | I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS = ['noscript']
|
---|
1020 |
|
---|
1021 | NESTABLE_TAGS = buildTagMap([], BeautifulSoup.NESTABLE_TAGS,
|
---|
1022 | I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS,
|
---|
1023 | I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS)
|
---|
1024 |
|
---|
1025 | class BeautifulSOAP(BeautifulStoneSoup):
|
---|
1026 | """This class will push a tag with only a single string child into
|
---|
1027 | the tag's parent as an attribute. The attribute's name is the tag
|
---|
1028 | name, and the value is the string child. An example should give
|
---|
1029 | the flavor of the change:
|
---|
1030 |
|
---|
1031 | <foo><bar>baz</bar></foo>
|
---|
1032 | =>
|
---|
1033 | <foo bar="baz"><bar>baz</bar></foo>
|
---|
1034 |
|
---|
1035 | You can then access fooTag['bar'] instead of fooTag.barTag.string.
|
---|
1036 |
|
---|
1037 | This is, of course, useful for scraping structures that tend to
|
---|
1038 | use subelements instead of attributes, such as SOAP messages. Note
|
---|
1039 | that it modifies its input, so don't print the modified version
|
---|
1040 | out.
|
---|
1041 |
|
---|
1042 | I'm not sure how many people really want to use this class; let me
|
---|
1043 | know if you do. Mainly I like the name."""
|
---|
1044 |
|
---|
1045 | def popTag(self):
|
---|
1046 | if len(self.tagStack) > 1:
|
---|
1047 | tag = self.tagStack[-1]
|
---|
1048 | parent = self.tagStack[-2]
|
---|
1049 | parent._getAttrMap()
|
---|
1050 | if (isinstance(tag, Tag) and len(tag.contents) == 1 and
|
---|
1051 | isinstance(tag.contents[0], NavigableText) and
|
---|
1052 | not parent.attrMap.has_key(tag.name)):
|
---|
1053 | parent[tag.name] = tag.contents[0]
|
---|
1054 | BeautifulStoneSoup.popTag(self)
|
---|
1055 |
|
---|
1056 | #Enterprise class names! It has come to our attention that some people
|
---|
1057 | #think the names of the Beautiful Soup parser classes are too silly
|
---|
1058 | #and "unprofessional" for use in enterprise screen-scraping. We feel
|
---|
1059 | #your pain! For such-minded folk, the Beautiful Soup Consortium And
|
---|
1060 | #All-Night Kosher Bakery recommends renaming this file to
|
---|
1061 | #"RobustParser.py" (or, in cases of extreme enterprisitude,
|
---|
1062 | #"RobustParserBeanInterface.class") and using the following
|
---|
1063 | #enterprise-friendly class aliases:
|
---|
1064 | class RobustXMLParser(BeautifulStoneSoup):
|
---|
1065 | pass
|
---|
1066 | class RobustHTMLParser(BeautifulSoup):
|
---|
1067 | pass
|
---|
1068 | class RobustWackAssHTMLParser(ICantBelieveItsBeautifulSoup):
|
---|
1069 | pass
|
---|
1070 | class SimplifyingSOAPParser(BeautifulSOAP):
|
---|
1071 | pass
|
---|
1072 |
|
---|
1073 | ###
|
---|
1074 |
|
---|
1075 |
|
---|
1076 | #By default, act as an HTML pretty-printer.
|
---|
1077 | if __name__ == '__main__':
|
---|
1078 | import sys
|
---|
1079 | soup = BeautifulStoneSoup(sys.stdin.read())
|
---|
1080 | print soup.prettify()
|
---|