From d63137159b0e0cdbaec5d4cb98b3de5940173ee3 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Thu, 31 Jul 2008 16:23:04 +0000 Subject: Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r65209 | raymond.hettinger | 2008-07-23 19:08:18 -0500 (Wed, 23 Jul 2008) | 1 line Finish-up the partial conversion from int to Py_ssize_t for deque indices and length. ........ r65210 | raymond.hettinger | 2008-07-23 19:53:49 -0500 (Wed, 23 Jul 2008) | 1 line Parse to the correct datatype. ........ r65211 | benjamin.peterson | 2008-07-23 21:27:46 -0500 (Wed, 23 Jul 2008) | 1 line fix spacing ........ r65212 | benjamin.peterson | 2008-07-23 21:31:28 -0500 (Wed, 23 Jul 2008) | 1 line fix markup ........ r65213 | benjamin.peterson | 2008-07-23 21:45:37 -0500 (Wed, 23 Jul 2008) | 1 line add some documentation for 2to3 ........ r65214 | raymond.hettinger | 2008-07-24 00:38:48 -0500 (Thu, 24 Jul 2008) | 1 line Finish conversion from int to Py_ssize_t. ........ r65215 | raymond.hettinger | 2008-07-24 02:04:55 -0500 (Thu, 24 Jul 2008) | 1 line Convert from long to Py_ssize_t. ........ r65216 | georg.brandl | 2008-07-24 02:09:21 -0500 (Thu, 24 Jul 2008) | 2 lines Fix indentation. ........ r65225 | benjamin.peterson | 2008-07-25 11:55:37 -0500 (Fri, 25 Jul 2008) | 1 line teach .bzrignore about doc tools ........ r65226 | benjamin.peterson | 2008-07-25 12:02:11 -0500 (Fri, 25 Jul 2008) | 1 line document default value for fillvalue ........ r65233 | raymond.hettinger | 2008-07-25 13:43:33 -0500 (Fri, 25 Jul 2008) | 1 line Issue 1592: Better error reporting for operations on closed shelves. ........ r65239 | benjamin.peterson | 2008-07-25 16:59:53 -0500 (Fri, 25 Jul 2008) | 1 line fix indentation ........ r65246 | andrew.kuchling | 2008-07-26 08:08:19 -0500 (Sat, 26 Jul 2008) | 1 line This sentence continues to bug me; rewrite it for the second time ........ r65247 | andrew.kuchling | 2008-07-26 08:09:06 -0500 (Sat, 26 Jul 2008) | 1 line Remove extra words ........ r65255 | skip.montanaro | 2008-07-26 19:49:02 -0500 (Sat, 26 Jul 2008) | 3 lines Close issue 3437 - missing state change when Allow lines are processed. Adds test cases which use Allow: as well. ........ r65256 | skip.montanaro | 2008-07-26 19:50:41 -0500 (Sat, 26 Jul 2008) | 2 lines note robotparser bug fix. ........ --- Lib/urllib/robotparser.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'Lib/urllib/robotparser.py') diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index c55fb5082f6..bafb611a1c5 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -76,6 +76,10 @@ class RobotFileParser: We allow that a user-agent: line is not preceded by one or more blank lines. """ + # states: + # 0: start state + # 1: saw user-agent line + # 2: saw an allow or disallow line state = 0 entry = Entry() @@ -112,6 +116,7 @@ class RobotFileParser: elif line[0] == "allow": if state != 0: entry.rulelines.append(RuleLine(line[1], True)) + state = 2 if state == 2: self.entries.append(entry) -- cgit v1.2.3