Fix RecursionError in parsing config with comments

kobo/conf.py

@@ -184,18 +184,18 @@ def _assert_token(self, *args):
 
     def _get_token(self, skip_newline=True):
         """Get a new token from token generator."""
-        self._tok_number, self._tok_value, self._tok_begin, self._tok_end, self._tok_line = next(self._tokens)
-        self._tok_name = token.tok_name.get(self._tok_number, None)
-
-        if self._debug:
-            print("%2s %16s %s" % (self._tok_number, self._tok_name, self._tok_value.strip()))
+        while True:
+            self._tok_number, self._tok_value, self._tok_begin, self._tok_end, self._tok_line = next(self._tokens)
+            self._tok_name = token.tok_name.get(self._tok_number, None)
 
-        # skip some tokens
-        if self._tok_name in ["COMMENT", "INDENT", "DEDENT"]:
-            self._get_token(skip_newline=skip_newline)
+            if self._debug:
+                print("%2s %16s %s" % (self._tok_number, self._tok_name, self._tok_value.strip()))
 
-        if skip_newline and self._tok_name in ["NL", "NEWLINE"]:
-            self._get_token()
+            if self._tok_name in ["COMMENT", "INDENT", "DEDENT"]:
+                continue
+            if skip_newline and self._tok_name in ["NL", "NEWLINE"]:
+                continue
+            break
 
     def _get_NAME(self):
         """Return a NAME token value."""

tests/test_conf.py

@@ -184,3 +184,39 @@ def test_duplicate_keys(self):
 
         self.assertEqual(str(ctx.exception),
                          "Duplicate dict key 'bar' in file None on line 3")
+
+
+class TestManyComments(unittest.TestCase):
+    """
+    Regression tests for RecursionError with many consecutive comment/blank
+    lines.
+    """
+
+    def _make_config_with_comments(self, n):
+        lines = ["# comment %d" % i for i in range(n)]
+        lines.append("x = 1")
+        return "\n".join(lines)
+
+    def test_many_comments_from_string(self):
+        cfg = self._make_config_with_comments(1000)
+        conf = PyConfigParser()
+        conf.load_from_string(cfg)
+        self.assertEqual(conf["x"], 1)
+
+    def test_many_comments_from_file(self):
+        cfg = self._make_config_with_comments(1000)
+        conf = PyConfigParser()
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".conf", delete=False) as f:
+            f.write(cfg)
+            path = f.name
+        try:
+            conf.load_from_file(path)
+            self.assertEqual(conf["x"], 1)
+        finally:
+            os.unlink(path)
+
+    def test_many_blank_lines_from_string(self):
+        cfg = "\n" * 1000 + "x = 1\n"
+        conf = PyConfigParser()
+        conf.load_from_string(cfg)
+        self.assertEqual(conf["x"], 1)