Reconstructor: separate ? from the item that follows it

lark/reconstruct.py

@@ -102,7 +102,8 @@ def reconstruct(self, tree: ParseTree, postproc: Optional[Callable[[Iterable[str
         y = []
         prev_item = ''
         for item in x:
-            if insert_spaces and prev_item and item and is_id_continue(prev_item[-1]) and is_id_continue(item[0]):
+            if insert_spaces and prev_item and item and is_id_continue(item[0]) and (
+                    is_id_continue(prev_item[-1]) or prev_item[-1] == '?'):
                 y.append(' ')
             y.append(item)
             prev_item = item

tests/test_reconstructor.py

@@ -168,6 +168,18 @@ def test_keep_all_tokens(self):
         for code in examples:
             self.assert_reconstruct(g, code, keep_all_tokens=True)
 
+    def test_optional_operator_is_separated(self):
+        """
+        The `?` operator must not glue to the item that follows it. In Lark's own grammar a
+        leading `?` is part of a rule name, so `r: a? b` came back as `r:a?b`, where `?b` is a
+        single name: the optional operator is lost and a rule reference changes identity.
+        """
+        parser = Lark.open_from_package('lark', 'lark.lark', ['grammars'],
+                                        parser='lalr', maybe_placeholders=False)
+        tree = parser.parse('r: a? b\n')
+        new = Reconstructor(parser).reconstruct(tree)
+        self.assertEqual(parser.parse(new + '\n'), tree)
+
     def test_switch_grammar_unicode_terminal(self):
         """
         This test checks that a parse tree built with a grammar containing only ascii characters can be reconstructed