[3.13] gh-151464: exclude '<>' token from tokenize output (GH-154854) (#158191)

* gh-151464: exclude '<>' token from tokenize output

Was:
```
$ echo '1 <> 2' | python -m tokenize
1,0-1,1:            NUMBER         '1'
1,2-1,4:            OP             '<>'
1,5-1,6:            NUMBER         '2'
1,6-1,7:            NEWLINE        '\n'
2,0-2,0:            ENDMARKER      ''
```

Now (regardless on ``__future__.barry_as_FLUFL`` import):
```
$ echo '1 <> 2' | ./python -m tokenize
1,0-1,1:            NUMBER         '1'
1,2-1,3:            OP             '<'
1,3-1,4:            OP             '>'
1,5-1,6:            NUMBER         '2'
1,6-1,7:            NEWLINE        '\n'
2,0-2,0:            ENDMARKER      ''
```
in accordance with the Grammar:
https://docs.python.org/3.14/reference/lexical_analysis.html#operators-and-delimiters

Also adds a custom error message for ``<>`` ("not equal" in Pascal
and Python 2).

* +1

* address review: lowercase and move invalid rule

* address review: news

* address review: move test_guido_as_bdfl_ineq_tokens()

* address review: revert _PyTokenizer_From* changes

* + revert unrelated change

* address review: remove whatsnew entry

(cherry picked from commit 198bc76)

Co-authored-by: Sergey B Kirpichev <skirpichev@gmail.com>

Author: pablogsal

Grammar/python.gram

@@ -748,6 +748,7 @@ compare_op_bitwise_or_pair[CmpopExprPair*]:
     | eq_bitwise_or
     | noteq_bitwise_or
     | lte_bitwise_or
+    | invalid_noteq
     | lt_bitwise_or
     | gte_bitwise_or
     | gt_bitwise_or
@@ -1439,3 +1440,10 @@ invalid_type_params:
         RAISE_SYNTAX_ERROR_STARTING_FROM(
             token,
             "Type parameter list cannot be empty")}
+
+invalid_noteq:
+    | a='<' b='>' {
+        _PyPegen_tokens_are_adjacent(a, b)
+        ? RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax.  Maybe you meant '!=' instead of '<>'?")
+        : NULL
+    }

Lib/test/test_syntax.py

@@ -2728,6 +2728,30 @@ def test_deep_invalid_rule(self):
         with self.assertRaises(SyntaxError):
             compile(source, "<string>", "exec")
 
+    def test_diamond_operator(self):
+        self._check_error(
+            "1<>2",
+            r"Maybe you meant '!=' instead of '<>'\?",
+            lineno=1,
+            end_lineno=1,
+            offset=2,
+            end_offset=4,
+        )
+
+    def test_diamond_operator_barry_as_flufl(self):
+        compile(
+            "from __future__ import barry_as_FLUFL\n1<>2",
+            "<test>", "exec",
+        )
+        self._check_error(
+            "from __future__ import barry_as_FLUFL\na != b",
+            "with Barry as BDFL, use '<>' instead of '!='",
+            lineno=2,
+            end_lineno=2,
+            offset=3,
+            end_offset=5,
+        )
+
 
 def load_tests(loader, tests, pattern):
     tests.addTest(doctest.DocTestSuite())

Lib/test/test_tokenize.py

@@ -1229,6 +1229,20 @@ def test_multiline_non_ascii_fstring_with_expr(self):
     FSTRING_END \'"\'           (2, 2) (2, 3)
     """)
 
+    def test_ineq_tokens(self):
+        self.check_tokenize("1 != 2", """\
+    NUMBER     '1'           (1, 0) (1, 1)
+    OP         '!='          (1, 2) (1, 4)
+    NUMBER     '2'           (1, 5) (1, 6)
+""")
+        self.check_tokenize("1 <> 2", """\
+    NUMBER     '1'           (1, 0) (1, 1)
+    OP         '<'           (1, 2) (1, 3)
+    OP         '>'           (1, 3) (1, 4)
+    NUMBER     '2'           (1, 5) (1, 6)
+""")
+
+
 class GenerateTokensTest(TokenizeTest):
     def check_tokenize(self, s, expected):
         # Format the tokens in s in a table format.

Misc/NEWS.d/next/Core_and_Builtins/2026-07-29-08-43-14.gh-issue-151464.o_mtmz.rst

@@ -0,0 +1,2 @@
+Exclude invalid token ``<>`` from :mod:`tokenize` output.  :exc:`SyntaxError`
+for ``<>`` now suggests ``!=``.

Parser/action_helpers.c

@@ -1726,6 +1726,7 @@ _PyPegen_checked_future_import(Parser *p, identifier module, asdl_alias_seq * na
             alias_ty alias = asdl_seq_GET(names, i);
             if (PyUnicode_CompareWithASCIIString(alias->name, "barry_as_FLUFL") == 0) {
                 p->flags |= PyPARSE_BARRY_AS_BDFL;
+                p->tok->barry_as_bdfl = 1;
             }
         }
     }

Parser/lexer/lexer.c

@@ -1144,6 +1144,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
     {
         int c2 = tok_nextc(tok);
         int current_token = _PyToken_TwoChars(c, c2);
+        if (c == '<' && c2 == '>' && !tok->barry_as_bdfl) {
+            current_token = OP;
+        }
         if (current_token != OP) {
             int c3 = tok_nextc(tok);
             int current_token3 = _PyToken_ThreeChars(c, c2, c3);

Parser/lexer/state.h

@@ -131,6 +131,7 @@ struct tok_state {
 #ifdef Py_DEBUG
     int debug;
 #endif
+    int barry_as_bdfl;
 };
 
 int _PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset,