Skip to content

Commit f1e09ef

Browse files
masatakemasatake
authored
Merge pull request #4353 from masatake/python--type-statements
Python: support type statements
2 parents 6a5177d + 0d1fc15 commit f1e09ef

File tree

8 files changed

+148
-3
lines changed

8 files changed

+148
-3
lines changed

‎Units/parser-python.r/type-statements.d/args.ctags‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
--sort=no
2+
--kinds-Python=+z
3+
--fields=+{signature}{typeref}

‎Units/parser-python.r/type-statements.d/expected.tags‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
MyList0 input.py /^MyList0 = list[str]$/;" v
2+
MyList1 input.py /^MyList1: TypeAlias = list[str]$/;" v typeref:typename:TypeAlias
3+
MyList2 input.py /^type MyList2 = list[str]$/;" v typeref:typename:TypeAliasType
4+
ListOrSet input.py /^type ListOrSet[T] = list[T] | set[T]$/;" v typeref:typename:TypeAliasType
5+
x input-0.py /^x = type(1)$/;" v
6+
f input-0.py /^def f(x, a):$/;" f signature:(x, a)
7+
x input-0.py /^def f(x, a):$/;" z function:f file:
8+
a input-0.py /^def f(x, a):$/;" z function:f file:
9+
y input-0.py /^y = f(type, ...)$/;" v
10+
type input-0.py /^class type: ...$/;" c
11+
type input-0.py /^def type(a): ...$/;" f signature:(a)
12+
a input-0.py /^def type(a): ...$/;" z function:type file:
13+
type input-0.py /^type = lambda t: ...$/;" f signature:(t)
14+
fn input-0.py /^def fn(type, a): ...$/;" f signature:(type, a)
15+
type input-0.py /^def fn(type, a): ...$/;" z function:fn file:
16+
a input-0.py /^def fn(type, a): ...$/;" z function:fn file:

‎Units/parser-python.r/type-statements.d/input-0.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# Derived from the comment in #4351 submitted by @mgedmin
2+
x = type(1)
3+
def f(x, a):
4+
return 1
5+
y = f(type, ...)
6+
class type: ...
7+
def type(a): ...
8+
type = lambda t: ...
9+
def fn(type, a): ...

‎Units/parser-python.r/type-statements.d/input.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
from typing import TypeAlias
2+
3+
MyList0 = list[str]
4+
MyList1: TypeAlias = list[str]
5+
type MyList2 = list[str]
6+
type ListOrSet[T] = list[T] | set[T]

‎Units/parser-python.r/typehint.d/validator‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
python

‎docs/news/HEAD.rst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,10 @@ Meson:
4747

4848
* Extract config vars defined inside configuration_data({...}).
4949

50+
Python:
51+
52+
* Support type statements.
53+
5054
New parsers
5155
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
5256
The following parsers have been added:

‎misc/validators/validator-python‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# -*- sh -*-
2+
# validator-python --- validating Python input files
3+
#
4+
# Copyright (c) 2025, Masatake YAMATO
5+
#
6+
# This program is free software; you can redistribute it and/or
7+
# modify it under the terms of the GNU General Public License
8+
# as published by the Free Software Foundation; either version 2
9+
# of the License, or (at your option) any later version.
10+
#
11+
# This program is distributed in the hope that it will be useful,
12+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14+
# GNU General Public License for more details.
15+
#
16+
# You should have received a copy of the GNU General Public License
17+
# along with this program; if not, write to the Free Software
18+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
19+
# USA.
20+
PYTHON=${PYTHON:-python}
21+
action=$1
22+
input=$2
23+
24+
case "$action" in
25+
is_runnable)
26+
type ${PYTHON} > /dev/null 2>&1
27+
exit $?
28+
;;
29+
validate)
30+
${PYTHON} -m py_compile "$input"
31+
s=$?
32+
d=$(dirname "$input")
33+
b=$(basename "$input" .py)
34+
rm $d/__pycache__/$b*pyc
35+
rmdir $d/__pycache__
36+
exit $s
37+
;;
38+
esac

‎parsers/python.c‎

Lines changed: 71 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,15 @@
3333
#define newToken() (objPoolGet (TokenPool))
3434
#define deleteToken(t) (objPoolPut (TokenPool, (t)))
3535

36+
/* A token holding a soft keyword
37+
* has TOKEN_IDENTIFIER as its type member and
38+
* SOFT_KEYWORD_* as its keyword member.
39+
*/
40+
#define isSoftKeyword(K) (K < 0 && K != KEYWORD_NONE)
3641
enum {
37-
KEYWORD_as,
42+
SOFT_KEYWORD_type = -2,
43+
/* KEYWORD_NONE occupies -1. */
44+
KEYWORD_as = 0,
3845
KEYWORD_async,
3946
KEYWORD_cdef,
4047
KEYWORD_class,
@@ -157,6 +164,7 @@ static const keywordTable PythonKeywordTable[] = {
157164
{ "lambda", KEYWORD_lambda },
158165
{ "pass", KEYWORD_pass },
159166
{ "return", KEYWORD_return },
167+
{ "type", SOFT_KEYWORD_type },
160168
};
161169

162170
/* Taken from https://docs.python.org/3/reference/lexical_analysis.html#keywords */
@@ -661,7 +669,8 @@ static void readTokenFull (tokenInfo *const token, bool inclWhitespaces)
661669
/* FIXME: handle U, B, R and F string prefixes? */
662670
readIdentifier (token->string, c);
663671
token->keyword = lookupKeyword (vStringValue (token->string), Lang_python);
664-
if (token->keyword == KEYWORD_NONE)
672+
if (token->keyword == KEYWORD_NONE
673+
|| isSoftKeyword(token->keyword))
665674
token->type = TOKEN_IDENTIFIER;
666675
else
667676
token->type = TOKEN_KEYWORD;
@@ -1747,6 +1756,41 @@ static void setIndent (tokenInfo *const token)
17471756
}
17481757
}
17491758

1759+
static bool parseType (tokenInfo *const token, pythonKind kind)
1760+
{
1761+
TRACE_ENTER();
1762+
/* https://docs.python.org/3.14/reference/simple_stmts.html#type */
1763+
int index = makeSimplePythonTag (token, kind);
1764+
tagEntryInfo *e = getEntryInCorkQueue (index);
1765+
if (e)
1766+
{
1767+
/* Technically the type statement creates variables of type typing.TypeAliasType,
1768+
* which is a different thing from typing.TypeAlias (which is deprecated).
1769+
* Thus it would be misleading to claim they're of type TypeAlias. */
1770+
e->extensionFields.typeRef [0] = eStrdup ("typename");
1771+
e->extensionFields.typeRef [1] = eStrdup ("TypeAliasType");
1772+
}
1773+
1774+
readToken (token);
1775+
1776+
if (token->type == '[')
1777+
{
1778+
if (skipOverPair (token, '[', ']', NULL, false))
1779+
readToken (token);
1780+
}
1781+
if (token->type == TOKEN_EOF)
1782+
{
1783+
TRACE_LEAVE_TEXT("Unexpected EOF");
1784+
return false;
1785+
}
1786+
1787+
vString *tspec = vStringNew ();
1788+
bool r = skipVariableTypeAnnotation (token, tspec);
1789+
vStringDelete (tspec);
1790+
TRACE_LEAVE();
1791+
return r;
1792+
}
1793+
17501794
static void findPythonTags (void)
17511795
{
17521796
TRACE_ENTER();
@@ -1797,11 +1841,35 @@ static void findPythonTags (void)
17971841
NestingLevel *lv = nestingLevelsGetCurrent (PythonNestingLevels);
17981842
tagEntryInfo *lvEntry = getEntryOfNestingLevel (lv);
17991843
pythonKind kind = PYTHON_VARIABLE_KIND;
1844+
bool isTypeStatement = false;
18001845

18011846
if (lvEntry && lvEntry->kindIndex != PYTHON_CLASS_KIND)
18021847
kind = PYTHON_LOCAL_VARIABLE_KIND;
18031848

1804-
readNext = parseVariable (token, kind);
1849+
if (token->keyword == SOFT_KEYWORD_type)
1850+
{
1851+
/* Is a type statement? */
1852+
tokenInfo *const type = newToken ();
1853+
1854+
copyToken (type, token);
1855+
readToken (token);
1856+
if (token->type == TOKEN_IDENTIFIER)
1857+
{
1858+
/* Yes. this is a type statement. */
1859+
isTypeStatement = true;
1860+
readNext = parseType (token, kind);
1861+
}
1862+
else
1863+
{
1864+
/* No. */
1865+
ungetToken (token);
1866+
copyToken (token, type);
1867+
}
1868+
deleteToken (type);
1869+
}
1870+
1871+
if (!isTypeStatement)
1872+
readNext = parseVariable (token, kind);
18051873
}
18061874
else if (token->type == '@' && atStatementStart &&
18071875
PythonFields[F_DECORATORS].enabled)

0 commit comments

Comments
 (0)