Skip to content
This repository was archived by the owner on Mar 8, 2020. It is now read-only.

Commit cfa8cc9

Browse files
author
Juanjo Alvarez
committed
Str and Bytes nodes un-sugarcoated
1 parent d9724ed commit cfa8cc9

39 files changed

Lines changed: 204 additions & 215 deletions

‎ANNOTATION.md‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,9 @@
3636
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Invert'\] | OpBitwiseComplement |
3737
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='UAdd'\] | OpPositive |
3838
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='USub'\] | OpNegative |
39-
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='StringLiteral'\] | StringLiteral, Expression |
40-
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='ByteLiteral'\] | ByteStringLiteral, Expression |
41-
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='NumLiteral'\] | NumberLiteral, Expression |
4239
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Str'\] | StringLiteral, Expression |
40+
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Bytes'\] | ByteStringLiteral, Expression |
41+
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='NumLiteral'\] | NumberLiteral, Expression |
4342
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='BoolLiteral'\] | BooleanLiteral, Expression |
4443
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='JoinedStr'\] | StringLiteral, Expression |
4544
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='JoinedStr'\]/\*\[@InternalType='FormattedValue'\] | Expression, Incomplete |

‎driver/normalizer/annotation.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -88,10 +88,9 @@ var AnnotationRules = On(Any).Self(
8888
On(HasInternalType(pyast.UAdd)).Roles(OpPositive),
8989
On(HasInternalType(pyast.USub)).Roles(OpNegative),
9090

91-
On(HasInternalType(pyast.StringLiteral)).Roles(StringLiteral, Expression),
92-
On(HasInternalType(pyast.ByteLiteral)).Roles(ByteStringLiteral, Expression),
93-
On(HasInternalType(pyast.NumLiteral)).Roles(NumberLiteral, Expression),
9491
On(HasInternalType(pyast.Str)).Roles(StringLiteral, Expression),
92+
On(HasInternalType(pyast.Bytes)).Roles(ByteStringLiteral, Expression),
93+
On(HasInternalType(pyast.NumLiteral)).Roles(NumberLiteral, Expression),
9594
On(HasInternalType(pyast.BoolLiteral)).Roles(BooleanLiteral, Expression),
9695
On(HasInternalType(pyast.JoinedStr)).Roles(StringLiteral, Expression).Children(
9796
On(HasInternalType(pyast.FormattedValue)).Roles(Expression, Incomplete),

‎driver/normalizer/parser.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ var ToNoder = &native.ObjectToNoder{
1919
"id": true, // Name nodes
2020
"attr": true, // something.attr
2121
"arg": true, // function arguments
22-
"LiteralValue": true, // string/num/byte/constant literal
22+
"LiteralValue": true, // num/constant literal
23+
"s": true, // string/byte
2324
"noop_line": true, // Comment/Noop (non significative whitespace)
2425
},
2526
SyntheticTokens: map[string]string{

‎driver/normalizer/pyast/pyast.go‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ const (
3737
BoolOp = "BoolOp"
3838
BoolopInternal = "boolop"
3939
Break = "Break"
40-
ByteLiteral = "ByteLiteral"
4140
Bytes = "Bytes"
4241
Call = "Call"
4342
ClassDef = "ClassDef"
@@ -126,7 +125,6 @@ const (
126125
Stmt = "stmt"
127126
Store = "Store"
128127
Str = "Str"
129-
StringLiteral = "StringLiteral"
130128
Sub = "Sub"
131129
Subscript = "Subscript"
132130
Suite = "Suite"

‎native/python_package/python_driver/astimprove.py‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -286,7 +286,7 @@ def remainder_noops(self):
286286

287287

288288
_TOKEN_KEYS = set(
289-
("module", "name", "id", "attr", "arg", "LiteralValue")
289+
("module", "name", "id", "attr", "arg", "LiteralValue", "s")
290290
)
291291

292292
_SYNTHETIC_TOKENS = {
@@ -461,11 +461,6 @@ def visit_str(self, node):
461461
"""
462462
return str(node)
463463

464-
def visit_Str(self, node):
465-
node.update({"LiteralValue": node["s"], "ast_type": "StringLiteral"})
466-
node.pop("s", None)
467-
return node
468-
469464
def visit_Bytes(self, node):
470465
try:
471466
s = node["s"].decode()
@@ -475,10 +470,7 @@ def visit_Bytes(self, node):
475470
s = encode(node["s"], 'base64').decode().strip()
476471
encoding = 'base64'
477472

478-
node.update({"LiteralValue": s,
479-
"encoding": encoding,
480-
"ast_type": "ByteLiteral"})
481-
node.pop("s", None)
473+
node.update({"s": s, "encoding": encoding})
482474
return node
483475

484476
def visit_NoneType(self, node):

‎tests/assert_constant.py.native‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,12 +46,12 @@
4646
"end_lineno": 3,
4747
"lineno": 3,
4848
"msg": {
49-
"LiteralValue": "assert message",
50-
"ast_type": "StringLiteral",
49+
"ast_type": "Str",
5150
"col_offset": 14,
5251
"end_col_offset": 29,
5352
"end_lineno": 3,
54-
"lineno": 3
53+
"lineno": 3,
54+
"s": "assert message"
5555
},
5656
"test": {
5757
"LiteralValue": true,

‎tests/assert_constant.py.uast‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ Module {
9797
. . . . internalRole: body
9898
. . . }
9999
. . . Children: {
100-
. . . . 0: StringLiteral {
100+
. . . . 0: Str {
101101
. . . . . Roles: StringLiteral,Expression
102102
. . . . . TOKEN "assert message"
103103
. . . . . StartPosition: {

‎tests/declarations_annotated.py.native‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,12 +62,12 @@
6262
"lineno": 2
6363
},
6464
"value": {
65-
"LiteralValue": "foo",
66-
"ast_type": "StringLiteral",
65+
"ast_type": "Str",
6766
"col_offset": 10,
6867
"end_col_offset": 14,
6968
"end_lineno": 2,
70-
"lineno": 2
69+
"lineno": 2,
70+
"s": "foo"
7171
}
7272
}
7373
]

‎tests/declarations_annotated.py.uast‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ Module {
120120
. . . . . . internalRole: target
121121
. . . . . }
122122
. . . . }
123-
. . . . 2: StringLiteral {
123+
. . . . 2: Str {
124124
. . . . . Roles: StringLiteral,Expression
125125
. . . . . TOKEN "foo"
126126
. . . . . StartPosition: {

‎tests/except.py.native‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -42,12 +42,12 @@
4242
"exc": {
4343
"args": [
4444
{
45-
"LiteralValue": "gogogo",
46-
"ast_type": "StringLiteral",
45+
"ast_type": "Str",
4746
"col_offset": 21,
4847
"end_col_offset": 28,
4948
"end_lineno": 3,
50-
"lineno": 3
49+
"lineno": 3,
50+
"s": "gogogo"
5151
}
5252
],
5353
"ast_type": "Call",
@@ -78,12 +78,12 @@
7878
"value": {
7979
"args": [
8080
{
81-
"LiteralValue": "here we are",
82-
"ast_type": "StringLiteral",
81+
"ast_type": "Str",
8382
"col_offset": 11,
8483
"end_col_offset": 23,
8584
"end_lineno": 9,
86-
"lineno": 9
85+
"lineno": 9,
86+
"s": "here we are"
8787
}
8888
],
8989
"ast_type": "Call",
@@ -113,12 +113,12 @@
113113
"value": {
114114
"args": [
115115
{
116-
"LiteralValue": "someexception catched",
117-
"ast_type": "StringLiteral",
116+
"ast_type": "Str",
118117
"col_offset": 11,
119118
"end_col_offset": 33,
120119
"end_lineno": 5,
121-
"lineno": 5
120+
"lineno": 5,
121+
"s": "someexception catched"
122122
}
123123
],
124124
"ast_type": "Call",
@@ -162,12 +162,12 @@
162162
"value": {
163163
"args": [
164164
{
165-
"LiteralValue": "ayyyy",
166-
"ast_type": "StringLiteral",
165+
"ast_type": "Str",
167166
"col_offset": 11,
168167
"end_col_offset": 17,
169168
"end_lineno": 7,
170-
"lineno": 7
169+
"lineno": 7,
170+
"s": "ayyyy"
171171
}
172172
],
173173
"ast_type": "Call",

0 commit comments

Comments
 (0)