Skip to content

Commit 8555198

Browse files
telmomenezestelmomenezes
committed
configured ruff and fixed errors
1 parent dbc4bc5 commit 8555198

13 files changed

Lines changed: 751 additions & 646 deletions

‎.pre-commit-config.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
repos:
2+
- repo: https://github.com/astral-sh/ruff-pre-commit
3+
rev: v0.15.8
4+
hooks:
5+
- id: ruff
6+
args: [--fix]
7+
- id: ruff-format

‎pyproject.toml‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ dependencies = [
3131
"spacy>=3.8.0",
3232
"torch>=2.0.0",
3333
"transformers>=4.46.0",
34-
"pip", # so that spaCy models can be easily installed with uv
34+
"pip", # so that spaCy models can be easily installed with uv
3535
]
3636

3737
[tool.uv.sources]
@@ -46,6 +46,7 @@ dev = [
4646
"coverage>=7.4.3",
4747
"datasets>=4.0.0",
4848
"pytest>=9.0.0",
49+
"evaluate>=0.4.6",
4950
]
5051

5152
[project.urls]
@@ -70,3 +71,15 @@ strict = true
7071

7172
[tool.ruff]
7273
target-version = "py310"
74+
75+
[tool.ruff.lint]
76+
select = ["E", "F", "W", "I", "UP", "B", "SIM", "RUF", "Q", "C4", "PT", "N", "ANN"]
77+
78+
[tool.ruff.lint.per-file-ignores]
79+
"tests/*" = ["E501", "ANN001", "ANN201", "ANN202", "ANN205", "D100", "D101", "D102", "D400", "D415"]
80+
81+
[tool.ruff.lint.flake8-quotes]
82+
inline-quotes = "double"
83+
84+
[tool.ruff.format]
85+
quote-style = "double"

‎scripts/generate_alpha_training_data.py‎

Lines changed: 59 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,16 @@
22
import json
33

44
from hyperbase import hedge
5-
from hyperbase_parser_ab import AlphaBetaParser
65

6+
from hyperbase_parser_ab import AlphaBetaParser
77

8-
if __name__ == '__main__':
9-
arg_parser = argparse.ArgumentParser(
10-
description='Generate alpha training data.')
11-
arg_parser.add_argument('infile', type=str, help='input jsonl file')
12-
arg_parser.add_argument('outfile', type=str, help='output tsv file')
8+
if __name__ == "__main__":
9+
arg_parser = argparse.ArgumentParser(description="Generate alpha training data.")
10+
arg_parser.add_argument("infile", type=str, help="input jsonl file")
11+
arg_parser.add_argument("outfile", type=str, help="output tsv file")
1312
arg_parser.add_argument(
14-
'--lang', type=str, default='en', help='language (default: en)')
13+
"--lang", type=str, default="en", help="language (default: en)"
14+
)
1515
args = arg_parser.parse_args()
1616

1717
total_sentences = 0
@@ -21,16 +21,16 @@
2121

2222
parser = AlphaBetaParser(lang=args.lang)
2323

24-
with open(args.infile, 'r') as infile, open(args.outfile, 'w') as outfile:
24+
with open(args.infile) as infile, open(args.outfile, "w") as outfile:
2525
for line in infile.readlines():
2626
case = json.loads(line)
27-
sentence = case['sentence']
28-
atoms = case['atoms']
27+
sentence = case["sentence"]
28+
atoms = case["atoms"]
2929
parses = parser.parse_sentence(sentence)
30-
spacy_sentence = list(parser.doc.sents)[0] if parser.doc else None
30+
spacy_sentence = next(iter(parser.doc.sents)) if parser.doc else None
3131
if not spacy_sentence or not parses:
3232
failed_parses += 1
33-
elif case['ignore']:
33+
elif case["ignore"]:
3434
ignored_sentences += 1
3535
elif len(atoms) == len(spacy_sentence):
3636
total_sentences += 1
@@ -43,65 +43,71 @@
4343
if atom_edge is None:
4444
continue
4545

46-
word_before = ''
47-
word_after = ''
48-
pos_before = ''
49-
pos_after = ''
50-
tag_before = ''
51-
tag_after = ''
52-
dep_before = ''
53-
dep_after = ''
46+
word_before = ""
47+
word_after = ""
48+
pos_before = ""
49+
pos_after = ""
50+
tag_before = ""
51+
tag_after = ""
52+
dep_before = ""
53+
dep_after = ""
5454
punct_before = False
5555
punct_after = False
5656
if i > 0:
5757
word_before = str(spacy_sentence[i - 1])
5858
pos_before = spacy_sentence[i - 1].pos_
5959
tag_before = spacy_sentence[i - 1].tag_
6060
dep_before = spacy_sentence[i - 1].dep_
61-
if spacy_sentence[i - 1].pos_ == 'PUNCT':
61+
if spacy_sentence[i - 1].pos_ == "PUNCT":
6262
punct_before = True
6363
if i < len(atoms) - 1:
6464
word_after = str(spacy_sentence[i + 1])
6565
pos_after = spacy_sentence[i + 1].pos_
6666
tag_after = spacy_sentence[i + 1].tag_
6767
dep_after = spacy_sentence[i + 1].dep_
68-
if spacy_sentence[i + 1].pos_ == 'PUNCT':
68+
if spacy_sentence[i + 1].pos_ == "PUNCT":
6969
punct_after = True
7070

7171
head = token.head
7272
is_root = head is None
7373
has_lefts = token.n_lefts > 0
7474
has_rights = token.n_rights > 0
75-
outfile.write(('{}' + '\t{}' * 25 + '\n').format(
76-
atom_edge.mtype(),
77-
str(token),
78-
token.pos_,
79-
token.tag_,
80-
token.dep_,
81-
str(head) if head else '',
82-
head.pos_ if head else '',
83-
head.tag_ if head else '',
84-
head.dep_ if head else '',
85-
is_root,
86-
has_lefts,
87-
has_rights,
88-
token.ent_type_,
89-
token.shape_[:2],
90-
word_before,
91-
word_after,
92-
punct_before,
93-
punct_after,
94-
pos_before,
95-
pos_after,
96-
tag_before,
97-
tag_after,
98-
dep_before,
99-
dep_after,
100-
case['correct'],
101-
case['source']))
75+
outfile.write(
76+
("{}" + "\t{}" * 25 + "\n").format(
77+
atom_edge.mtype(),
78+
str(token),
79+
token.pos_,
80+
token.tag_,
81+
token.dep_,
82+
str(head) if head else "",
83+
head.pos_ if head else "",
84+
head.tag_ if head else "",
85+
head.dep_ if head else "",
86+
is_root,
87+
has_lefts,
88+
has_rights,
89+
token.ent_type_,
90+
token.shape_[:2],
91+
word_before,
92+
word_after,
93+
punct_before,
94+
punct_after,
95+
pos_before,
96+
pos_after,
97+
tag_before,
98+
tag_after,
99+
dep_before,
100+
dep_after,
101+
case["correct"],
102+
case["source"],
103+
)
104+
)
102105
else:
103106
failed_parses += 1
104-
print('sentences: {}; ignored: {}; failed: {}; atoms: {}'.format(
105-
total_sentences, ignored_sentences, failed_parses,
106-
total_atoms))
107-
print('done.')
107+
print(
108+
f"sentences: {total_sentences}; "
109+
f"ignored: {ignored_sentences}; "
110+
f"failed: {failed_parses}; "
111+
f"atoms: {total_atoms}"
112+
)
113+
print("done.")

‎scripts/train_atomizer.py‎

Lines changed: 29 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,28 @@
11
import json
22

3+
import evaluate
34
import numpy as np
4-
from numpy.typing import NDArray
55
from datasets import Dataset
6+
from numpy.typing import NDArray
67
from transformers import (
7-
AutoTokenizer,
88
AutoModelForTokenClassification,
9+
AutoTokenizer,
10+
Trainer,
911
TrainingArguments,
10-
Trainer
1112
)
1213

1314

1415
def tokenize_and_align_labels(examples: dict[str, list]) -> dict[str, list]:
1516
"""Tokenize each sample and align the original token labels
16-
to the new subword (tokenized) structure."""
17+
to the new subword (tokenized) structure."""
1718

1819
tokenized_outputs = tokenizer(
1920
examples["tokens"],
2021
truncation=True,
21-
is_split_into_words=True, # Important for token-based tasks
22+
is_split_into_words=True, # Important for token-based tasks
2223
return_offsets_mapping=True, # We'll use this if needed
23-
padding="max_length", # or "longest" / "do_not_pad"
24-
max_length=200 # adjust as needed
24+
padding="max_length", # or "longest" / "do_not_pad"
25+
max_length=200, # adjust as needed
2526
)
2627

2728
labels_aligned: list[list[int]] = []
@@ -31,15 +32,13 @@ def tokenize_and_align_labels(examples: dict[str, list]) -> dict[str, list]:
3132
# repeating the label for all subwords of the original token.
3233
word_ids: list[int | None] = tokenized_outputs.word_ids(batch_index=i)
3334
label_ids: list[int] = []
34-
previous_word_idx: int | None = None
3535

3636
for word_idx in word_ids:
3737
if word_idx is None:
3838
# This is a special token like [CLS], [SEP], or padding
3939
label_ids.append(-100)
4040
else:
4141
label_ids.append(label_to_id[labels[word_idx]])
42-
previous_word_idx = word_idx
4342

4443
labels_aligned.append(label_ids)
4544

@@ -52,8 +51,8 @@ def tokenize_and_align_labels(examples: dict[str, list]) -> dict[str, list]:
5251

5352
def compute_metrics(eval_pred: tuple[NDArray, NDArray]) -> dict[str, float]:
5453
"""Compute accuracy at the token level (simple example).
55-
You can also compute F1, precision, recall, etc. by ignoring
56-
the -100 special tokens."""
54+
You can also compute F1, precision, recall, etc. by ignoring
55+
the -100 special tokens."""
5756
logits: NDArray
5857
labels: NDArray
5958
logits, labels = eval_pred
@@ -62,33 +61,35 @@ def compute_metrics(eval_pred: tuple[NDArray, NDArray]) -> dict[str, float]:
6261
# Flatten ignoring -100
6362
true_predictions: list[int] = []
6463
true_labels: list[int] = []
65-
for pred, lab in zip(predictions, labels):
66-
for p, l in zip(pred, lab):
67-
if l != -100: # skip special tokens
68-
true_predictions.append(p)
69-
true_labels.append(l)
64+
for pred, lab in zip(predictions, labels, strict=True):
65+
for _pred, _lab in zip(
66+
pred,
67+
lab,
68+
strict=False,
69+
):
70+
if _lab != -100: # skip special tokens
71+
true_predictions.append(_pred)
72+
true_labels.append(_lab)
7073

7174
results: dict[str, float] = accuracy_metric.compute(
72-
references=true_labels,
73-
predictions=true_predictions
75+
references=true_labels, predictions=true_predictions
7476
)
7577
return {"accuracy": results["accuracy"]}
7678

7779

78-
if __name__ == '__main__':
79-
with open("sentences.jsonl", "rt") as f:
80+
if __name__ == "__main__":
81+
with open("sentences.jsonl") as f:
8082
sentences: list[dict] = [json.loads(line) for line in f]
8183

8284
dataset_dict: dict[str, list] = {
8385
"tokens": [sentence["words"] for sentence in sentences],
84-
"labels": [sentence["types"] for sentence in sentences]
86+
"labels": [sentence["types"] for sentence in sentences],
8587
}
8688

8789
full_dataset: Dataset = Dataset.from_dict(dataset_dict)
8890

8991
max_words: int = max([len(sentence["words"]) for sentence in sentences])
9092

91-
9293
labels: set[str] = set()
9394
for sentence in sentences:
9495
labels |= set(sentence["types"])
@@ -103,9 +104,10 @@ def compute_metrics(eval_pred: tuple[NDArray, NDArray]) -> dict[str, float]:
103104
print("Num train samples:", len(train_dataset))
104105
print("Num test samples: ", len(test_dataset))
105106

106-
107107
model_checkpoint: str = "distilbert-base-multilingual-cased"
108-
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, use_fast=True, add_prefix_space=True)
108+
tokenizer = AutoTokenizer.from_pretrained(
109+
model_checkpoint, use_fast=True, add_prefix_space=True
110+
)
109111

110112
# Apply to train/test datasets
111113
train_dataset = train_dataset.map(tokenize_and_align_labels, batched=True)
@@ -123,7 +125,7 @@ def compute_metrics(eval_pred: tuple[NDArray, NDArray]) -> dict[str, float]:
123125
model_checkpoint,
124126
num_labels=len(labels),
125127
id2label=id_to_label,
126-
label2id=label_to_id
128+
label2id=label_to_id,
127129
)
128130

129131
accuracy_metric = evaluate.load("accuracy") # type: ignore[attr-defined]
@@ -139,7 +141,7 @@ def compute_metrics(eval_pred: tuple[NDArray, NDArray]) -> dict[str, float]:
139141
weight_decay=0.01,
140142
logging_dir="./logs",
141143
logging_steps=10,
142-
report_to="none" # Set to "tensorboard" if you want logs
144+
report_to="none", # Set to "tensorboard" if you want logs
143145
)
144146

145147
trainer: Trainer = Trainer(
@@ -148,7 +150,7 @@ def compute_metrics(eval_pred: tuple[NDArray, NDArray]) -> dict[str, float]:
148150
train_dataset=train_dataset,
149151
eval_dataset=test_dataset,
150152
processing_class=tokenizer,
151-
compute_metrics=compute_metrics
153+
compute_metrics=compute_metrics,
152154
)
153155

154156
trainer.train()

0 commit comments

Comments
 (0)