Skip to content

Commit c5e7b87

Browse files
Copilotwannaphong
andauthored
Fix np.load allow_pickle=False to work correctly with .npz format (#1329)
* Initial plan * Fix np.load allow_pickle=False to work with .npz NpzFile format - Replace .item().get(key) with [key] dict-style access on NpzFile - Remove variables instance attribute; use local variable instead - Add type annotation for variables local var as np.lib.npyio.NpzFile - Add allow_pickle=False to embeddings.npy load in words_spelling_correction.py Co-authored-by: wannaphong <8536487+wannaphong@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: wannaphong <8536487+wannaphong@users.noreply.github.com>
1 parent 2c19966 commit c5e7b87

2 files changed

Lines changed: 18 additions & 36 deletions

File tree

pythainlp/spell/words_spelling_correction.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,9 @@ def _load_embeddings(self) -> tuple[list[str], NDArray[np.float32]]:
8484
"""Loads embeddings matrix and vocabulary list."""
8585
import numpy as np
8686

87-
input_matrix = np.load(os.path.join(self.model_dir, "embeddings.npy"))
87+
input_matrix = np.load(
88+
os.path.join(self.model_dir, "embeddings.npy"), allow_pickle=False
89+
)
8890
words = []
8991
vocab_path = os.path.join(self.model_dir, "vocabulary.txt")
9092
with open(vocab_path, encoding="utf-8") as f:

pythainlp/transliterate/w2p.py

Lines changed: 15 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,6 @@ class Thai_W2P:
6060
p2idx: dict[str, int]
6161
idx2p: dict[int, str]
6262
checkpoint: Optional[str]
63-
variables: "NDArray"
6463
enc_emb: "NDArray"
6564
enc_w_ih: "NDArray"
6665
enc_w_hh: "NDArray"
@@ -101,52 +100,33 @@ def _load_variables(self) -> None:
101100

102101
if self.checkpoint is None:
103102
raise RuntimeError("checkpoint path is not set")
104-
self.variables: "NDArray" = np.load(self.checkpoint, allow_pickle=False)
105-
# (29, 64). (len(graphemes), emb)
106-
self.enc_emb: "NDArray" = self.variables.item().get(
107-
"encoder.emb.weight"
103+
variables: "np.lib.npyio.NpzFile" = np.load(
104+
self.checkpoint, allow_pickle=False
108105
)
106+
# (29, 64). (len(graphemes), emb)
107+
self.enc_emb: "NDArray" = variables["encoder.emb.weight"]
109108
# (3*128, 64)
110-
self.enc_w_ih: "NDArray" = self.variables.item().get(
111-
"encoder.rnn.weight_ih_l0"
112-
)
109+
self.enc_w_ih: "NDArray" = variables["encoder.rnn.weight_ih_l0"]
113110
# (3*128, 128)
114-
self.enc_w_hh: "NDArray" = self.variables.item().get(
115-
"encoder.rnn.weight_hh_l0"
116-
)
111+
self.enc_w_hh: "NDArray" = variables["encoder.rnn.weight_hh_l0"]
117112
# (3*128,)
118-
self.enc_b_ih: "NDArray" = self.variables.item().get(
119-
"encoder.rnn.bias_ih_l0"
120-
)
113+
self.enc_b_ih: "NDArray" = variables["encoder.rnn.bias_ih_l0"]
121114
# (3*128,)
122-
self.enc_b_hh: "NDArray" = self.variables.item().get(
123-
"encoder.rnn.bias_hh_l0"
124-
)
125-
115+
self.enc_b_hh: "NDArray" = variables["encoder.rnn.bias_hh_l0"]
126116
# (74, 64). (len(phonemes), emb)
127-
self.dec_emb: "NDArray" = self.variables.item().get(
128-
"decoder.emb.weight"
129-
)
117+
self.dec_emb: "NDArray" = variables["decoder.emb.weight"]
130118
# (3*128, 64)
131-
self.dec_w_ih: "NDArray" = self.variables.item().get(
132-
"decoder.rnn.weight_ih_l0"
133-
)
119+
self.dec_w_ih: "NDArray" = variables["decoder.rnn.weight_ih_l0"]
134120
# (3*128, 128)
135-
self.dec_w_hh: "NDArray" = self.variables.item().get(
136-
"decoder.rnn.weight_hh_l0"
137-
)
121+
self.dec_w_hh: "NDArray" = variables["decoder.rnn.weight_hh_l0"]
138122
# (3*128,)
139-
self.dec_b_ih: "NDArray" = self.variables.item().get(
140-
"decoder.rnn.bias_ih_l0"
141-
)
123+
self.dec_b_ih: "NDArray" = variables["decoder.rnn.bias_ih_l0"]
142124
# (3*128,)
143-
self.dec_b_hh: "NDArray" = self.variables.item().get(
144-
"decoder.rnn.bias_hh_l0"
145-
)
125+
self.dec_b_hh: "NDArray" = variables["decoder.rnn.bias_hh_l0"]
146126
# (74, 128)
147-
self.fc_w: "NDArray" = self.variables.item().get("decoder.fc.weight")
127+
self.fc_w: "NDArray" = variables["decoder.fc.weight"]
148128
# (74,)
149-
self.fc_b: "NDArray" = self.variables.item().get("decoder.fc.bias")
129+
self.fc_b: "NDArray" = variables["decoder.fc.bias"]
150130

151131
def _sigmoid(self, x: "np.ndarray") -> "np.ndarray":
152132
import numpy as np

0 commit comments

Comments
 (0)