diff --git a/itn/english/__init__.py b/itn/english/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/itn/english/data/currency.tsv b/itn/english/data/currency.tsv new file mode 100644 index 00000000..2f1fcc09 --- /dev/null +++ b/itn/english/data/currency.tsv @@ -0,0 +1,168 @@ +؋ afghan afghani +l albanian lek +دج algerian dinar +kz angolan kwanza +aed arab emirates dirham +$ argentine peso +֏ armenian dram +ƒ aruban florin +a$ australian dollar +₼ azerbaijani manat +b$ bahamian dollar +.د.ب bahraini dinar +৳ bangladeshi taka +$ barbadian dollar +br belarusian ruble +bz$ belize dollar +$ bermudian dollar +nu bhutanese ngultrum +₿ bitcoin +bs bolivian boliviano +km bosnia and herzegovina convertible mark +p botswana pula +r$ brazilian real +£ british pound +b$ brunei dollar +лв bulgarian lev +fbu burundian franc +៛ cambodian riel +ca$ canadian dollar +cve cape verde escudo +cve cape verdean escudo +ci$ cayman islands dollar +cfa cfa franc +$ chilean peso +元 chinese yuan +$ colombian peso +cf comorian franc +fc congolese franc +₡ costa rican colón +kn croatian kuna +cuc$ cuban peso +cyp cypriot pound +kč czech koruna +d dalasi +kr danish krone +fdj djiboutian franc +$ dollar +rd$ dominican peso +$ east caribbean dollar +e£ egyptian pound +aed emirati dirham +nkf eritrean nakfa +Ξ ethereum +br ethiopian birr +€ euro +fk£ falkland islands pound +fim finnish markka +₾ georgian lari +dm german mark +gh₵ ghanaian cedi +q guatemalan quetzal +fg guinean franc +g$ guyanese dollar +g haitian gourde +l honduran lempira +hk$ hong kong dollar +ft hungarian forint +₹ inr +íkr icelandic króna +₹ indian rupee +rp indonesian rupiah +﷼ iranian rial +ع.د iraqi dinar +₪ israeli new shekel +₪ israeli shekel +j$ jamaican dollar +¥ japanese yen +ينار jordanian dinar +₸ kazakhstani tenge +ksh kenyan shilling +ك kuwaiti dinar +k kyat +som kyrgyzstani som +₭ lao kip +ლარი lari +ل.ل lebanese pound +l lesotho loti +ld$ liberian dollar +ld libyan dinar +l lilangeni +ł litecoin +mop$ macanese pataca +ar malagasy ariary +k malawian kwacha +rm malaysian ringgit +rf maldivian rufiyaa +mro mauritanian ouguiya +₨ mauritian rupee +$ mexican peso +l moldovan leu +ɱ monero +₮ mongolian tögrög +dh moroccan dirham +mt mozambican metical +n$ namibian dollar +rs nepalese rupee +ƒ netherlands antillean guilder +nt$ new taiwan dollar +nz$ new zealand dollar +c$ nicaraguan córdoba +₦ nigerian naira +₩ north korean won +kr norwegian krone +ر.ع omani rial +um ouguiya +pkr pakistani rupee +b/. panamanian balboa +₲ paraguayan guaraní +s/. peruvian sol +₱ philippine peso +zł polish zloty +£ pounds sterling +ر.ق qatari riyal +£ quid +r real +xrp ripples +lei romanian leu +₹ rupee +₽ russian ruble +r₣ rwandan franc +shp saint helena pounds +db são tomé and príncipe dobra +sar saudi riyal +ден second macedonian denar +rsd serbian dinar +scr seychelles rupee +le sierra leonean leone +s$ singapore dollar +sh.so. somali shilling +tjs somoni +r south african rand +₩ south korean won +ss£ south sudanese pound +රු sri lankan rupee +sdg sudanese pound +sr$ surinamese dollar +kr swedish krona +chf swiss franc +£s syrian pound +৳ taka +tzs tanzanian shilling +₮ tether +฿ thai baht +tt$ trinidad and tobago dollar +د.ت tunisian dinar +₺ turkish lira +m turkmen new manat +ush ugandan shilling +₴ ukrainian hryvna +$ united states dollar +$u uruguayan peso +$ us dollar +som uzbekistan som +bs. venezuelan bolívar +₫ vietnamese đồng +₩ won +﷼ yemeni rial +¥ yen diff --git a/itn/english/data/date_period.tsv b/itn/english/data/date_period.tsv new file mode 100644 index 00000000..b53ef7db --- /dev/null +++ b/itn/english/data/date_period.tsv @@ -0,0 +1,10 @@ +Q1 first quarter +Q1 q one +Q2 second quarter +Q2 q two +Q3 third quarter +Q3 q three +Q4 fourth quarter +Q4 q four +H1 first half +H2 second half \ No newline at end of file diff --git a/itn/english/data/electronic/domain.tsv b/itn/english/data/electronic/domain.tsv new file mode 100644 index 00000000..207a14ae --- /dev/null +++ b/itn/english/data/electronic/domain.tsv @@ -0,0 +1,10 @@ +com +uk +fr +net +br +in +ru +de +it +ai diff --git a/itn/english/data/electronic/server_name.tsv b/itn/english/data/electronic/server_name.tsv new file mode 100644 index 00000000..ca7f97bc --- /dev/null +++ b/itn/english/data/electronic/server_name.tsv @@ -0,0 +1,17 @@ +g mail gmail +gmail +n vidia nvidia +nvidia +outlook +hotmail +yahoo +aol +gmx +msn +live +yandex +orange +wanadoo +web +comcast +google diff --git a/itn/english/data/electronic/symbols.tsv b/itn/english/data/electronic/symbols.tsv new file mode 100644 index 00000000..5be8da87 --- /dev/null +++ b/itn/english/data/electronic/symbols.tsv @@ -0,0 +1,22 @@ +. dot +- dash +- hyphen +_ underscore +! exclamation mark +# number sign +$ dollar sign +% percent sign +& ampersand +' quote +* asterisk ++ plus +/ slash += equal sign +? question mark +^ circumflex +` right single quote +{ left brace +| vertical bar +} right brace +~ tilde +, comma \ No newline at end of file diff --git a/itn/english/data/electronic/url_symbols.tsv b/itn/english/data/electronic/url_symbols.tsv new file mode 100644 index 00000000..2946fde3 --- /dev/null +++ b/itn/english/data/electronic/url_symbols.tsv @@ -0,0 +1,5 @@ +. dot +- dash +- hyphen +_ underscore +/ slash \ No newline at end of file diff --git a/itn/english/data/magnitudes.tsv b/itn/english/data/magnitudes.tsv new file mode 100644 index 00000000..3bfa4849 --- /dev/null +++ b/itn/english/data/magnitudes.tsv @@ -0,0 +1,4 @@ +k thousand +m million +b billion +t trillion \ No newline at end of file diff --git a/itn/english/data/measurements.tsv b/itn/english/data/measurements.tsv new file mode 100644 index 00000000..894eacd4 --- /dev/null +++ b/itn/english/data/measurements.tsv @@ -0,0 +1,145 @@ +°F fahrenheit +°C celsius +°F degree fahrenheit +°C degree celsius +K kelvin +km kilometer +m meter +cm centimeter +mm millimeter +ha hectare +mi mile +m² square meter +km² square kilometer +ft foot +% percent +hz hertz +kW kilowatt +kW kilo watt +kWh kilo watt hour +kWh kilowatt hour +Wh watt hour +W watt +hp horsepower +hp horse power +mg milligram +kg kilogram +ghz gigahertz +khz kilohertz +mhz megahertz +v volt +h hour +mc mega coulomb +s second +nm nanometer +rpm revolution per minute +min minute +mA milli ampere +% per cent +m³ cubic meter +mph mile per hour +tW tera watt +mv milli volt +mW megawatt +mW mega watt +μm micrometer +" inch +cc c c +g gram +da dalton +atm atmosphere +ω ohm +db decibel +ps peta second +oz ounce +hl hecto liter +μg microgram +pg petagram +kb kilobit +mb megabit +gb gigabit +tb terabit +pb petabit +ev electron volt +mb megabyte +kb kilobyte +gb gigabyte +tb terabyte +pb peta byte +bps bit per second +kbps kilobit per second +mbps megabit per second +gbps gigabit per second +kbps kilo bit per second +mbps mega bit per second +mbps mega bit per second +tbps terabit per second +tbps tera bit per second +pbps petabit per second +pbps peta bit per second +kb/s kilobyte per second +kb/s kilo byte per second +mb/s megabyte per second +mb/s mega byte per second +gb/s gigabyte per second +gb/s giga byte per second +tb/s terabyte per second +tb/s tera byte per second +pb/s petabyte per second +pb/s peta byte per second +st stone +kl kilo liter +tj tera joule +kv kilo volt +mv mega volt +kn kilo newton +kn kilonewton +mm megameter +au astronomical unit +yd yard +rad radian +lm lumen +hs hecto second +mol mole +gpa giga pascal +ml milliliter +gW gigawatt +gW gigaWatt +A ampere +mA mili ampere +µA micro ampere +MA mega ampere +kt knot +kgf kilogram force +ng nano gram +ns nanosecond +ms mega siemens +bar bar +gl giga liter +μs microsecond +dA deci ampere +pa pascal +ds deci second +ms milli second +dm deci meter +dm³ cubic deci meter +amu atomic mass unit +mf mega farad +bq becquerel +pb petabit +mm² square millimeter +cm² square centimeter +sq mi square mile +sq ft square foot +kpa kilopascal +cd candela +tl tera liter +ms mega second +mpa megapascal +pm peta meter +gWh giga watt hour +kcal kilo calory +gy gray +sv sievert +cwt hundredweight +cc c c diff --git a/itn/english/data/months.tsv b/itn/english/data/months.tsv new file mode 100644 index 00000000..1c05ac02 --- /dev/null +++ b/itn/english/data/months.tsv @@ -0,0 +1,12 @@ +january +february +march +april +may +june +july +august +september +october +november +december diff --git a/itn/english/data/months_cased.tsv b/itn/english/data/months_cased.tsv new file mode 100644 index 00000000..750d7937 --- /dev/null +++ b/itn/english/data/months_cased.tsv @@ -0,0 +1,12 @@ +January +February +March +April +May +June +July +August +September +October +November +December diff --git a/itn/english/data/numbers/digit.tsv b/itn/english/data/numbers/digit.tsv new file mode 100644 index 00000000..11416c03 --- /dev/null +++ b/itn/english/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +one 1 +two 2 +three 3 +four 4 +five 5 +six 6 +seven 7 +eight 8 +nine 9 diff --git a/itn/english/data/numbers/hundred.tsv b/itn/english/data/numbers/hundred.tsv new file mode 100644 index 00000000..719ba84f --- /dev/null +++ b/itn/english/data/numbers/hundred.tsv @@ -0,0 +1 @@ +hundred diff --git a/itn/english/data/numbers/teen.tsv b/itn/english/data/numbers/teen.tsv new file mode 100644 index 00000000..445245c6 --- /dev/null +++ b/itn/english/data/numbers/teen.tsv @@ -0,0 +1,10 @@ +ten 10 +eleven 11 +twelve 12 +thirteen 13 +fourteen 14 +fifteen 15 +sixteen 16 +seventeen 17 +eighteen 18 +nineteen 19 diff --git a/itn/english/data/numbers/thousands.tsv b/itn/english/data/numbers/thousands.tsv new file mode 100644 index 00000000..35c2d413 --- /dev/null +++ b/itn/english/data/numbers/thousands.tsv @@ -0,0 +1,22 @@ +thousand +million +billion +trillion +quadrillion +quintillion +sextillion +septillion +octillion +nonillion +decillion +undecillion +duodecillion +tredecillion +quattuordecillion +quindecillion +sexdecillion +septendecillion +octodecillion +novemdecillion +vigintillion +centillion diff --git a/itn/english/data/numbers/ties.tsv b/itn/english/data/numbers/ties.tsv new file mode 100644 index 00000000..21057ae2 --- /dev/null +++ b/itn/english/data/numbers/ties.tsv @@ -0,0 +1,9 @@ +twenty 2 +thirty 3 +forty 4 +fourty 4 +fifty 5 +sixty 6 +seventy 7 +eighty 8 +ninety 9 diff --git a/itn/english/data/numbers/zero.tsv b/itn/english/data/numbers/zero.tsv new file mode 100644 index 00000000..a1b116c1 --- /dev/null +++ b/itn/english/data/numbers/zero.tsv @@ -0,0 +1 @@ +zero 0 diff --git a/itn/english/data/ordinals/digit.tsv b/itn/english/data/ordinals/digit.tsv new file mode 100644 index 00000000..4b7fc24f --- /dev/null +++ b/itn/english/data/ordinals/digit.tsv @@ -0,0 +1,9 @@ +first one +second two +third three +fourth four +fifth five +sixth sixth +seventh seven +eighth eight +ninth nine \ No newline at end of file diff --git a/itn/english/data/ordinals/teen.tsv b/itn/english/data/ordinals/teen.tsv new file mode 100644 index 00000000..496fefc0 --- /dev/null +++ b/itn/english/data/ordinals/teen.tsv @@ -0,0 +1 @@ +twelfth twelve \ No newline at end of file diff --git a/itn/english/data/time/minute_to.tsv b/itn/english/data/time/minute_to.tsv new file mode 100644 index 00000000..edab4d5b --- /dev/null +++ b/itn/english/data/time/minute_to.tsv @@ -0,0 +1,59 @@ +1 59 +2 58 +3 57 +4 56 +5 55 +6 54 +7 53 +8 52 +9 51 +10 50 +11 49 +12 48 +13 47 +14 46 +15 45 +16 44 +17 43 +18 42 +19 41 +20 40 +21 39 +22 38 +23 37 +24 36 +25 35 +26 34 +27 33 +28 32 +29 31 +30 30 +31 29 +32 28 +33 27 +34 26 +35 25 +36 24 +37 23 +38 22 +39 21 +40 20 +41 19 +42 18 +43 17 +44 16 +45 15 +46 14 +47 13 +48 12 +49 11 +50 10 +51 9 +52 8 +53 7 +54 6 +55 5 +56 4 +57 3 +58 2 +59 1 diff --git a/itn/english/data/time/time_suffix.tsv b/itn/english/data/time/time_suffix.tsv new file mode 100644 index 00000000..b39a9ef9 --- /dev/null +++ b/itn/english/data/time/time_suffix.tsv @@ -0,0 +1,8 @@ +p m p.m. +pm p.m. +p.m. +p.m p.m. +am a.m. +a.m. +a.m a.m. +a m a.m. \ No newline at end of file diff --git a/itn/english/data/time/time_suffix_cased.tsv b/itn/english/data/time/time_suffix_cased.tsv new file mode 100644 index 00000000..97a4cf5c --- /dev/null +++ b/itn/english/data/time/time_suffix_cased.tsv @@ -0,0 +1,8 @@ +P M P.M. +PM P.M. +P.M. +P.M P.M. +AM A.M. +A.M. +A.M A.M. +A M A.M. \ No newline at end of file diff --git a/itn/english/data/time/time_zone.tsv b/itn/english/data/time/time_zone.tsv new file mode 100644 index 00000000..3e0ade46 --- /dev/null +++ b/itn/english/data/time/time_zone.tsv @@ -0,0 +1,7 @@ +cst c s t +cet c e t +pst p s t +est e s t +pt p t +et e t +gmt g m t diff --git a/itn/english/data/time/time_zone_cased.tsv b/itn/english/data/time/time_zone_cased.tsv new file mode 100644 index 00000000..65c70da2 --- /dev/null +++ b/itn/english/data/time/time_zone_cased.tsv @@ -0,0 +1,7 @@ +CST C S T +CET C E T +PST P S T +EST E S T +PT P T +ET E T +GMT G M T diff --git a/itn/english/data/time/to_hour.tsv b/itn/english/data/time/to_hour.tsv new file mode 100644 index 00000000..fba67f1b --- /dev/null +++ b/itn/english/data/time/to_hour.tsv @@ -0,0 +1,12 @@ +one 12 +two 1 +three 2 +four 3 +five 4 +six 5 +seven 6 +eight 7 +nine 8 +ten 9 +eleven 10 +twelve 11 \ No newline at end of file diff --git a/itn/english/data/whitelist.tsv b/itn/english/data/whitelist.tsv new file mode 100644 index 00000000..c7201aef --- /dev/null +++ b/itn/english/data/whitelist.tsv @@ -0,0 +1,417 @@ +10K ten k +1D one d +1G one g +1K one k +2.5G two point five g +2D two d +2G two g +2K two k +3D three d +3G three g +4D four d +4G four g +4K four k +5D five d +5G five g +6G six g +7-eleven seven eleven +7G seven g +8K eight k +AAA triple a +AC a c +ACL a c l +ADAS a das +AEB a e b +AES a e s +AGX a g x +AI a i +AI ai +AI ais +ALU a l u +AMD a m d +API a p i +API api +ARP a r p +ASAP a s a p +ASAP a sap +ASMC a s m c +ASR a s r +ASUS a sus +ASUS asus +AT&T a t and t +ATM a t m +AV1 a v one +AV1 av one +AVAS a v a s +AVAS a vas +AVRCP a v r c p +AWS a w s +AWS aws +AlphaFold alpha fold +AstraZeneca astra zeneca +AstraZeneca astrazeneca +Audio2Face audio to face +Audio2Face audio too face +BBC b b c +BCP b c p +BEV b e v +BGA b g a +BGP b g p +BIOS Bios +BIOS bios +BIS b i s +BSP b s p +BasePOD base pod +BasePOD basepod +Billion-X billion X +C# c sharp +C++ c plus plus +CAT5 cat five +CAT5e cat five e +CAT6 cat six +CAT6A cat six a +CAT7 cat seven +CAT8 cat eight +CBC c b c +CCS c c s +CCTV c c t v +CCTV c c tv +CDN c d n +CEO c e o +CES c e s +CFO c f o +CHAdeMO cha demo +CHAdeMO chai demo +CLI c l i +CMO c m o +CMS c m s +CNN c n n +COVID covid +COVID-19 covid nineteen +CPU c p u +CSO c s o +CSP c s p +CSS c s s +CV c v +CVT c v t +CalTech cal tech +CalTech caltech +CapEx cap ex +CapEx capex +ChatGPT chat g p t +CloudXR cloud x r +ConnectX connect x +Core 2 core two +DALI d a l i +DB d b +DC d c +DCDB d c d b +DDR d d r +DDoS d d o s +DDoS d dos +DGL d g l +DGX d g x -1.1 +DGX A100 d g x a hundred -1.1 +DGX A100 d g x a one hundred -1.1 +DGX Station d g x station -1.1 +DHCP d h c p +DLSS d l s s +DLSS 2 d l s s two +DLSS 3 d l s s three +DNA d n a +DNN d n n +DNS d n s +DOS dos +DP d p +DPDT d p d t +DPU d p u +DPX d p x +DeepMind deep mind +DeepMind deepmind +DeepStream deep stream +DeepStream deepstream +DevOps dev ops +Direct3D direct three d +DisplayPort display port +EBC e b c +EBS e b s +EC2 e c two +ECC e c c +ECS e c s +EFS e f s +EGX e g x +EKS e k s +ELB e l b +ELCB e l c b +EMR e m r +EOL e o l +EV e v +Earth-2 earth two +FIFO fee foe +FIFO fifo +FOMO foe moe +FOMO fomo +FPS f p s +FSA f s a +FST f s t +FTP f t p +FTPS f t p s +FX f x +G-SYNC g sync +GCC g c c +GCP g c p +GCS g c s +GFN G f n +GPU g p u +GRPC g r p c +GRU g r u +GSK g s k +GTC g t c +GTX g t x -1.1 +GUI g u i +GUI gui +GeForce geforce +GeForce NOW geforce now +H.264 H two six four +H.264 H two sixty four +H.265 H two six five +H.265 H two sixty five +H100 H one hundred +HDMI h d m i +HEVC h e v c +HGX h g x +HMD Hmd +HPC h p c +HQ h q +HSM h s m +HT h t +HTML h t m l +HTTP h t t p +HTTPS h t t p s +HuggingFace hugging face +I/O i o +ICMP i c m p +IEC i e c +IEEE i triple e +IGMP i g m p +IGX i g x +IISc i i s c +IIT i i t +INT8 int eight +IP i p +IPSec i p sec +ISC i s c +ISI i s i +InfiniBand inifiband +IoT i o t +Isaac SIM isaac sim +JSON j son +JavaScript java script +JavaScript javascript +KVM k v m +LFP lee fei po +LFP life po +LGA l g a -1.1 +LLC l l c +LLM l l m +LLMs l l ms +LLP l l p +LSTM l s t m +LT l t +LTT l t t +LinkedIn linked in +LinkedIn linkedin +MB m b +MCB m c b +MCCB m c c b +MCP m c p +MDM m d m +MFA m f a +MIT m i t +ML m l +MLPerf m l perf +MONAI monai +MWC m w c +MXNet m x net +MacBook mac book +MacBook macbook +MacBook Pro mac book pro +Max-Q max q +Million-X million x +MoCap mo cap +NC n c +NGC n g c +NLP n l p +NLU n l u +NMC n m c +NO n o +NTP n t p +NVDec n v dec +NVDec n v deck +NVEnc n v enc +NVEnc n v inc +NVEnc n v ink +NVMe n v m e +Ni-CD knee cad +NiMH n i m h +NoSQL no s q l +NoSQL no sql +Nvidia A100 nvidia a hundred +Nvidia A100 nvidia a one hundred +Nvidia A30 nvidia a thirty +Nvidia A40 nvidia a forty +OCI o c i +OEM o e m +OEMs o e ms +OLTC o l t c +OLTP o l t p +ONNX o n n x +OVX o v x +OpEx op ex +OpEx opex +OpenCL open c l +OpenGL open g l +PCI p c i +PCIe p c i e -1.1 +PDF p d f +PHEV p h e v +POP3 pop three +PSO p s o +Photoshop photo shop +PhysX fizz ex +PhysX fizz x +PhysX phys ex +PhysX phys x +PlayStation play station +Premiere Pro premiere pro +ProVis pro vis +ProVis provis +PyG pi g +PyG py g +PyTorch pi torch +QODA q o d a +QOS q o s +RCBO r c b o +RCCB r c c b +RDS r d s +RFP r f p +RISC risc +RL r l +RNN r n n +ROI r o i +RSS r s s +RT r t +RTP r t p +RTX r t x -1.1 +RTX ON r t x on +RTX Off r t x off +RacerX racer x +Ryzen 2 rye zen two +Ryzen 3 rye zen three +Ryzen 5 rye zen five +Ryzen 7 rye zen seven +Ryzen 9 rye zen nine +S&P s and p +S3 s three +SAE s a e +SDK s d k +SEO s e o +SFU s f u +SHIELD TV shield tv +SIGGRAPH sig graph +SIGGRAPH siggraph +SMPS s m p s +SMTP s m t p +SNMP s n m p +SNS s n s +SOL s o l +SPDT s p d t +SPN s p n +SPST s p s t +SQL s q l +SQS s q s +SSD s s d +SSH s s h +SSL s s l +SSO s s o +SSR s s r +SUV s u v +SWG s w g +SaaS saas +ServiceNow service now +SuperCloud super cloud +SuperPOD super pod +TCP t c p +TCP/IP t c p i p +TFTP t f t p +TJ t j +TOS t o s +TPM t p m +TPMS t p m s +TPN t p n +TSL t s l +TSMC t s m c +TTS t t s +Telnet tell net +TensorRT tensor r t +Tick-Tock tick tock +TikTok app tick tock app +TikTok is tick tock is +UDP u d p +UEFI u e f i +UFI u f i +UPS u p s +USB u s b +VAAPI v a a p i +VAAPI v a api +VDI v d i +VDPAU v d p a u +VDPAU v d paw +VDPAU v d pow +VFD v f d +VLAN v lan +VP v p +VPN v p n +Vcc v c c +Vdd v d d +VoIP v o i p +WFSA w f s a +WFST w f s t +Wi-Fi wi fi +Wi-Fi wifi +XFX x f x +XFX x fx +XML x m l +Zen 2 zen two +Zen 3 zen three +Zen 4 zen four +c# c sharp +c++ c plus plus +cuBLAS cue blah +cuBLAS cue blahs +cuBLAS q blahs +cuDNN c u d n n +cuDNN cue d n n +cuDNN q d n n +cuOpt cue opt +cuOpt q opt +cuQuantum cue quantum +cuQuantum q quantum +dr. doctor +e.g. for example +es3 e s three +ext4 e x t four +i3 i three +i5 i five +i7 i seven +i9 i nine +iOS ios +iTPMS i t p m s +mr. mister +mrs. misses +on TikTok on tick tock +sFTP s f t p +st. saint +x1 x one +x2 x two +x8 x eight \ No newline at end of file diff --git a/itn/english/data/whitelist_tech.tsv b/itn/english/data/whitelist_tech.tsv new file mode 100644 index 00000000..a2a14a41 --- /dev/null +++ b/itn/english/data/whitelist_tech.tsv @@ -0,0 +1,450 @@ +10K ten k +1D one d +1G one g +1K one k +2.5G two point five g +2D two d +2G two g +2K two k +3D three d +3G three g +4D four d +4G four g +4K four k +5D five d +5G five g +6G six g +7-eleven seven eleven +7G seven g +8K eight k +AAA triple a +AC a c +ACL a c l +ADAS a das +AEB a e b +AES a e s +AGX a g x +AI a i +AI ai +AI ais +ALU a l u +AMD a m d +API a p i +API api +ARP a r p +ASAP a s a p +ASAP a sap +ASMC a s m c +ASR a s r +ASUS a sus +ASUS asus +AT&T a t and t +ATM a t m +AV1 a v one +AV1 av one +AVAS a v a s +AVAS a vas +AVRCP a v r c p +AWS a w s +AWS aws +AlphaFold alpha fold +AstraZeneca astra zeneca +AstraZeneca astrazeneca +Audio2Face audio to face +Audio2Face audio too face +BBC b b c +BCP b c p +BEV b e v +BGA b g a +BGP b g p +BIOS Bios +BIOS bios +BIS b i s +BSP b s p +BasePOD base pod +BasePOD basepod +Billion-X billion X +C# c sharp +C++ c plus plus +CAT5 cat five +CAT5e cat five e +CAT6 cat six +CAT6A cat six a +CAT7 cat seven +CAT8 cat eight +CBC c b c +CCS c c s +CCTV c c t v +CCTV c c tv +CDN c d n +CEO c e o +CES c e s +CFO c f o +CHAdeMO cha demo +CHAdeMO chai demo +CLI c l i +CMO c m o +CMS c m s +CNN c n n +COVID covid +COVID-19 covid nineteen +CPU c p u +CSO c s o +CSP c s p +CSS c s s +CV c v +CVT c v t +CalTech cal tech +CalTech caltech +CapEx cap ex +CapEx capex +ChatGPT chat g p t +CloudXR cloud x r +ConnectX connect x +Core 2 core two +DALI d a l i +DB d b +DC d c +DCDB d c d b +DDR d d r +DDoS d d o s +DDoS d dos +DGL d g l +DGX d g x -1.1 +DGX A100 d g x a hundred -1.1 +DGX A100 d g x a one hundred -1.1 +DGX Station d g x station -1.1 +DHCP d h c p +DLSS d l s s +DLSS 2 d l s s two +DLSS 3 d l s s three +DNA d n a +DNN d n n +DNS d n s +DOS dos +DP d p +DPDT d p d t +DPU d p u +DPX d p x +DeepMind deep mind +DeepMind deepmind +DeepStream deep stream +DeepStream deepstream +DevOps dev ops +Direct3D direct three d +DisplayPort display port +EBC e b c +EBS e b s +EC2 e c two +ECC e c c +ECS e c s +EFS e f s +EGX e g x +EKS e k s +ELB e l b +ELCB e l c b +EMR e m r +EOL e o l +EV e v +Earth-2 earth two +FAT12 fat twelve +FAT16 fat sixteen +FAT32 fat thirty two +FIFO fee foe +FIFO fifo +FOMO foe moe +FOMO fomo +FPS f p s +FSA f s a +FST f s t +FTP f t p +FTPS f t p s +FX f x +G-SYNC g sync +GCC g c c +GCP g c p +GCS g c s +GFN G f n +GPL G P L +GPLv2 G P L two +GPLv3 G P L three +GPU g p u +GRPC g r p c +GRU g r u +GSK g s k +GTC g t c +GTX g t x -1.1 +GUI g u i +GUI gui +GeForce geforce +GeForce NOW geforce now +H.264 H two six four +H.264 H two sixty four +H.265 H two six five +H.265 H two sixty five +H100 H one hundred +HDMI h d m i +HEVC h e v c +HFS h f s +HFS+ h f s plus +HGX h g x +HMD Hmd +HPC h p c +HQ h q +HSM h s m +HT h t +HTML h t m l +HTTP h t t p +HTTPS h t t p s +HuggingFace hugging face +I/O i o +ICMP i c m p +IEC i e c +IEEE i triple e +IGMP i g m p +IGX i g x +IISc i i s c +IIT i i t +INT8 int eight +IP i p +IPSec i p sec +ISC i s c +ISI i s i +InfiniBand inifiband +IoT i o t +Isaac SIM isaac sim +JSON j son +JavaScript java script +JavaScript javascript +KVM k v m +LFP lee fei po +LFP life po +LGA l g a -1.1 +LLC l l c +LLM l l m +LLMs l l ms +LLP l l p +LSTM l s t m +LT l t +LTT l t t +LinkedIn linked in +LinkedIn linkedin +MB m b +MCB m c b +MCCB m c c b +MCP m c p +MDM m d m +MFA m f a +MIT m i t +ML m l +MLPerf m l perf +MONAI monai +MWC m w c +MXNet m x net +MacBook mac book +MacBook macbook +MacBook Pro mac book pro +Max-Q max q +Million-X million x +MoCap mo cap +NC n c +NGC n g c +NLP n l p +NLU n l u +NMC n m c +NO n o +NTFS n t f s +NTP n t p +NVDec n v dec +NVDec n v deck +NVEnc n v enc +NVEnc n v inc +NVEnc n v ink +NVMe n v m e +Ni-CD knee cad +NiMH n i m h +NoSQL no s q l +NoSQL no sql +Nvidia A100 Nvidia a hundred +Nvidia A100 Nvidia a one hundred +Nvidia A100 nvidia a hundred +Nvidia A100 nvidia a one hundred +Nvidia A30 Nvidia a thirty +Nvidia A30 nvidia a thirty +Nvidia A40 Nvidia a forty +Nvidia A40 nvidia a forty +OCI o c i +OEM o e m +OEMs o e ms +OLTC o l t c +OLTP o l t p +ONNX o n n x +OVX o v x +OpEx op ex +OpEx opex +OpenCL open c l +OpenGL open g l +PCI p c i +PCIe p c i e -1.1 +PDF p d f +PHEV p h e v +POP3 pop three +PSO p s o +Photoshop photo shop +PhysX fizz ex +PhysX fizz x +PhysX phys ex +PhysX phys x +PlayStation play station +Premiere Pro premiere pro +ProVis pro vis +ProVis provis +PyG pi g +PyG py g +PyTorch pi torch +QODA q o d a +QOS q o s +RCBO r c b o +RCCB r c c b +RDS r d s +RFP r f p +RISC risc +RL r l +RNN r n n +ROI r o i +RSS r s s +RT r t +RTP r t p +RTX r t x -1.1 +RTX ON r t x on +RTX Off r t x off +RacerX racer x +Ryzen 2 rye zen two +Ryzen 3 rye zen three +Ryzen 5 rye zen five +Ryzen 7 rye zen seven +Ryzen 9 rye zen nine +S&P s and p +S3 s three +SAE s a e +SDK s d k +SEO s e o +SFU s f u +SHIELD TV shield tv +SIGGRAPH sig graph +SIGGRAPH siggraph +SMPS s m p s +SMTP s m t p +SNMP s n m p +SNS s n s +SOL s o l +SPDT s p d t +SPN s p n +SPST s p s t +SQL s q l +SQS s q s +SSD s s d +SSH s s h +SSL s s l +SSO s s o +SSR s s r +SUV s u v +SWG s w g +SaaS saas +ServiceNow service now +SuperCloud super cloud +SuperPOD super pod +TCP t c p +TCP/IP t c p i p +TFTP t f t p +TJ t j +TOS t o s +TPM t p m +TPMS t p m s +TPN t p n +TSL t s l +TSMC t s m c +TTS t t s +Telnet tell net +TensorRT tensor r t +The A100 The a hundred +The A100 The a one hundred +The A100 the a hundred +The A100 the a one hundred +The A30 The a thirty +The A30 the a thirty +The A40 The a forty +The A40 the a forty +Tick-Tock tick tock +TikTok app tick tock app +TikTok is tick tock is +UDP u d p +UEFI u e f i +UFI u f i +UFS u f s +UPS u p s +USB u s b +VAAPI v a a p i +VAAPI v a api +VDI v d i +VDPAU v d p a u +VDPAU v d paw +VDPAU v d pow +VFD v f d +VLAN v lan +VMFS v m f s +VP v p +VPN v p n +Vcc v c c +Vdd v d d +VoIP v o i p +WFSA w f s a +WFST w f s t +Wi-Fi wi fi +Wi-Fi wifi +XFX x f x +XFX x fx +XML x m l +ZFS z f s +Zen 2 zen two +Zen 3 zen three +Zen 4 zen four +b-tree b t ree +btrfs b t r f s +btrfs b tree f s +btrfs butter f s +c# c sharp +c++ c plus plus +cuBLAS cue blah +cuBLAS cue blahs +cuBLAS q blahs +cuDNN c u d n n +cuDNN cue d n n +cuDNN q d n n +cuOpt cue opt +cuOpt q opt +cuQuantum cue quantum +cuQuantum q quantum +dr. doctor +e.g. for example +es3 e s three +exFAT ex fat +ext3 e x t three +ext4 e x t four +i3 i three +i5 i five +i7 i seven +i9 i nine +iOS ios +iTPMS i t p m s +mr. mister +mrs. misses +on TikTok on tick tock +sFTP s f t p +st. saint +x1 x one +x16 x sixteen +x2 x two +x4 x four +x8 x eight +xfs x f s diff --git a/itn/english/data/year_suffix.tsv b/itn/english/data/year_suffix.tsv new file mode 100644 index 00000000..65a05a18 --- /dev/null +++ b/itn/english/data/year_suffix.tsv @@ -0,0 +1,6 @@ +AD a d +BC b c +BCE b c e +BCE before common era +CE c e +CE common era diff --git a/itn/english/rules/__init__.py b/itn/english/rules/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/itn/english/rules/cardinal.py b/itn/english/rules/cardinal.py new file mode 100644 index 00000000..4f597199 --- /dev/null +++ b/itn/english/rules/cardinal.py @@ -0,0 +1,127 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini import closure, cross, string_file, union +from pynini.lib.pynutil import delete, insert + +from tn.processor import Processor +from tn.utils import get_abs_path + + +class Cardinal(Processor): + + def __init__(self): + super().__init__(name="cardinal", ordertype="itn") + self.build_tagger() + self.build_verbalizer() + + def build_tagger(self): + zero = string_file(get_abs_path("../itn/english/data/numbers/zero.tsv")) + digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv")) + teen = string_file(get_abs_path("../itn/english/data/numbers/teen.tsv")) + ties = string_file(get_abs_path("../itn/english/data/numbers/ties.tsv")) + ds = delete(" ") + + # 1~9, 10~19, 20~99 + one_digit = digit + two_digit = teen | (ties + (ds + digit | insert("0"))) + up_to_99 = one_digit | two_digit + + # one hundred, one hundred twenty three, one hundred one + hundreds = digit + ds + delete("hundred") + (ds + two_digit | ds + insert("0") + one_digit | insert("00")) + # eleven hundred => 1100, twenty one hundred => 2100 + hundreds_as_thousand = (teen | ties + ds + digit) + ds + delete("hundred") + ( + ds + two_digit | ds + insert("0") + one_digit | insert("00") + ) + + # 1~999 + up_to_999 = up_to_99 | hundreds + # 1~999 with zero-padding to 3 digits + up_to_999_padded = hundreds | insert("0") + two_digit | insert("00") + one_digit + + def _with_mag(name): + return up_to_999 + ds + delete(name) + + def _with_mag_padded(name): + return up_to_999_padded + ds + delete(name) + + graph = zero | up_to_999 | hundreds_as_thousand + graph |= _with_mag("thousand") + (ds + up_to_999_padded | insert("000")) + graph |= ( + _with_mag("million") + + (ds + _with_mag_padded("thousand") | insert("000")) + + (ds + up_to_999_padded | insert("000")) + ) + graph |= ( + _with_mag("billion") + + (ds + _with_mag_padded("million") | insert("000")) + + (ds + _with_mag_padded("thousand") | insert("000")) + + (ds + up_to_999_padded | insert("000")) + ) + graph |= ( + _with_mag("trillion") + + (ds + _with_mag_padded("billion") | insert("000")) + + (ds + _with_mag_padded("million") | insert("000")) + + (ds + _with_mag_padded("thousand") | insert("000")) + + (ds + up_to_999_padded | insert("000")) + ) + graph |= ( + _with_mag("quadrillion") + + (ds + _with_mag_padded("trillion") | insert("000")) + + (ds + _with_mag_padded("billion") | insert("000")) + + (ds + _with_mag_padded("million") | insert("000")) + + (ds + _with_mag_padded("thousand") | insert("000")) + + (ds + up_to_999_padded | insert("000")) + ) + graph |= ( + _with_mag("quintillion") + + (ds + _with_mag_padded("quadrillion") | insert("000")) + + (ds + _with_mag_padded("trillion") | insert("000")) + + (ds + _with_mag_padded("billion") | insert("000")) + + (ds + _with_mag_padded("million") | insert("000")) + + (ds + _with_mag_padded("thousand") | insert("000")) + + (ds + up_to_999_padded | insert("000")) + ) + graph |= ( + _with_mag("sextillion") + + (ds + _with_mag_padded("quintillion") | insert("000")) + + (ds + _with_mag_padded("quadrillion") | insert("000")) + + (ds + _with_mag_padded("trillion") | insert("000")) + + (ds + _with_mag_padded("billion") | insert("000")) + + (ds + _with_mag_padded("million") | insert("000")) + + (ds + _with_mag_padded("thousand") | insert("000")) + + (ds + up_to_999_padded | insert("000")) + ) + + # strip leading zeros + graph = graph @ union(delete(closure("0")) + (self.DIGIT - "0") + closure(self.DIGIT), "0") + # delete optional "and" + delete_and = self.build_rule(delete("and "), " ", self.ALPHA) + graph = (delete_and @ graph).optimize() + + self.graph = graph + + minus = delete("minus") | delete("negative") + optional_minus = closure(insert('negative: "-" ') + minus + ds, 0, 1) + final_graph = optional_minus + insert('integer: "') + graph + insert('"') + self.tagger = self.add_tokens(final_graph) + + def build_verbalizer(self): + optional_sign = closure( + delete('negative:') + self.DELETE_SPACE + delete('"') + self.NOT_QUOTE + delete('"') + self.DELETE_SPACE, + 0, 1, + ) + integer = delete("integer:") + self.DELETE_SPACE + delete('"') + self.NOT_QUOTE.plus + delete('"') + self.numbers = integer + self.verbalizer = self.delete_tokens(optional_sign + integer) diff --git a/itn/english/rules/char.py b/itn/english/rules/char.py new file mode 100644 index 00000000..05e6dd4f --- /dev/null +++ b/itn/english/rules/char.py @@ -0,0 +1,29 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib.pynutil import insert + +from tn.processor import Processor + + +class Char(Processor): + + def __init__(self): + super().__init__(name="char", ordertype="itn") + self.build_tagger() + self.build_verbalizer() + + def build_tagger(self): + tagger = insert('value: "') + self.CHAR + insert('"') + self.tagger = self.add_tokens(tagger) diff --git a/itn/english/rules/decimal.py b/itn/english/rules/decimal.py new file mode 100644 index 00000000..5446f09c --- /dev/null +++ b/itn/english/rules/decimal.py @@ -0,0 +1,61 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini import closure, cross, string_file +from pynini.lib.pynutil import delete, insert + +from itn.english.rules.cardinal import Cardinal +from tn.processor import Processor +from tn.utils import get_abs_path + + +class Decimal(Processor): + + def __init__(self, cardinal=None): + super().__init__(name="decimal", ordertype="itn") + self.cardinal = cardinal or Cardinal() + self.build_tagger() + self.build_verbalizer() + + def build_tagger(self): + digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv")) + zero = string_file(get_abs_path("../itn/english/data/numbers/zero.tsv")) + ds = delete(" ") + + # fractional part: digit by digit, "o" => 0 + frac_digit = digit | zero | cross("o", "0") + frac_graph = closure(frac_digit + ds) + frac_digit + + optional_negative = closure( + insert('negative: "true" ') + delete("minus") + ds, 0, 1 + ) + integer_part = insert('integer_part: "') + self.cardinal.graph + insert('"') + frac_part = insert('fractional_part: "') + frac_graph + insert('"') + point = delete("point") + + graph = optional_negative + closure(integer_part + ds, 0, 1) + point + ds + frac_part + self.tagger = self.add_tokens(graph) + + def build_verbalizer(self): + optional_sign = closure(cross('negative: "true"', "-") + self.DELETE_SPACE, 0, 1) + integer = delete('integer_part:') + self.DELETE_SPACE + delete('"') + self.NOT_QUOTE.plus + delete('"') + optional_integer = closure(integer + self.DELETE_SPACE, 0, 1) + fractional = ( + insert(".") + delete('fractional_part:') + self.DELETE_SPACE + + delete('"') + self.NOT_QUOTE.plus + delete('"') + ) + optional_fractional = closure(fractional + self.DELETE_SPACE, 0, 1) + graph = optional_sign + optional_integer + optional_fractional + self.numbers = graph + self.verbalizer = self.delete_tokens(graph) diff --git a/itn/english/rules/inverse_normalizer.py b/itn/english/rules/inverse_normalizer.py new file mode 100644 index 00000000..446ffe35 --- /dev/null +++ b/itn/english/rules/inverse_normalizer.py @@ -0,0 +1,56 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from importlib_resources import files +from pynini.lib.pynutil import add_weight, delete + +from itn.english.rules.cardinal import Cardinal +from itn.english.rules.char import Char +from itn.english.rules.decimal import Decimal +from itn.english.rules.ordinal import Ordinal +from tn.processor import Processor + + +class InverseNormalizer(Processor): + + def __init__(self, cache_dir=None, overwrite_cache=False): + super().__init__(name="en_inverse_normalizer", ordertype="itn") + if cache_dir is None: + cache_dir = files("itn") + self.build_fst("en_itn", cache_dir, overwrite_cache) + + def build_tagger_and_verbalizer(self): + cardinal = Cardinal() + ordinal = Ordinal(cardinal=cardinal) + decimal = Decimal(cardinal=cardinal) + char = Char() + + tagger = ( + add_weight(ordinal.tagger, 1.0) + | add_weight(decimal.tagger, 1.01) + | add_weight(cardinal.tagger, 1.02) + | add_weight(char.tagger, 100) + ).optimize() + + tagger = tagger.star + self.tagger = tagger @ self.build_rule(delete(" "), "", "[EOS]") + + verbalizer = ( + cardinal.verbalizer + | ordinal.verbalizer + | decimal.verbalizer + | char.verbalizer + ).optimize() + + self.verbalizer = verbalizer.star diff --git a/itn/english/rules/ordinal.py b/itn/english/rules/ordinal.py new file mode 100644 index 00000000..b61306f3 --- /dev/null +++ b/itn/english/rules/ordinal.py @@ -0,0 +1,52 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini import closure, cross, string_file +from pynini.lib.pynutil import delete, insert + +from itn.english.rules.cardinal import Cardinal +from tn.processor import Processor +from tn.utils import get_abs_path + + +class Ordinal(Processor): + + def __init__(self, cardinal=None): + super().__init__(name="ordinal", ordertype="itn") + self.cardinal = cardinal or Cardinal() + self.build_tagger() + self.build_verbalizer() + + def build_tagger(self): + graph_digit = string_file(get_abs_path("../itn/english/data/ordinals/digit.tsv")) + graph_teen = string_file(get_abs_path("../itn/english/data/ordinals/teen.tsv")) + # first => one => 1, twelfth => twelve => 12, twentieth => twenty => 2(0) + suffix = graph_digit | graph_teen | cross("tieth", "ty") | cross("th", "") + graph = closure(self.VCHAR) + suffix + self.graph = pynini.compose(graph, self.cardinal.graph) + + final_graph = insert('integer: "') + self.graph + insert('"') + self.tagger = self.add_tokens(final_graph) + + def build_verbalizer(self): + integer = delete("integer:") + self.DELETE_SPACE + delete('"') + self.NOT_QUOTE.plus + delete('"') + # 1 => 1st, 2 => 2nd, 3 => 3rd, 11 => 11th, 12 => 12th, 13 => 13th, rest => Xth + ordinal_suffix = ( + cross("11", "11th") | cross("12", "12th") | cross("13", "13th") + | cross("1", "1st") | cross("2", "2nd") | cross("3", "3rd") + | insert("th", weight=0.01) + ) + graph = integer @ pynini.cdrewrite(ordinal_suffix, "", "[EOS]", self.VSIGMA) + self.verbalizer = self.delete_tokens(graph) diff --git a/itn/english/test/__init__.py b/itn/english/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/itn/english/test/data/en_cardinal.txt b/itn/english/test/data/en_cardinal.txt new file mode 100644 index 00000000..1fd0c978 --- /dev/null +++ b/itn/english/test/data/en_cardinal.txt @@ -0,0 +1,22 @@ +twenty three => 23 +one hundred => 100 +one hundred and one => 101 +two hundred and fifty four => 254 +one thousand => 1000 +one thousand two hundred thirty four => 1234 +eleven hundred => 1100 +twenty one hundred => 2100 +twenty one hundred and eleven => 2111 +ten thousand => 10000 +one hundred thousand => 100000 +one million => 1000000 +one billion => 1000000000 +one trillion => 1000000000000 +one thousand and one => 1001 +one million one => 1000001 +one billion five hundred ninety three million seventy two thousand nine hundred sixty one => 1593072961 +zero => 0 +five => 5 +thirty => 30 +minus forty two => -42 +negative five => -5 diff --git a/itn/english/test/data/en_decimal.txt b/itn/english/test/data/en_decimal.txt new file mode 100644 index 00000000..e787ccab --- /dev/null +++ b/itn/english/test/data/en_decimal.txt @@ -0,0 +1,8 @@ +twelve point five => 12.5 +three point one four => 3.14 +minus three point one four => -3.14 +point o five => .05 +point five => .5 +one point zero => 1.0 +zero point five => 0.5 +twenty three point four five six => 23.456 diff --git a/itn/english/test/data/en_ordinal.txt b/itn/english/test/data/en_ordinal.txt new file mode 100644 index 00000000..16e43e6b --- /dev/null +++ b/itn/english/test/data/en_ordinal.txt @@ -0,0 +1,13 @@ +first => 1st +second => 2nd +third => 3rd +fourth => 4th +fifth => 5th +eleventh => 11th +twelfth => 12th +thirteenth => 13th +twenty first => 21st +thirty second => 32nd +forty third => 43rd +one hundredth => 100th +one hundred and first => 101st diff --git a/itn/english/test/normalizer_test.py b/itn/english/test/normalizer_test.py new file mode 100644 index 00000000..4fb07563 --- /dev/null +++ b/itn/english/test/normalizer_test.py @@ -0,0 +1,35 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from itertools import chain + +import pytest + +from itn.english.rules.inverse_normalizer import InverseNormalizer +from itn.english.test.utils import parse_test_case + + +class TestNormalizer: + + normalizer = InverseNormalizer(overwrite_cache=True) + + normalizer_cases = chain( + parse_test_case("data/en_cardinal.txt"), + parse_test_case("data/en_ordinal.txt"), + parse_test_case("data/en_decimal.txt"), + ) + + @pytest.mark.parametrize("spoken, written", normalizer_cases) + def test_normalizer(self, spoken, written): + assert self.normalizer.normalize(spoken) == written diff --git a/itn/english/test/utils.py b/itn/english/test/utils.py new file mode 120000 index 00000000..a2ade7bc --- /dev/null +++ b/itn/english/test/utils.py @@ -0,0 +1 @@ +../../../tn/chinese/test/utils.py \ No newline at end of file