Update to finalfusion 0.11

Author: danieldk

Cargo.lock

@@ -20,7 +20,7 @@ features = ["extension-module"]
 
 [dependencies]
 itertools = "0.8"
-finalfusion = { git = "https://github.com/finalfusion/finalfusion-rust", rev = "66388fe0e46c45d009ffdb8342c76fcb8c9c64d3"}
-ndarray = "0.12"
+finalfusion = "0.11"
+ndarray = "0.13"
 numpy = "0.7"
 toml = "0.5"

Cargo.toml

@@ -4,12 +4,14 @@ use std::fs::File;
 use std::io::{BufReader, BufWriter};
 use std::rc::Rc;
 
-use finalfusion::chunks::metadata::Metadata;
 use finalfusion::compat::text::{ReadText, ReadTextDims};
 use finalfusion::compat::word2vec::ReadWord2Vec;
-use finalfusion::io as ffio;
+use finalfusion::io::{self as ffio, WriteEmbeddings};
+use finalfusion::metadata::Metadata;
 use finalfusion::prelude::*;
 use finalfusion::similarity::*;
+use finalfusion::storage::{NdArray, Storage};
+use finalfusion::vocab::Vocab;
 use itertools::Itertools;
 use ndarray::Array1;
 use numpy::{IntoPyArray, NpyDataType, PyArray1};
@@ -252,7 +254,7 @@ impl PyEmbeddings {
             NonView(e) => e.metadata(),
         };
 
-        match metadata.map(|v| toml::ser::to_string_pretty(&v.0)) {
+        match metadata.map(|v| toml::ser::to_string_pretty(&**v)) {
             Some(Ok(toml)) => Ok(Some(toml)),
             Some(Err(err)) => Err(exceptions::IOError::py_err(format!(
                 "Metadata is invalid TOML: {}",
@@ -278,8 +280,8 @@ impl PyEmbeddings {
 
         use EmbeddingsWrap::*;
         match &mut *embeddings {
-            View(e) => e.set_metadata(Some(Metadata(value))),
-            NonView(e) => e.set_metadata(Some(Metadata(value))),
+            View(e) => e.set_metadata(Some(Metadata::new(value))),
+            NonView(e) => e.set_metadata(Some(Metadata::new(value))),
         };
 
         Ok(())

src/embeddings.rs

@@ -1,6 +1,7 @@
-use finalfusion::chunks::norms::NdNorms;
-use finalfusion::chunks::storage::CowArray1;
+use finalfusion::norms::NdNorms;
 use finalfusion::prelude::*;
+use finalfusion::storage::Storage;
+use ndarray::{CowArray, Ix1};
 
 pub enum EmbeddingsWrap {
     NonView(Embeddings<VocabWrap, StorageWrap>),
@@ -32,7 +33,7 @@ impl EmbeddingsWrap {
         }
     }
 
-    pub fn embedding(&self, word: &str) -> Option<CowArray1<f32>> {
+    pub fn embedding(&self, word: &str) -> Option<CowArray<f32, Ix1>> {
         use EmbeddingsWrap::*;
         match self {
             View(e) => e.embedding(word),

src/embeddings_wrap.rs

@@ -1,7 +1,7 @@
 use std::cell::RefCell;
 use std::rc::Rc;
 
-use finalfusion::prelude::*;
+use finalfusion::vocab::Vocab;
 use numpy::{IntoPyArray, PyArray1};
 use pyo3::class::iter::PyIterProtocol;
 use pyo3::prelude::*;
@@ -35,7 +35,7 @@ impl PyIterProtocol for PyEmbeddingIterator {
         if slf.idx < vocab.words_len() {
             let word = vocab.words()[slf.idx].to_string();
             let embed = embeddings.storage().embedding(slf.idx);
-            let norm = embeddings.norms().map(|n| n.0[slf.idx]).unwrap_or(1.);
+            let norm = embeddings.norms().map(|n| n[slf.idx]).unwrap_or(1.);
 
             slf.idx += 1;
 

src/iter.rs

@@ -1,7 +1,7 @@
 use std::cell::RefCell;
 use std::rc::Rc;
 
-use finalfusion::prelude::{Storage, StorageView, StorageWrap};
+use finalfusion::storage::{Storage, StorageView, StorageWrap};
 use ndarray::Array2;
 use numpy::{PyArray1, PyArray2, ToPyArray};
 use pyo3::class::sequence::PySequenceProtocol;
@@ -30,7 +30,7 @@ impl PyStorage {
 
         let mut array = Array2::<f32>::zeros((rows, dims));
         for idx in 0..rows {
-            array.row_mut(idx).assign(&storage.embedding(idx).as_view());
+            array.row_mut(idx).assign(&storage.embedding(idx).view());
         }
 
         array

src/storage.rs

@@ -1,8 +1,7 @@
 use std::cell::RefCell;
 use std::rc::Rc;
 
-use finalfusion::chunks::vocab::{NGramIndices, SubwordIndices, VocabWrap, WordIndex};
-use finalfusion::prelude::*;
+use finalfusion::vocab::{NGramIndices, SubwordIndices, Vocab, VocabWrap, WordIndex};
 use pyo3::class::sequence::PySequenceProtocol;
 use pyo3::exceptions::{IndexError, KeyError, ValueError};
 use pyo3::prelude::*;
@@ -45,8 +44,8 @@ impl PyVocab {
         let embeds = self.embeddings.borrow();
         Ok(match embeds.vocab() {
             VocabWrap::FastTextSubwordVocab(inner) => inner.ngram_indices(word),
-            VocabWrap::FinalfusionSubwordVocab(inner) => inner.ngram_indices(word),
-            VocabWrap::FinalfusionNGramVocab(inner) => inner.ngram_indices(word),
+            VocabWrap::BucketSubwordVocab(inner) => inner.ngram_indices(word),
+            VocabWrap::ExplicitSubwordVocab(inner) => inner.ngram_indices(word),
             VocabWrap::SimpleVocab(_) => {
                 return Err(ValueError::py_err(
                     "querying n-gram indices is not supported for this vocabulary",
@@ -59,8 +58,8 @@ impl PyVocab {
         let embeds = self.embeddings.borrow();
         match embeds.vocab() {
             VocabWrap::FastTextSubwordVocab(inner) => Ok(inner.subword_indices(word)),
-            VocabWrap::FinalfusionSubwordVocab(inner) => Ok(inner.subword_indices(word)),
-            VocabWrap::FinalfusionNGramVocab(inner) => Ok(inner.subword_indices(word)),
+            VocabWrap::BucketSubwordVocab(inner) => Ok(inner.subword_indices(word)),
+            VocabWrap::ExplicitSubwordVocab(inner) => Ok(inner.subword_indices(word)),
             VocabWrap::SimpleVocab(_) => Err(ValueError::py_err(
                 "querying subwords' indices is not supported for this vocabulary",
             )),