diff --git a/Cargo.lock b/Cargo.lock index 119cbe46a..3e3353394 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + [[package]] name = "anstream" version = "0.6.19" @@ -52,6 +61,29 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "bindgen" +version = "0.66.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b84e06fc203107bfbad243f4aba2af864eb7db3b1cf46ea0a023b0b433d2a7" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "log", + "peeking_take_while", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", + "which", +] + [[package]] name = "bitflags" version = "2.9.1" @@ -77,12 +109,41 @@ dependencies = [ "toml", ] +[[package]] +name = "cc" +version = "1.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc" +dependencies = [ + "shlex", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.5.40" @@ -116,6 +177,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "equivalent" version = "1.0.2" @@ -150,6 +217,12 @@ dependencies = [ "wasi", ] +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + [[package]] name = "hashbrown" version = "0.15.4" @@ -162,11 +235,22 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "home" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys", +] + [[package]] name = "index" version = "0.1.0" dependencies = [ "cbindgen", + "libc", + "ruby-prism", ] [[package]] @@ -191,11 +275,39 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" -version = "0.2.172" +version = "0.2.174" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" + +[[package]] +name = "libloading" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" +dependencies = [ + "cfg-if", + "windows-targets", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "linux-raw-sys" @@ -215,6 +327,22 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -227,6 +355,22 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + +[[package]] +name = "prettyplease" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837b9e10d61f45f987d50808f83d1ee3d206c66acf650c3e4ae2e1f6ddedf55" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro2" version = "1.0.95" @@ -251,6 +395,75 @@ version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "ruby-prism" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45a3a71142531d9dcb33cf162543a68c1d1e5567dd96ac5cc38b8b99b3166b8b" +dependencies = [ + "ruby-prism-sys", + "serde", + "serde_yaml", +] + +[[package]] +name = "ruby-prism-sys" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f77ce848645f060a1f3d60af962f481c58618863390184f96adf5d107e48954" +dependencies = [ + "bindgen", + "cc", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys", +] + [[package]] name = "rustix" version = "1.0.7" @@ -260,7 +473,7 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.9.4", "windows-sys", ] @@ -311,6 +524,25 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "strsim" version = "0.11.1" @@ -337,7 +569,7 @@ dependencies = [ "fastrand", "getrandom", "once_cell", - "rustix", + "rustix 1.0.7", "windows-sys", ] @@ -388,6 +620,12 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "utf8parse" version = "0.2.2" @@ -403,6 +641,18 @@ dependencies = [ "wit-bindgen-rt", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "windows-sys" version = "0.59.0" diff --git a/Cargo.toml b/Cargo.toml index fa3160d12..479b7a019 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,10 @@ crate-type = ["cdylib", "staticlib"] [build-dependencies] cbindgen = "0.29" +[dependencies] +libc = "0.2.174" +ruby-prism = "1.4.0" + [profile.release] lto = true opt-level = 3 diff --git a/cbindgen.toml b/cbindgen.toml index aa90494c6..cf9b01840 100644 --- a/cbindgen.toml +++ b/cbindgen.toml @@ -7,3 +7,6 @@ autogen_warning = "/* Warning, this file is autogenerated by cbindgen. Don't mod tab_width = 4 usize_is_size_t = true + +[enum] +rename_variants = "ScreamingSnakeCase" diff --git a/ext/index/index.c b/ext/index/index.c index da6473349..db2646710 100644 --- a/ext/index/index.c +++ b/ext/index/index.c @@ -1,88 +1,10 @@ #include "ruby.h" #include "rustbindings.h" -#include -#include +#include "repository.h" -// Store the class references -static VALUE cRepository; -static VALUE cEntry; +VALUE mIndex; -static void index_free(void *ptr) { - if (ptr) { - dealloc_repository((CRepository*)ptr); - } -} - -static const rb_data_type_t index_type = { - "Index", - {0, index_free, 0,}, - 0, 0, RUBY_TYPED_FREE_IMMEDIATELY -}; - -static VALUE rb_index_new(VALUE klass) { - CRepository* index = get_repository(); - // TODO: Need to determine a potential way to allow interaction with the index without - // wrapping the object - return TypedData_Wrap_Struct(klass, &index_type, index); -} - -static VALUE rb_repository_add_entry(VALUE self, VALUE name, VALUE value) { - CRepository* index; - TypedData_Get_Struct(self, CRepository, &index_type, index); - - const char *c_name = StringValueCStr(name); - const char *c_value = StringValueCStr(value); - - repository_add_entry(index, c_name, c_value); - return Qnil; -} - -static VALUE rb_repository_get_entry(VALUE self, VALUE name) { - CRepository* index; - TypedData_Get_Struct(self, CRepository, &index_type, index); - - const char *c_name = StringValueCStr(name); - CEntry *c_entry = repository_get_entry(index, c_name); - - if (!c_entry) { - return Qnil; - } - - const char *entry_name = entry_get_name(c_entry); - const char *entry_value = entry_get_value(c_entry); - - // Create a plain Ruby Entry object and set instance variables - // instead of using TypedData_Wrap_Struct - VALUE ruby_entry = rb_class_new_instance(0, NULL, cEntry); - rb_ivar_set(ruby_entry, rb_intern("@name"), entry_name ? rb_str_new_cstr(entry_name) : Qnil); - rb_ivar_set(ruby_entry, rb_intern("@value"), entry_value ? rb_str_new_cstr(entry_value) : Qnil); - - delloc_entry(c_entry); - - return ruby_entry; -} - -static VALUE rb_entry_name(VALUE self) { - return rb_ivar_get(self, rb_intern("@name")); -} - -static VALUE rb_entry_value(VALUE self) { - return rb_ivar_get(self, rb_intern("@value")); -} - -// Initialization function for the Ruby extension void Init_index(void) { - VALUE mIndex = rb_define_module("Index"); - - cRepository = rb_define_class_under(mIndex, "Repository", rb_cObject); - rb_define_singleton_method(cRepository, "new", rb_index_new, 0); - rb_define_method(cRepository, "add_entry", rb_repository_add_entry, 2); - rb_define_method(cRepository, "get_entry", rb_repository_get_entry, 1); - - cEntry = rb_define_class_under(mIndex, "Entry", rb_cObject); - rb_define_method(cEntry, "name", rb_entry_name, 0); - rb_define_method(cEntry, "value", rb_entry_value, 0); - - rb_define_attr(cEntry, "name", 1, 1); - rb_define_attr(cEntry, "value", 1, 1); + mIndex = rb_define_module("Index"); + initialize_repository(mIndex); } diff --git a/ext/index/repository.c b/ext/index/repository.c new file mode 100644 index 000000000..941790eb5 --- /dev/null +++ b/ext/index/repository.c @@ -0,0 +1,122 @@ +#include "ruby.h" +#include "rustbindings.h" + +static VALUE cRepository; +static VALUE cDeclaration; + +static void repository_free(void *ptr) { + if (ptr) { + idx_repository_free(ptr); + } +} + +static const rb_data_type_t repository_type = { + "Repository", + { 0, repository_free, 0 }, + 0, 0, RUBY_TYPED_FREE_IMMEDIATELY +}; + +static VALUE rb_repository_alloc(VALUE klass) { + void* repository = idx_repository_new(); + return TypedData_Wrap_Struct(klass, &repository_type, repository); +} + +// Convert a Ruby array of strings into a double char pointer so that we can pass that to Rust. This copies the data +// so it must be freed +static char** str_array_to_char(VALUE array, long length) { + char **converted_array = malloc(length * sizeof(char*)); + + for (long i = 0; i < length; i++) { + VALUE item = rb_ary_entry(array, i); + const char* string = StringValueCStr(item); + + converted_array[i] = malloc(strlen(string) + 1); + strcpy(converted_array[i], string); + } + + return converted_array; +} + +static VALUE rb_repository_index_all(VALUE self, VALUE file_paths) { + rb_gc_register_address(&file_paths); + long length = RARRAY_LEN(file_paths); + char **converted_file_paths = str_array_to_char(file_paths, length); + + void* repository; + TypedData_Get_Struct(self, void*, &repository_type, repository); + idx_index_all_c(repository, converted_file_paths, length); + + for (long i = 0; i < length; i++) { + free(converted_file_paths[i]); + } + free(converted_file_paths); + rb_gc_unregister_address(&file_paths); + return Qnil; +} + +static VALUE rb_repository_size(VALUE self) { + void* repository; + TypedData_Get_Struct(self, void*, &repository_type, repository); + size_t size = idx_repository_size(repository); + return SIZET2NUM(size); +} + +static VALUE rb_repository_add_class(VALUE self, VALUE name, VALUE start_offset, VALUE end_offset) { + void* repository; + TypedData_Get_Struct(self, void*, &repository_type, repository); + + char* c_name = StringValueCStr(name); + uint32_t c_start_offset = NUM2UINT(start_offset); + uint32_t c_end_offset = NUM2UINT(end_offset); + idx_repository_add_class_declaration(repository, c_name, c_start_offset, c_end_offset); + return Qnil; +} + +static VALUE instantiate_declaration(const Declaration* declaration) { + uint32_t start_offset, end_offset; + uint8_t tag = declaration->tag; + + switch (tag) { + case CLASS: + start_offset = declaration->class_.location.start_offset; + end_offset = declaration->class_.location.end_offset; + break; + case MODULE: + start_offset = declaration->module.location.start_offset; + end_offset = declaration->module.location.end_offset; + break; + default: + rb_raise(rb_eArgError, "Unknown declaration type"); + return Qnil; + } + + VALUE args[] = { UINT2NUM(start_offset), UINT2NUM(end_offset) }; + return rb_class_new_instance(2, args, cDeclaration); +} + +static VALUE rb_repository_get(VALUE self, VALUE name) { + void* repository; + TypedData_Get_Struct(self, void*, &repository_type, repository); + + char* c_name = StringValueCStr(name); + const Declaration* declaration = idx_repository_get(repository, c_name); + + if (declaration == NULL) { + return Qnil; + } + + VALUE ruby_declaration = instantiate_declaration(declaration); + idx_declaration_free(declaration); + return ruby_declaration; +} + +void initialize_repository(VALUE mIndex) { + cRepository = rb_define_class_under(mIndex, "Repository", rb_cObject); + cDeclaration = rb_define_class_under(mIndex, "Declaration", rb_cObject); + + rb_define_alloc_func(cRepository, rb_repository_alloc); + rb_define_method(cRepository, "index_all", rb_repository_index_all, 1); + rb_define_method(cRepository, "size", rb_repository_size, 0); + rb_define_method(cRepository, "add_class", rb_repository_add_class, 3); + rb_define_method(cRepository, "get", rb_repository_get, 1); +} diff --git a/ext/index/repository.h b/ext/index/repository.h new file mode 100644 index 000000000..43f3cc6d1 --- /dev/null +++ b/ext/index/repository.h @@ -0,0 +1,6 @@ +#ifndef REPOSITORY_H +#define REPOSITORY_H + +void initialize_repository(VALUE mIndex); + +#endif diff --git a/lib/index.rb b/lib/index.rb index 7c154ead9..923fcd85a 100644 --- a/lib/index.rb +++ b/lib/index.rb @@ -2,6 +2,7 @@ require "index/version" require "index/index" +require "index/declaration" module Index end diff --git a/lib/index/declaration.rb b/lib/index/declaration.rb new file mode 100644 index 000000000..d538f38be --- /dev/null +++ b/lib/index/declaration.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Index + # Represents the base declaration class for all indexed items. This class is partially defined in C to connect with + # the Rust layer + class Declaration + attr_reader :start_offset, :end_offset + + def initialize(start_offset, end_offset) + @start_offset = start_offset + @end_offset = end_offset + end + end +end diff --git a/src/c_interface.rs b/src/c_interface.rs index 51ca587e5..51ad20fbf 100644 --- a/src/c_interface.rs +++ b/src/c_interface.rs @@ -1,105 +1,148 @@ -use std::ffi::{CString, c_char}; +use crate::declaration::{Declaration, Location}; +use crate::{Repository, index_in_parallel}; +use libc::{c_char, c_void, size_t}; +use std::ffi::CStr; +use std::slice::from_raw_parts; +use std::str::Utf8Error; +use std::sync::{Arc, Mutex}; -use crate::internal::{Entry, Repository}; +// This file contains the Rust to C interface that we provide to the Ruby VM so that we can bind Ruby methods and +// objects to Rust implementations. The only time code should be added here is if we're exposing something to Ruby. All +// public functions that we expose to C should be placed in here so that we have clear separation between the Rust +// crate's API and the C interface used by the Ruby VM. +// +// # Important implementation notes: +// +// ## Returning values to C +// +// The C interface cannot return references to Rust objects because Rust may decide to move, drop or otherwise +// reorganize them. The only exception is returning Arcs. When returning an Arc, we need to ensure that their strong +// count is correctly incremented and that they are dropped when expected. See `retain_repository` and +// `idx_repository_free`. +// +// The other option is returning boxed owned values that we return into pointers with `Box::into_raw`. In these cases, +// data is cloned on demand so that the Ruby layer doesn't keep a reference to Rust data. +// +// ## Freeing memory +// +// Memory allocated by Rust has to be freed by Rust since it can change which allocator is used. Do not try to free Rust +// allocated data from C directly. +// +// ## Naming conventions +// +// C has no support for namespaces, modules or classes. To improve IDE completion and readability, we prefix every +// public C interface function with `idx_` to identify this project + the entity that the function relates to. For +// example, `idx_repository_new` creates a new repository. That way, when typing `idx_repo` we can easily see all +// functions related to repositories and so on. -pub type CRepository = std::ffi::c_void; -pub type CEntry = std::ffi::c_void; +type RepositoryPointer = *mut c_void; +type DeclarationPointer = *const Declaration; -#[unsafe(no_mangle)] -pub extern "C" fn get_repository() -> *mut CRepository { - let repository = Repository::new(); - Box::into_raw(Box::new(repository)) as *mut CRepository +/// Converts an array of C strings (const **char) to a Vec +unsafe fn convert_double_pointer_to_vec(data: &mut &mut c_char, len: size_t) -> Result, Utf8Error> { + unsafe { + from_raw_parts(data, len) + .iter() + .map(|arg| CStr::from_ptr(*arg).to_str().map(ToString::to_string)) + .collect() + } } -/// This will likely be removed -/// -/// # Safety -/// -/// This function is unsafe +/// Frees a pointer to a Declaration object #[unsafe(no_mangle)] -pub unsafe extern "C" fn repository_get_entry(repository: *const CRepository, name: *const c_char) -> *mut CEntry { - if repository.is_null() || name.is_null() { - return std::ptr::null_mut(); - } +pub extern "C" fn idx_declaration_free(pointer: DeclarationPointer) { unsafe { - let repository_ref = &*(repository as *const Repository); - let c_str = std::ffi::CStr::from_ptr(name); - let name_str = match c_str.to_str() { - Ok(s) => s, - Err(_) => return std::ptr::null_mut(), - }; - - match repository_ref.get_entry(name_str) { - Some(entry) => Box::into_raw(Box::new(entry.clone())) as *mut CEntry, - None => std::ptr::null_mut(), - } + let _ = Box::from_raw(pointer as *mut Declaration); } } -/// This will likely be removed -/// -/// # Safety -/// -/// This function is unsafe +/// Creates a new Repository object inside an Arc> and returns a pointer to it. This function is used to create +/// the data that will back the Ruby objects representing repositories #[unsafe(no_mangle)] -pub unsafe extern "C" fn repository_add_entry(repository: *mut CRepository, name: *const c_char, value: *const c_char) { - if repository.is_null() || name.is_null() || value.is_null() { - return; - } - unsafe { - let repository_ref = &mut *(repository as *mut Repository); - let name_cstr = std::ffi::CStr::from_ptr(name); - let value_cstr = std::ffi::CStr::from_ptr(value); - - if let (Ok(name_str), Ok(value_str)) = (name_cstr.to_str(), value_cstr.to_str()) { - let entry = Entry::new(name_str.to_string(), value_str.to_string()); - repository_ref.add_entry(entry); - } - } +pub extern "C" fn idx_repository_new() -> RepositoryPointer { + let repo_arc = Arc::new(Mutex::new(Repository::new())); + Arc::into_raw(repo_arc) as RepositoryPointer } +/// Frees a pointer to a Repository object #[unsafe(no_mangle)] -pub extern "C" fn entry_get_name(entry: *const CEntry) -> *const c_char { - if entry.is_null() { - return std::ptr::null(); - } +pub extern "C" fn idx_repository_free(pointer: RepositoryPointer) { unsafe { - let entry_ref = &*(entry as *const Entry); - match CString::new(entry_ref.name.clone()) { - Ok(c_string) => c_string.into_raw(), - Err(_) => std::ptr::null(), - } + Arc::from_raw(pointer as *mut Mutex); } } -#[unsafe(no_mangle)] -pub extern "C" fn entry_get_value(entry: *const CEntry) -> *const c_char { - if entry.is_null() { - return std::ptr::null(); - } +/// Retains a pointer to a Repository object, incrementing its Arc reference count. This function should always be used +/// when transforming a void pointer back into a Repository object, to ensure that the reference count is updated as +/// expected +fn retain_repository(pointer: RepositoryPointer) -> Arc> { unsafe { - let entry_ref = &*(entry as *const Entry); - match CString::new(entry_ref.value.clone()) { - Ok(c_string) => c_string.into_raw(), - Err(_) => std::ptr::null(), - } + Arc::increment_strong_count(pointer); + Arc::from_raw(pointer as *mut Mutex) } } +/// Indexes all given file paths in parallel using the provided repository pointer +#[unsafe(no_mangle)] +pub extern "C" fn idx_index_all_c(pointer: RepositoryPointer, file_paths: &mut &mut c_char, count: usize) { + let file_paths: Vec = unsafe { convert_double_pointer_to_vec(file_paths, count).unwrap() }; + let file_queue = Arc::new(Mutex::new(file_paths)); + let repository = retain_repository(pointer); + index_in_parallel(&repository, &file_queue); +} + +/// Returns the total number of declarations in the repository +#[unsafe(no_mangle)] +pub extern "C" fn idx_repository_size(pointer: RepositoryPointer) -> usize { + let repository = retain_repository(pointer); + repository.lock().unwrap().size() +} + +/// Allows adding a class declaration to the repository +/// +/// This function has to ensure that all data coming from Ruby is cloned, otherwise we may keep references to data owned +/// by the VM, which could be garbage collected at any time. #[unsafe(no_mangle)] -pub extern "C" fn delloc_entry(entry: *mut CEntry) { - if !entry.is_null() { - unsafe { - let _ = Box::from_raw(entry as *mut Entry); +pub extern "C" fn idx_repository_add_class_declaration( + pointer: RepositoryPointer, + name: &mut c_char, + start_offset: u32, + end_offset: u32, +) { + let repository_arc = retain_repository(pointer); + + let string_name = match unsafe { CStr::from_ptr(name).to_str().map(ToString::to_string) } { + Ok(name) => name, + Err(_) => { + eprintln!("Invalid UTF-8 string for class name"); + return; } - } + }; + + let mut repository = repository_arc.lock().unwrap(); + let location = Location::new(start_offset, end_offset); + repository.add_class(string_name, location); } +/// Retrieve an entry from the repository by its name +/// +/// This function always clones the data. +/// +/// # Safety +/// This function assumes that the name and repository pointer are valid. #[unsafe(no_mangle)] -pub extern "C" fn dealloc_repository(repository: *mut CRepository) { - if !repository.is_null() { - unsafe { - let _ = Box::from_raw(repository as *mut Repository); +pub unsafe extern "C" fn idx_repository_get(pointer: RepositoryPointer, name: *const c_char) -> DeclarationPointer { + let repository_arc = retain_repository(pointer); + let string_name = match unsafe { CStr::from_ptr(name).to_str().map(ToString::to_string) } { + Ok(name) => name, + Err(_) => { + eprintln!("Invalid UTF-8 string for class name"); + return std::ptr::null(); } - } + }; + + let repository = repository_arc.lock().unwrap(); + repository.get(&string_name).map_or_else(std::ptr::null, |declaration| { + Box::into_raw(Box::new(declaration.clone())) as DeclarationPointer + }) } diff --git a/src/declaration.rs b/src/declaration.rs new file mode 100644 index 000000000..e34ed8621 --- /dev/null +++ b/src/declaration.rs @@ -0,0 +1,80 @@ +#[derive(Debug, Clone)] +#[repr(C)] +pub enum Declaration { + Class(ClassDeclaration), + Module(ModuleDeclaration), +} + +#[derive(Debug, Clone)] +#[repr(C)] +pub struct Location { + start_offset: u32, + end_offset: u32, +} + +impl Location { + pub const fn new(start_offset: u32, end_offset: u32) -> Self { + Self { + start_offset, + end_offset, + } + } + + pub fn from_prism_location(prism_location: ruby_prism::Location) -> Self { + Self::from_offsets(prism_location.start_offset(), prism_location.end_offset()) + } + + pub fn from_offsets(start_offset: usize, end_offset: usize) -> Self { + Self { + start_offset: start_offset.try_into().unwrap(), + end_offset: end_offset.try_into().unwrap(), + } + } + + pub fn start_offset(&self) -> usize { + self.start_offset.try_into().unwrap() + } + + pub fn end_offset(&self) -> usize { + self.end_offset.try_into().unwrap() + } +} + +#[derive(Debug, Clone)] +#[repr(C)] +pub struct ClassDeclaration { + location: Location, +} + +#[derive(Debug, Clone)] +#[repr(C)] +pub struct ModuleDeclaration { + location: Location, +} + +impl ClassDeclaration { + pub const fn new(location: Location) -> Self { + Self { location } + } +} + +impl ModuleDeclaration { + pub const fn new(location: Location) -> Self { + Self { location } + } +} + +impl std::fmt::Display for Declaration { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Class(class_decl) => { + let location = &class_decl.location; + write!(f, "Class({}->{})", location.start_offset(), location.end_offset()) + } + Self::Module(module_decl) => { + let location = &module_decl.location; + write!(f, "Module({}->{})", location.start_offset(), location.end_offset()) + } + } + } +} diff --git a/src/internal.rs b/src/internal.rs deleted file mode 100644 index f367597de..000000000 --- a/src/internal.rs +++ /dev/null @@ -1,38 +0,0 @@ -use std::collections::HashMap; - -pub struct Repository { - pub entries: HashMap, -} - -impl Default for Repository { - fn default() -> Self { - Self::new() - } -} - -impl Repository { - pub fn new() -> Self { - Self { - entries: HashMap::new(), - } - } - pub fn add_entry(&mut self, entry: Entry) { - self.entries.insert(entry.name.clone(), entry); - } - - pub fn get_entry(&self, key: &str) -> Option<&Entry> { - self.entries.get(key) - } -} - -#[derive(Clone)] -pub struct Entry { - pub name: String, - pub value: String, -} - -impl Entry { - pub fn new(name: String, value: String) -> Self { - Self { name, value } - } -} diff --git a/src/lib.rs b/src/lib.rs index ee3e05029..52ddde169 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,2 +1,45 @@ +use std::{ + sync::{Arc, Mutex}, + thread, +}; + +use crate::{repository::Repository, ruby_indexer::RubyIndexer}; + pub mod c_interface; -pub mod internal; +mod declaration; +mod repository; +mod ruby_indexer; + +// Index all of the given file paths in parallel, populating the repository with the results +pub fn index_in_parallel(repository: &Arc>, file_paths: &Arc>>) { + let num_threads = thread::available_parallelism().map(|n| n.get()).unwrap_or(4); + let mut threads = Vec::with_capacity(num_threads); + + for _ in 0..num_threads { + let repository_clone = Arc::clone(repository); + let queue_clone = Arc::clone(file_paths); + + let handle = thread::spawn(move || { + let mut ruby_indexer = RubyIndexer::new(&repository_clone); + + loop { + let maybe_path = { + let mut queue = queue_clone.lock().unwrap(); + queue.pop() + }; + + if let Some(path) = maybe_path { + ruby_indexer.index(path); + } else { + break; + } + } + }); + + threads.push(handle); + } + + for handle in threads { + handle.join().unwrap(); + } +} diff --git a/src/repository.rs b/src/repository.rs new file mode 100644 index 000000000..b9ef7e4be --- /dev/null +++ b/src/repository.rs @@ -0,0 +1,40 @@ +use std::collections::HashMap; + +use crate::declaration::{ClassDeclaration, Declaration, Location, ModuleDeclaration}; + +#[derive(Debug)] +pub struct Repository { + entries: HashMap, +} + +impl Repository { + pub fn new() -> Self { + Self { + entries: HashMap::new(), + } + } + + pub fn get(&self, name: &str) -> Option<&Declaration> { + self.entries.get(name) + } + + pub fn add_class(&mut self, name: String, location: Location) { + let declaration = Declaration::Class(ClassDeclaration::new(location)); + self.entries.insert(name, declaration); + } + + pub fn add_module(&mut self, name: String, location: Location) { + let declaration = Declaration::Module(ModuleDeclaration::new(location)); + self.entries.insert(name, declaration); + } + + pub fn size(&self) -> usize { + self.entries.len() + } +} + +impl Default for Repository { + fn default() -> Self { + Self::new() + } +} diff --git a/src/ruby_indexer.rs b/src/ruby_indexer.rs new file mode 100644 index 000000000..45b9286c6 --- /dev/null +++ b/src/ruby_indexer.rs @@ -0,0 +1,62 @@ +use ruby_prism::Visit; +use std::sync::{Arc, Mutex}; + +use crate::{Repository, declaration::Location}; + +pub struct RubyIndexer<'a> { + nesting: Vec, + repository: &'a Arc>, +} + +impl<'a> RubyIndexer<'a> { + pub const fn new(repository: &'a Arc>) -> Self { + Self { + nesting: Vec::new(), + repository, + } + } + + pub fn index(&mut self, file_path: String) { + let source = std::fs::read_to_string(&file_path).unwrap(); + let result = ruby_prism::parse(source.as_ref()); + + self.visit(&result.node()); + self.nesting.clear(); + } +} + +impl Visit<'_> for RubyIndexer<'_> { + fn visit_class_node(&mut self, node: &ruby_prism::ClassNode) { + match std::str::from_utf8(node.constant_path().location().as_slice()).map(str::to_string) { + Ok(name) => { + let mut repo_guard = self.repository.lock().unwrap(); + let loc = Location::from_prism_location(node.location()); + repo_guard.add_class(name, loc); + } + Err(_) => { + // Name is not a valid UTF-8 string + } + } + + if let Some(body) = node.body() { + self.visit(&body); + } + } + + fn visit_module_node(&mut self, node: &ruby_prism::ModuleNode) { + match std::str::from_utf8(node.constant_path().location().as_slice()).map(str::to_string) { + Ok(name) => { + let mut repo_guard = self.repository.lock().unwrap(); + let loc = Location::from_prism_location(node.location()); + repo_guard.add_module(name, loc); + } + Err(_) => { + // Name is not a valid UTF-8 string + } + } + + if let Some(body) = node.body() { + self.visit(&body); + } + } +} diff --git a/test/index_test.rb b/test/index_test.rb index 827eaa371..e9d930d2f 100644 --- a/test/index_test.rb +++ b/test/index_test.rb @@ -7,10 +7,17 @@ def test_that_it_has_a_version_number refute_nil(::Index::VERSION) end - def test_add_entry + def test_index_all repository = Index::Repository.new - repository.add_entry("key", "value") - entry = repository.get_entry("key") - assert_equal(entry.value, "value") + repository.index_all([__FILE__]) + end + + def test_add_and_get + repository = Index::Repository.new + repository.add_class("MyClass", 0, 30) + entry = repository.get("MyClass") + + assert_equal(0, entry.start_offset) + assert_equal(30, entry.end_offset) end end