From 3b26f736e19cbd3474a12ac4b0a70e03bd962b56 Mon Sep 17 00:00:00 2001 From: Amy Rogin <32721672+rogin123@users.noreply.github.com> Date: Fri, 20 Mar 2026 10:09:57 -0400 Subject: [PATCH 1/7] update file linkers This is a large commit with my updates to file_linkes.R. Also includes updates to processer and standardizer helper functions for data cleaning. --- R/filing_linkers.R | 249 ++++++++++++++++++++++++++++++++++----------- R/processors.R | 132 ++++++++++++++++++++---- R/standardizers.R | 178 ++++++++++++++++++++++++-------- load_results.R | 32 +++++- 4 files changed, 466 insertions(+), 125 deletions(-) diff --git a/R/filing_linkers.R b/R/filing_linkers.R index 419ddf9..6dccb0f 100644 --- a/R/filing_linkers.R +++ b/R/filing_linkers.R @@ -1,125 +1,256 @@ -source("R/globals.R") source("R/standardizers.R") source("R/loaders.R") -source("R/run_utils.R") -match_nearby_filings <- function(parcel_points_df, miles = 0.1) { - #' Match filings to nearby parcels. +# OVERVIEW +# The goal of this script is to link eviction filings and their plaintiffs to property owners + +# STEP 1: Match eviction filing addresses to assessor records addresses +# STEP 1A: Match on address, city, zip, or address, city +# STEP 1B: For evictions that don't match directly on address, do a +# spatial nearest neighbor join (defined in match_nearby_filings() helper function) +# to parcel data and then try to match addresses. +# +# STEP 2: Match eviction plantiffs to owners +# STEP 2A: Direct string match of plantiff and owner name +# STEP 2B: Fuzzy match plantiff and owner name +# STEP 2C: For plantiffs unmatched in 2A-C, match by name within parcel. +# +# STEP 3: Combine into one file + +# HELPER FUNCTIONS -------------------------------------------------------- + +match_nearby_filings <- function(parcel_points_df, filings_df, miles = 0.1) { + #' Match filings to 4 nearest parcels. #' #' @param parcel_points_df Dataframe containing parcel centroids. #' @param miles Bandwidth distance, in miles. #' @returns A dataframe. #' @export assess_and_filings <- sf::st_join( - assess_points_df, - filings_with_geometry, - join = nngeo::st_nn, - maxdist = 5280 * mile_multiplier, - k = 2, - progress = FALSE) |> - dplyr::filter(!sf::st_is_empty(geometry)) |> + filings_df |> + sf::st_as_sf() |> + sf::st_transform(2249) |> + dplyr::filter(!sf::st_is_empty(geometry)), + parcel_points_df, + join = nngeo::st_nn, + k = 4, + maxdist = 5280 * miles, + progress = FALSE) |> + dplyr::filter(!sf::st_is_empty(geometry)) |> dplyr::filter(!is.na(street)) } process_filings <- function(df) { - #' Process eviction filings. + #' Clean and standardize eviction filings. #' #' @param df A dataframe of eviction filings. #' @returns A dataframe. #' @export df |> - std_flow_strings(c("add1", "city")) |> - std_zip(c("zip")) |> - std_flow_addresses(c("add1")) |> - std_flow_cities(c("city")) + proc_evic("street", postal_col = "zip", zips = zips, places= places, state_col = "state", muni_col = "city") |> + dplyr::distinct() } -process_link_filings <- function(town_ids = FALSE, crs = 2249) { +# CLEAN PLAINTIFF NAMES --------------------------------------------------- +proc_evic_plantiff <- function(df, type = "plantiff") { + + plaintiffs |> + dplyr::mutate(type = "plantiff") |> + proc_name_co_dba_attn( + "name", + "name", + retain = TRUE + ) |> + proc_name( + "name", + multiname = FALSE, + type="plantiff" + ) |> + std_uppercase("name") |> + std_remove_special("name") |> + dplyr::select(-id) + +} +# NEW WORKFLOW - +# 1. tie address to address in addresses table (or append if not found). +# 2. if address not found (or if existing address has no location), associate parcel loc_id with address. +# 3. join to owners on name/address. +# 4. join to owners on name (cosine similarity)/address. +# 5. for those filings unmatched in 3-4, match by name within parcel. + + +process_link_filings <- function(assess_df, evic_df = filings, parcels_points, town_ids = FALSE, crs = 2249) { #' Workflow to connect eviction filings to assessors records. #' + #' @param assess_df Assessors data (sites) + #' @param evic_df Eviction filings #' @param town_ids List of numerical town ids. #' @param crs Bandwidth distance, in miles. #' @returns A dataframe. #' @export - assess_file <- file.path( - RESULTS_DIR, - stringr::str_c(ASSESS_OUT_NAME, "rds", sep = ".") - ) - if (file.exists(assess_file)) { - log_message("Found previously processed assessors table. Loading...") - assess <- readRDS(assess_file) - } else { - log_message("Loading and processing assessors table...") - assess <- load_assess(path = file.path(DATA_DIR, ASSESS_GDB)) |> - process_assess() - } - - log_message("Joining filings to parcels by address and city.") - filings <- load_filings(town_ids, crs = crs) |> - process_filings() |> + + # join assessors data (sites) to addresses + assessor <- sites |> dplyr::left_join( - dplyr::select(assess, c(loc_id, site_addr, city)), - by = c("add1" = "site_addr", "city" = "city"), - na_matches = "never" - ) + addresses, + by = c("addr_id" = "id", + "muni_id" = "muni_id"), + na_matches = "never") - parcels <- load_parcels( - file.path(DATA_DIR, ASSESS_GDB), - town_ids = town_ids, - crs = crs - ) |> - dplyr::filter(loc_id %in% dplyr::pull(assess, loc_id)) + # PART 1A - MATCH EVICTION FILINGS TO ASSESSORS RECORDS ADDRESSES BASED ON A COMBINATION OF ADDRESS, CITY, ZIP + # Join eviction filings to assessors data by address, city, and zip code + filings_clean <- filings |> + # clean and standardize eviction filings addresses + process_filings() |> + tidylog::left_join( + dplyr::select(assessor, c(loc_id, addr, muni, postal)) |> dplyr::distinct(), + by = c("street" = "addr", "city" = "muni", "zip" = "postal"), + na_matches = "never") - filings_address_match <- filings |> + # df of eviction filings with direct address matches to assessors data (from city/zip join above) + filings_address_match <- filings_clean |> dplyr::filter(!is.na(loc_id)) |> dplyr::mutate( link_type = "address_city" ) - filings_no_address <- filings |> + # if didn't match on addr, city, and zip - match just on addr and zip + filings_no_address <- filings_clean |> dplyr::filter(is.na(loc_id)) |> dplyr::select(-c(loc_id)) |> - dplyr::left_join( - dplyr::select(assess, c(loc_id, site_addr, zip)), - by = c("add1" = "site_addr", "zip" = "zip"), + tidylog::left_join( + dplyr::select(assessor, c(loc_id, addr, postal)), + by = c("street" = "addr", "zip" = "postal"), na_matches = "never" ) + # filings that match on addr + zip filings_zip_match <- filings_no_address |> dplyr::filter(!is.na(loc_id)) |> dplyr::mutate( link_type = "address_zip" ) + # still unmatched filings filings_unmatched <- filings_no_address |> dplyr::filter(is.na(loc_id)) |> dplyr::select(-c(loc_id)) + # unmatchable filings because no parcel match potential - CHECK THIS filings_unmatchable <- filings_unmatched |> dplyr::filter( !(match_type %in% c("building", "parcel", "rooftop")) - ) |> + ) |> dplyr::mutate( link_type = NA_character_ ) - log_message("Find assess parcels that contain filings") + + # filter parcels to just points in assessor data + parcels <- parcels_point |> + dplyr::filter(loc_id %in% dplyr::pull(assessor, loc_id)) + + + # PART 1B - FOR EVICTIONS WITHOUT A DIRECT ADDRESS MATCH, TRY TO SPATIALLY JOIN TO PARCEL DATA + # AND THEN FUZZY MATCH ADDRESSES + + # Filter to eviction filings unmatched on address filings_no_match <- filings_unmatched |> dplyr::filter( match_type %in% c("building", "parcel", "rooftop") - ) + ) - filings <- filings_no_match |> - sf::st_join(parcels, join=sf::st_within) |> + # spatial join with parcel data + filings_spatial <- match_nearby_filings(parcel_points_df=parcels, filings_df =filings_no_match) |> # helper function defined above + # filter out unmatched rows + dplyr::filter(!is.na(loc_id)) |> + # create a flag for values spatially joined dplyr::mutate( link_type = dplyr::case_when( !is.na(loc_id) ~ "spatial" - ) - ) |> - dplyr::bind_rows(filings_address_match, filings_zip_match, filings_unmatchable) |> - sf::st_drop_geometry() |> - dplyr::select(docket_id, loc_id, city, zip, link_type) |> - write_multi(FILINGS_OUT_NAME) + ) + )|> + # join address data to be able to fuzzy match in next step + tidylog::left_join(addresses, by = c("loc_id"), suffix = c("_evic", "_assess")) + + + # Now we have a df of eviction filings and the 4 nearest parcel addresses + # We need to clean this dataframe to just keep relevant address matches + + # First, match eviction addresses and parcel addresses directly + filings_spatial_direct <- filings_spatial |> + # keep rows with perfect string match either in range or distinct + dplyr::filter(body_evic == body_assess & start_evic >= start_assess & start_evic <= end_assess | + start_evic == start_assess & end_evic == end_assess & body_evic == body_assess) |> + # CHECK - remove duplicates - in this case not keeping distinct loc_id because there are cases where the same address is in twice + # CHECK - Case like 17 Court where there is the single address (17 court) and also a range (17-19 court) + dplyr::distinct(docket_id, .keep_all = TRUE) |> + dplyr::mutate( + link_type = "spatial_direct") + + # list of unmatched filing ids + unmatched_filings <- filings_spatial_direct |> + dplyr::pull(docket_id) + + # now do fuzzy match if address range is the same + filings_spatial_fuzzy <- filings_spatial |> + # joing w/ original unmatched filings to see which didn't work w/ the full filter + dplyr::filter(!docket_id %in% unmatched_filings, + # make sure address numbers match + start_evic >= start_assess & end_evic<=end_assess) |> + # calculate stringdit using full Damerau-Levenshtein distance + dplyr::mutate( + dl_dist = stringdist::stringdist(body_evic, body_assess, method = "dl")) |> + # looking at the results, dl <4 are valid matches, more than that is not + dplyr::filter(dl_dist <4) |> + dplyr::distinct(loc_id, docket_id, .keep_all = TRUE) |> + dplyr::mutate( + link_type = "spatial_fuzzy") + + # COMBINE INTO ONE DF + filings_spatial_clean <- dplyr::bind_rows(filings_spatial_direct |> sf::st_as_sf() |> sf::st_transform(2249) , + filings_spatial_fuzzy |> sf::st_as_sf() |> sf::st_transform(2249), + filings_address_match |> sf::st_as_sf() |> sf::st_transform(2249), + filings_zip_match |> sf::st_as_sf() |> sf::st_transform(2249)) |> + # join plantiff name by docket_id - NOTE: there are ~21K filings that we have location data for + # so also have limited plantiff docket id match + # there are ~3K duplicates which represent when multiple plantiffs are attached to the same docket ID + dplyr::left_join(proc_evic_plantiff(), by = c("docket_id")) + + + # STEP 2 - MATCH PLANTIFFS TO OWNERS + + # MATCH TO OWNERS + filings_spatial_owners <- filings_spatial_clean |> + tidylog::left_join(owners, by = c("name")) + # NOTE 48,725 (w/ duplicates) MATCH - 12,450 dont. Some simply aren't there + # also found a case of METHUNION MANOR COOP CORPORATION vs. METHUNION MANOR COOPERATIVE CORPORATION + # TRYING FUZZY MATCH + + unmatched_plantiff <- filings_spatial_clean |> + # filter evistions already matched + dplyr::anti_join(owners, by = c("name")) |> + # Fuzzy match left join based on names + fuzzyjoin::stringdist_left_join(owners, by = c("name"), max_dist = 1, method = "dl") + + # match cleaned plantiff names to eviction filings by docket_id + filings_spatial_names <- filings_spatial_clean |> + tidylog::left_join(plantiff_clean, by = c("docket_id")) + + # filings_spatial <- filings_no_match |> + # sf::st_as_sf() |> + # sf::st_transform(2249) |> + # sf::st_join(parcels, join=sf::st_intersects) |> + # dplyr::mutate( + # link_type = dplyr::case_when( + # !is.na(loc_id) ~ "spatial" + # ) + # ) |> + # dplyr::bind_rows(filings_address_match, filings_zip_match, filings_unmatchable) + # + # sf::st_drop_geometry() |> + # dplyr::select(docket_id, loc_id, city, zip, link_type) |> + # write_multi(FILINGS_OUT_NAME) filings_by_parcel <- filings |> dplyr::group_by(loc_id) |> diff --git a/R/processors.R b/R/processors.R index 1d12763..3be012c 100644 --- a/R/processors.R +++ b/R/processors.R @@ -79,7 +79,7 @@ proc_address_to_range <- function(df, cols, prefixes=c()) { stringr::str_extract( get(paste0(dplyr::cur_column(), "_num_")), "[0-9\\.]+(?=[A-Z]?$)" - ) + ) ), .default = NA ) @@ -187,11 +187,11 @@ proc_address_postal <- function(df, col, state_col, muni_col, zips, state_constr } proc_address_muni <- function(df, - col, - state_col, - postal_col, - places, - state_val = "MA") { + col, + state_col, + postal_col, + places, + state_val = "MA") { df <- df |> std_directions(col) |> @@ -299,7 +299,7 @@ proc_name <- function(df, col, multiname = TRUE, type="") { dplyr::filter(!inst & !trust) |> # This also removes roman numerals. std_remove_titles(c(col)) - + if (multiname) { inds <- inds |> std_multiname(col) @@ -319,6 +319,7 @@ proc_name <- function(df, col, multiname = TRUE, type="") { } proc_name_co_dba_attn <- function(df, col, target, clear_cols = c(), retain = TRUE) { + # look at string - if there is the label (e.g. CO / ATTN - split text after into a new row ) df |> std_separate_and_label( col = col, @@ -495,7 +496,7 @@ proc_parcels_to_dry_points <- function(parcels, hydro, crs, quiet = FALSE) { hydro |> sf::st_set_agr("constant") |> sf::st_union() - ) |> + ) |> sf::st_set_agr("constant") |> sf::st_point_on_surface() @@ -510,6 +511,94 @@ proc_parcels_to_dry_points <- function(parcels, hydro, crs, quiet = FALSE) { ) } +# Eviction Filings ==== +proc_evic_address_postal <- function(df, col, state_col, muni_col, zips, state_constraint = "") { + if (state_constraint != "") { + df <- df |> + dplyr::filter(.data[[state_col]] == state_constraint) |> + std_zip_format( + col, + state_col=state_col, + zips=zips, + state_constraint=state_constraint + ) |> + dplyr::bind_rows( + df |> + dplyr::filter(.data[[state_col]] != state_constraint | is.na(.data[[state_col]])) |> + std_zip_format( + col, + state_col=state_col, + zips=zips + ) + ) + } else { + df <- df |> + std_zip_format( + col, + state_col=state_col, + zips=zips + ) + } +} + +# run full processing on evictions file +proc_evic <- function(df, col, postal_col, muni_col, state_col, zips, places, po_pmb = FALSE, state_constraint = "") { + + df |> + load_generic_preprocess(c(col, state_col, muni_col, postal_col)) |> + proc_address_text(col) |> + proc_address_addr2( + col, + po_pmb=po_pmb + ) |> + proc_address_to_range(col) |> + proc_evic_address_postal( + postal_col, + state_col=state_col, + muni_col=muni_col, + zips, + state_constraint + ) |> + proc_address_muni( + muni_col, + state_col=state_col, + postal_col=postal_col, + places=places + ) +} + +# process plantiff names + + +proc_evic_plantiff <- function(df = plaintiffs, name_col = "name", address_col= "name", type = "owners", quiet=FALSE) { + # df <- df |> + # dplyr::mutate( + # type = "owners" + # ) |> + # std_uppercase("name") |> + # std_remove_special("name") |> + # proc_name("name") |> + # proc_name_co_dba_attn( + # "name", + # "name" + # ) + + plaintiffs |> + dplyr::mutate(type = "plantiff") |> + std_uppercase("name") |> + std_remove_special("name") |> + proc_name_co_dba_attn( + "name", + "name", + retain = TRUE + ) |> + proc_name( + "name", + multiname = FALSE, + type="plantiff" + ) +} + # Assessor's Database ==== @@ -538,7 +627,7 @@ proc_assess_split <- function(df, site_prefix, own_prefix, quiet = FALSE) { dplyr::rename( match_site_id = own_site_id, match_muni_id = own_muni_id - ) |> + ) |> dplyr::select( c(site_id, site_muni_id, match_site_id, match_muni_id, dplyr::starts_with(own_prefix)) ) |> @@ -558,7 +647,7 @@ proc_assess_split <- function(df, site_prefix, own_prefix, quiet = FALSE) { by = dplyr::join_by( match_site_id == id, match_muni_id == muni_id - ), + ), multiple="any", na_matches="never" ) |> @@ -567,7 +656,7 @@ proc_assess_split <- function(df, site_prefix, own_prefix, quiet = FALSE) { dplyr::filter(!(!is.na(match_site_id) & !is.na(match_muni_id))) ) |> dplyr::select(-c(match_site_id, match_muni_id)) - + list( sites = sites, owners = owners @@ -619,7 +708,7 @@ proc_assess_sites_units <- function(df, luc_col, addresses) { muni_id_col="muni_id", count_col="addr_count", addresses=addresses - ) |> + ) |> dplyr::bind_rows( df |> dplyr::filter(units_valid) @@ -636,7 +725,7 @@ proc_assess_sites <- function(df, addresses, quiet=FALSE) { luc_col="luc", id_cols=c("loc_id", "body"), units_col="units" - ) |> + ) |> proc_assess_sites_units( "luc", addresses @@ -805,7 +894,7 @@ proc_assess_address_addr2 <- function(df, site_prefix, own_prefix, quiet = FALSE "site_addr", po_pmb = FALSE, prefixes=c(site_prefix) - ) + ) df |> dplyr::filter(is.na(own_site_id) | is.na(own_muni_id)) |> @@ -813,7 +902,7 @@ proc_assess_address_addr2 <- function(df, site_prefix, own_prefix, quiet = FALSE c("site_addr", "own_addr"), prefixes=c(site_prefix, own_prefix), po_pmb = TRUE - ) |> + ) |> proc_address_match_simp(c("site_addr", "own_addr")) |> dplyr::mutate( own_site_id = dplyr::case_when( @@ -877,7 +966,7 @@ proc_assess_address_postal <- function(df, site_prefix, own_prefix, zips, parcel muni_col="site_muni", zips=zips, state_constraint=state_constraint - ) + ) df <- df |> dplyr::mutate( @@ -891,7 +980,7 @@ proc_assess_address_postal <- function(df, site_prefix, own_prefix, zips, parcel state_col="own_state", muni_col="own_muni", zips=zips - ) |> + ) |> dplyr::mutate( site_postal = dplyr::case_when( is.na(site_postal) & @@ -904,7 +993,7 @@ proc_assess_address_postal <- function(df, site_prefix, own_prefix, zips, parcel dplyr::group_by(site_loc_id) |> tidyr::fill(site_postal) |> dplyr::ungroup() - + df |> dplyr::filter(is.na(site_postal)) |> std_fill_ma_zip_sp( @@ -969,6 +1058,7 @@ proc_assess <- function(df, places <- places |> dplyr::select(-id) + # THIS IS THE FLOW TO COPY df <- df |> proc_assess_address_text( site_prefix = site_prefix, @@ -1027,7 +1117,7 @@ proc_all <- function(assess, push_db = "", refresh = FALSE, quiet = FALSE - ) { +) { if (is.null(tables)) { if (!quiet) { @@ -1055,7 +1145,7 @@ proc_all <- function(assess, loader=proc_parcels_to_dry_points( parcels, crs=crs - ), + ), id_col="loc_id", refresh=refresh ) @@ -1113,7 +1203,7 @@ proc_all <- function(assess, ), id_col=c("id", "muni_id"), refresh=refresh - ) + ) # load_add_fk(util_conn(push_db), "proc_sites", "munis", "muni_id", "muni_id") # load_add_fk(util_conn(push_db), "proc_sites", "parcels_point", "loc_id", "loc_id") diff --git a/R/standardizers.R b/R/standardizers.R index 5885c94..2c4a769 100644 --- a/R/standardizers.R +++ b/R/standardizers.R @@ -263,7 +263,7 @@ std_remove_special <- function(df, cols, rm_commas=TRUE) { "(?<=[0-9]) *\\. *(?!5)" = "\\-", "(? std_replace_generic( cols, @@ -374,7 +374,7 @@ std_replace_blank <- function(df, cols) { stringr::str_detect(., "^(SAME|NONE|UNKNOWN)(?=\\s|\\,|$)") ~ NA_character_, .default = . ) - ) + ) ) } @@ -386,7 +386,7 @@ std_replace_newline <- function(df, cols) { #' @export newline <- c( ",?\\\\n" - ) |> + ) |> std_collapse_regex(full_string = FALSE) std_replace_generic( df, @@ -408,7 +408,7 @@ std_fix_concatenated_ranges <- function(df, cols) { pattern = c( "^([0-9]{3,4})[A-Z]?(?=(\\1){1})" = "", "(?<=^([0-9]{2,3})[0-9][A-Z]?)(?=(\\1){1}[0-9])" = "-" - ) + ) ) } @@ -636,17 +636,17 @@ std_remove_counties <- function(df, col, state_col, places, state_val="MA") { std_collapse_regex() |> suppressMessages() - df |> - dplyr::filter(.data[[state_col]] == state_val) |> - std_replace_generic( - col, - stringr::str_c("(?<=[A-Z] )(", counties, ")$", collapse=""), - "" - ) |> - dplyr::bind_rows( - df |> - dplyr::filter(.data[[state_col]] != state_val | is.na(.data[[state_col]])) - ) + df |> + dplyr::filter(.data[[state_col]] == state_val) |> + std_replace_generic( + col, + stringr::str_c("(?<=[A-Z] )(", counties, ")$", collapse=""), + "" + ) |> + dplyr::bind_rows( + df |> + dplyr::filter(.data[[state_col]] != state_val | is.na(.data[[state_col]])) + ) } std_match_state_and_zips <- function(df, state_col, zip_col, zips, state_val = "MA") { @@ -1000,7 +1000,7 @@ std_test_units <- function(df, col, luc_col, muni_id_col) { FALSE, # BOSTON, 112: 7-30 Unit .data[[luc_col]] == '112' & !dplyr::between(.data[[col]], 7, 30) ~ - FALSE,s + FALSE, # BOSTON, 113: 31-99 Unit .data[[luc_col]] == '113' & !dplyr::between(.data[[col]], 31, 99) ~ FALSE, @@ -1063,7 +1063,7 @@ std_estimate_units <- function(df, col, luc_col, muni_id_col, count_col, address ) |> dplyr::mutate( units_by_area = ceiling(res_area / est_size) - ) + ) boston <- df |> dplyr::filter(.data[[muni_id_col]] == '035') |> @@ -1302,8 +1302,8 @@ std_addr2_parser <- function(df, cols, regex, from_end = TRUE) { !is.na(.x) & !is.na(get(stringr::str_replace(dplyr::cur_column(), "_addr2", "_temp"))) ~ stringr::str_c(get(stringr::str_replace(dplyr::cur_column(), "_addr2", "_temp")), - .x, - sep=" " + .x, + sep=" " ), is.na(.x) & !is.na(get(stringr::str_replace(dplyr::cur_column(), "_addr2", "_temp"))) @@ -1442,8 +1442,8 @@ std_hyphenate_range <- function (df, cols) { ., "(?<=^[0-9]{1,6}[A-Z]{0,2}) (?=[0-9]{1,6}[A-Z]{0,1} )", "-" - ) - ), + ) + ), dplyr::across( dplyr::all_of(cols), @@ -1535,7 +1535,7 @@ std_fill_zip_by_muni <- function(df, col, muni_col, zips) { df <- df |> dplyr::filter( is.na(.data[[col]]) & !is.na(.data[[muni_col]]) - ) |> + ) |> dplyr::left_join( zips |> sf::st_drop_geometry() |> @@ -1544,7 +1544,7 @@ std_fill_zip_by_muni <- function(df, col, muni_col, zips) { dplyr::distinct(), by = dplyr::join_by( !!muni_col == muni_unambig_to - ) + ) ) |> dplyr::mutate( !!col := dplyr::case_when( @@ -1625,7 +1625,7 @@ std_munis_by_places <- function(df, state |> dplyr::filter(match | is.na(.data[[col]])) ) - + state <- state |> dplyr::filter(!match & !is.na(.data[[col]])) |> fuzzyjoin::stringdist_left_join( @@ -1651,7 +1651,7 @@ std_munis_by_places <- function(df, state |> dplyr::filter(match | is.na(.data[[col]])) ) - + state |> dplyr::group_by(temp_id) |> dplyr::mutate( @@ -1759,8 +1759,8 @@ std_multiname <- function(df, col) { stringr::str_count(.data[[col]], "([A-Z]{2,}\\s)") > 0) |> dplyr::select(-last) |> dplyr::bind_rows( - df |> - dplyr::filter(!name_and | trust | inst) + df |> + dplyr::filter(!name_and | trust | inst) ) |> dplyr::select(-c(name_and, temp_id)) } @@ -1824,7 +1824,99 @@ std_separate_and_label <- function(df, .default = type ) ) + + if(target_col != col) { + match <- match |> + dplyr::mutate( + !!target_col := dplyr::case_when( + type == label ~ .data[[col]], + .default = .data[[target_col]] + ), + !!col := dplyr::case_when( + type == label ~ NA_character_, + .default = .data[[col]] + ) + ) + } else { + match <- match |> + dplyr::group_by(t_id) |> + dplyr::arrange(.data[[col]]) |> + dplyr::mutate( + rm_test = sum(!is.na(.data[[col]])) == 1, + type = dplyr::first(type) + ) |> + dplyr::ungroup() |> + dplyr::filter(!(rm_test & is.na(.data[[col]]))) |> + dplyr::select(-rm_test) + } + + match <- match |> + dplyr::mutate( + dplyr::across( + dplyr::any_of(clear_cols) & tidyselect::where(is.character), + ~ dplyr::case_when( + type != label ~ + NA_character_, + .default = .x + ) + ) + ) + + if (!retain) { + match <- match |> + dplyr::filter(type != label) + } + + match |> + dplyr::bind_rows( + df |> + dplyr::filter(!flag) + ) |> + dplyr::select(-c(row, count, flag, t_id)) + +} +# Std Plantiff +std_separate_and_label_plant <- function(df, + col, + regex, + label = "", + target_col = "name", + clear_cols = c(), + retain = TRUE) { + regex = stringr::regex(regex) + df <- df |> + tibble::rowid_to_column("t_id") |> + dplyr::mutate( + flag = dplyr::case_when( + stringr::str_detect(.data[[col]], regex) ~ TRUE, + .default = FALSE + ) + ) + + match <- df |> + dplyr::filter(flag) |> + tidyr::separate_longer_delim( + tidyselect::all_of(col), + regex + ) |> + dplyr::mutate( + !!col := dplyr::na_if(.data[[col]], "") + ) |> + dplyr::group_by(dplyr::across(-dplyr::all_of(col))) |> + dplyr::mutate( + row = dplyr::row_number(), + count = dplyr::n() + ) |> + dplyr::ungroup() |> + dplyr::mutate( + type = dplyr::case_when( + nchar(label) == 0 ~ type, + row > 1 | row == count ~ label, + .default = type + ) + ) + if(target_col != col) { match <- match |> dplyr::mutate( @@ -1873,7 +1965,7 @@ std_separate_and_label <- function(df, dplyr::filter(!flag) ) |> dplyr::select(-c(row, count, flag, t_id)) - + } @@ -1960,12 +2052,12 @@ std_flag_inst <- function(df, col) { dplyr::mutate( inst := dplyr::case_when( stringr::str_detect( - .data[[col]], - stringr::str_c( - "\\b(", - stringr::str_c(SEARCH$inst, collapse = "|"), - ")\\b", - sep = "") + .data[[col]], + stringr::str_c( + "\\b(", + stringr::str_c(SEARCH$inst, collapse = "|"), + ")\\b", + sep = "") ) ~ TRUE, .default = FALSE ) @@ -1985,13 +2077,13 @@ std_flag_trust <- function(df, col) { trust = dplyr::case_when( stringr::str_detect(.data[[col]], "TRUST(?!EES)") ~ TRUE, stringr::str_detect(.data[[col]], "^TRUSTEES OF ") - & !stringr::str_detect(.data[[col]], "UNIVERSITY|COLLEGE|INSTITUTE") ~ TRUE, + & !stringr::str_detect(.data[[col]], "UNIVERSITY|COLLEGE|INSTITUTE") ~ TRUE, stringr::str_detect(.data[[col]], - stringr::str_c( - "\\b(", - stringr::str_c(SEARCH$trust_definite, collapse = "|"), - ")\\b", - sep = "") + stringr::str_c( + "\\b(", + stringr::str_c(SEARCH$trust_definite, collapse = "|"), + ")\\b", + sep = "") ) ~ TRUE, .default = FALSE ), @@ -2115,14 +2207,14 @@ std_fill_ma_zip_sp <- function(df, col, site_loc_id, site_muni_id, parcels_point by=dplyr::join_by( !!site_loc_id == loc_id, !!site_muni_id == muni_id - ), + ), na_matches="never", multiple="any" - ) |> + ) |> sf::st_as_sf() |> sf::st_join( dplyr::select(ma_zips, zip) - ) |> + ) |> dplyr::mutate( !!col := zip ) |> diff --git a/load_results.R b/load_results.R index 0291579..2a9feb5 100644 --- a/load_results.R +++ b/load_results.R @@ -8,14 +8,14 @@ load_results <- function(prefix, load_boundaries=TRUE, summarize=TRUE) { util_test_conn(prefix) conn <- util_conn(prefix) on.exit(DBI::dbDisconnect(conn)) - + tables <- c( "owners", "companies", "officers", "metacorps_cosine", "parcels_point", "metacorps_network", "sites", "sites_to_owners", "addresses") if (load_boundaries) { tables <- c( tables, - c("zips", "munis", "tracts", "block_groups") + c("zips", "munis", "tracts","block_groups") ) } tables_exist <- util_check_for_tables( @@ -51,3 +51,31 @@ load_results <- function(prefix, load_boundaries=TRUE, summarize=TRUE) { summ_officers_innetwork_companies(metacorps_network) } } + +# load eviction filing results +load_evic_results <- function(prefix) { + + util_test_conn(prefix) + conn <- util_conn(prefix) + on.exit(DBI::dbDisconnect(conn)) + + tables <- c( + "filings", "plaintiffs" + ) + + tables_exist <- util_check_for_tables( + conn, + tables + ) + if (!tables_exist) { + stop(glue::glue("VALIDATION: Tables don't seem to exist on '{prefix}' database.")) + } else { + util_log_message(glue::glue("VALIDATION: Loading tables from '{prefix}' database."), header=TRUE) + for(t in tables) { + util_log_message(glue::glue("INPUT/OUTPUT: Loading {t} from '{prefix}' database.")) + assign(t, load_postgis_read(conn, t), envir = .GlobalEnv) + } + } + invisible(NULL) +} + From 44d45944f0cc6052345c8dd2d8a78acfd6c4421b Mon Sep 17 00:00:00 2001 From: Amy Rogin <32721672+rogin123@users.noreply.github.com> Date: Thu, 2 Apr 2026 17:18:25 -0400 Subject: [PATCH 2/7] Update filing_linkers.R --- R/filing_linkers.R | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/R/filing_linkers.R b/R/filing_linkers.R index 6dccb0f..af9e724 100644 --- a/R/filing_linkers.R +++ b/R/filing_linkers.R @@ -227,8 +227,10 @@ process_link_filings <- function(assess_df, evic_df = filings, parcels_points, t # also found a case of METHUNION MANOR COOP CORPORATION vs. METHUNION MANOR COOPERATIVE CORPORATION # TRYING FUZZY MATCH + + # THIS IS STILL A WORK IN PROGRESS - TIMES OUT BEFORE IT FINISHES RUNNING unmatched_plantiff <- filings_spatial_clean |> - # filter evistions already matched + # filter evictions already matched dplyr::anti_join(owners, by = c("name")) |> # Fuzzy match left join based on names fuzzyjoin::stringdist_left_join(owners, by = c("name"), max_dist = 1, method = "dl") From 3109ba36098db4a859928ff00a8633220afe17dc Mon Sep 17 00:00:00 2001 From: Amy Rogin <32721672+rogin123@users.noreply.github.com> Date: Thu, 2 Apr 2026 17:53:03 -0400 Subject: [PATCH 3/7] Update filing_linkers.R --- R/filing_linkers.R | 1 + 1 file changed, 1 insertion(+) diff --git a/R/filing_linkers.R b/R/filing_linkers.R index af9e724..23fc691 100644 --- a/R/filing_linkers.R +++ b/R/filing_linkers.R @@ -14,6 +14,7 @@ source("R/loaders.R") # STEP 2A: Direct string match of plantiff and owner name # STEP 2B: Fuzzy match plantiff and owner name # STEP 2C: For plantiffs unmatched in 2A-C, match by name within parcel. +# STEP 2D: Filter by date range to make sure that eviction falls within when the owner owned the property # # STEP 3: Combine into one file From e55929bb98f1026581a66bd02abab3a17b450379 Mon Sep 17 00:00:00 2001 From: Amy Rogin <32721672+rogin123@users.noreply.github.com> Date: Fri, 10 Apr 2026 10:07:12 -0400 Subject: [PATCH 4/7] update owner match --- R/filing_linkers.R | 93 +++++++++++++++--------- test_file_linkers.R | 170 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 230 insertions(+), 33 deletions(-) create mode 100644 test_file_linkers.R diff --git a/R/filing_linkers.R b/R/filing_linkers.R index 23fc691..6bb1dc8 100644 --- a/R/filing_linkers.R +++ b/R/filing_linkers.R @@ -9,14 +9,16 @@ source("R/loaders.R") # STEP 1B: For evictions that don't match directly on address, do a # spatial nearest neighbor join (defined in match_nearby_filings() helper function) # to parcel data and then try to match addresses. -# # STEP 2: Match eviction plantiffs to owners -# STEP 2A: Direct string match of plantiff and owner name -# STEP 2B: Fuzzy match plantiff and owner name -# STEP 2C: For plantiffs unmatched in 2A-C, match by name within parcel. -# STEP 2D: Filter by date range to make sure that eviction falls within when the owner owned the property -# -# STEP 3: Combine into one file + + +# NEW WORKFLOW - +# 1. tie address to address in addresses table (or append if not found). +# 2. if address not found (or if existing address has no location), associate parcel loc_id with address. +# 3. join to owners on name/address. +# 4. join to owners on name (cosine similarity)/address. +# 5. for those filings unmatched in 3-4, match by name within parcel. + # HELPER FUNCTIONS -------------------------------------------------------- @@ -72,14 +74,8 @@ proc_evic_plantiff <- function(df, type = "plantiff") { dplyr::select(-id) } -# NEW WORKFLOW - -# 1. tie address to address in addresses table (or append if not found). -# 2. if address not found (or if existing address has no location), associate parcel loc_id with address. -# 3. join to owners on name/address. -# 4. join to owners on name (cosine similarity)/address. -# 5. for those filings unmatched in 3-4, match by name within parcel. - +# Function to run everything process_link_filings <- function(assess_df, evic_df = filings, parcels_points, town_ids = FALSE, crs = 2249) { #' Workflow to connect eviction filings to assessors records. #' @@ -106,7 +102,7 @@ process_link_filings <- function(assess_df, evic_df = filings, parcels_points, t tidylog::left_join( dplyr::select(assessor, c(loc_id, addr, muni, postal)) |> dplyr::distinct(), by = c("street" = "addr", "city" = "muni", "zip" = "postal"), - na_matches = "never") + na_matches = "never") # 21,195 matched rows # df of eviction filings with direct address matches to assessors data (from city/zip join above) filings_address_match <- filings_clean |> @@ -123,7 +119,7 @@ process_link_filings <- function(assess_df, evic_df = filings, parcels_points, t dplyr::select(assessor, c(loc_id, addr, postal)), by = c("street" = "addr", "zip" = "postal"), na_matches = "never" - ) + ) # 1,097 matched rows # filings that match on addr + zip filings_zip_match <- filings_no_address |> @@ -137,14 +133,14 @@ process_link_filings <- function(assess_df, evic_df = filings, parcels_points, t dplyr::filter(is.na(loc_id)) |> dplyr::select(-c(loc_id)) - # unmatchable filings because no parcel match potential - CHECK THIS + # unmatchable filings because no parcel match potential - e.g., match_type %in% c(approximate, point, road, street) filings_unmatchable <- filings_unmatched |> dplyr::filter( !(match_type %in% c("building", "parcel", "rooftop")) ) |> dplyr::mutate( link_type = NA_character_ - ) + ) # 3,527 unmatchable # filter parcels to just points in assessor data @@ -155,7 +151,7 @@ process_link_filings <- function(assess_df, evic_df = filings, parcels_points, t # PART 1B - FOR EVICTIONS WITHOUT A DIRECT ADDRESS MATCH, TRY TO SPATIALLY JOIN TO PARCEL DATA # AND THEN FUZZY MATCH ADDRESSES - # Filter to eviction filings unmatched on address + # Filter to eviction filings unmatched on address - 27,606 records filings_no_match <- filings_unmatched |> dplyr::filter( match_type %in% c("building", "parcel", "rooftop") @@ -178,7 +174,7 @@ process_link_filings <- function(assess_df, evic_df = filings, parcels_points, t # Now we have a df of eviction filings and the 4 nearest parcel addresses # We need to clean this dataframe to just keep relevant address matches - # First, match eviction addresses and parcel addresses directly + # First, match eviction addresses and parcel addresses directly - 9,682 records filings_spatial_direct <- filings_spatial |> # keep rows with perfect string match either in range or distinct dplyr::filter(body_evic == body_assess & start_evic >= start_assess & start_evic <= end_assess | @@ -189,11 +185,12 @@ process_link_filings <- function(assess_df, evic_df = filings, parcels_points, t dplyr::mutate( link_type = "spatial_direct") + # list of unmatched filing ids unmatched_filings <- filings_spatial_direct |> dplyr::pull(docket_id) - # now do fuzzy match if address range is the same + # now do fuzzy match if address range is the same - 770 obs filings_spatial_fuzzy <- filings_spatial |> # joing w/ original unmatched filings to see which didn't work w/ the full filter dplyr::filter(!docket_id %in% unmatched_filings, @@ -208,7 +205,7 @@ process_link_filings <- function(assess_df, evic_df = filings, parcels_points, t dplyr::mutate( link_type = "spatial_fuzzy") - # COMBINE INTO ONE DF + # COMBINE INTO ONE DF - 35,756 records - all values from filings match, but not all plaintiffs are in eviction records filings_spatial_clean <- dplyr::bind_rows(filings_spatial_direct |> sf::st_as_sf() |> sf::st_transform(2249) , filings_spatial_fuzzy |> sf::st_as_sf() |> sf::st_transform(2249), filings_address_match |> sf::st_as_sf() |> sf::st_transform(2249), @@ -216,27 +213,57 @@ process_link_filings <- function(assess_df, evic_df = filings, parcels_points, t # join plantiff name by docket_id - NOTE: there are ~21K filings that we have location data for # so also have limited plantiff docket id match # there are ~3K duplicates which represent when multiple plantiffs are attached to the same docket ID - dplyr::left_join(proc_evic_plantiff(), by = c("docket_id")) + tidylog::left_join(proc_evic_plantiff(), by = c("docket_id")) # STEP 2 - MATCH PLANTIFFS TO OWNERS - # MATCH TO OWNERS + # join assessors records to owners + con <- duckdb::dbConnect(duckdb::duckdb()) + + # Register your R dataframes as virtual tables in DuckDB + duckdb::duckdb_register(con, "owners", owners) + duckdb::duckdb_register(con, "assessor", assessor) + + # Join them using dplyr syntax (executed via DuckDB) + owners_assess <- dplyr::tbl(con, "owners") |> + dplyr::distinct(name, addr_id, cosine_group, network_group) |> + tidylog::left_join(dplyr::tbl(con, "assessor")|> dplyr::select(c(loc_id, addr, addr_id, muni, postal)) |> dplyr::distinct(), by = "addr_id") |> + dplyr::collect() |># Pulls the final result back into R + dplyr::distinct(name, addr_id, cosine_group, network_group,.keep_all= TRUE) # get one record per name and address + + # STEP 2A - DIRECT STRING MATCH TO OWNERS - 3,136 plaintiffs match owners + loc_id, 33,341 don't filings_spatial_owners <- filings_spatial_clean |> - tidylog::left_join(owners, by = c("name")) - # NOTE 48,725 (w/ duplicates) MATCH - 12,450 dont. Some simply aren't there - # also found a case of METHUNION MANOR COOP CORPORATION vs. METHUNION MANOR COOPERATIVE CORPORATION + tidylog::left_join(owners_assess, by = c("name", "loc_id")) # TRYING FUZZY MATCH + # data frame of matched + filings_direct_own_match <- filings_spatial_owners |> + dplyr::filter(!is.na(addr_id)) |> + dplyr::mutate(owner_link = "name_loc_id") - # THIS IS STILL A WORK IN PROGRESS - TIMES OUT BEFORE IT FINISHES RUNNING - unmatched_plantiff <- filings_spatial_clean |> + # STEP 2b: FUZZY MATCH USING COSINE SIMILARITY + # Filter out plaintiffs that matched - 1,718 new matches + fuzzy_match_plantiff <- filings_spatial_clean |> # filter evictions already matched - dplyr::anti_join(owners, by = c("name")) |> - # Fuzzy match left join based on names - fuzzyjoin::stringdist_left_join(owners, by = c("name"), max_dist = 1, method = "dl") + dplyr::anti_join(owners_assess, by = c("name")) |> + tidylog::left_join(owners_assess, by = c("loc_id")) |> + # calculate stringdit using cosine similarity and character level overlap + dplyr::mutate( + # 1. Cosine Similarity (q=2 captures "chunks" of names) + # This prevents "Jon" and "Ian" from looking too similar + cosine_sim = stringdist::stringsim(name.x, name.y, method = "cosine", q = 2), + + # 2. Character-level overlap (how many characters they share) + # Using q=1 makes it a "bag of characters" comparison + char_overlap = stringdist::stringsim(name.x, name.y, method = "cosine", q = 1) + ) |> + dplyr::filter(cosine_sim > 0.5) + + # STEP 2C - FOR REMAINING UNMATCHED - MATCH BY NAME WITHIN PARCEL + filin - # match cleaned plantiff names to eviction filings by docket_id + # match cleaned plantiff names toMatc eviction filings by docket_id filings_spatial_names <- filings_spatial_clean |> tidylog::left_join(plantiff_clean, by = c("docket_id")) diff --git a/test_file_linkers.R b/test_file_linkers.R new file mode 100644 index 0000000..6938ad7 --- /dev/null +++ b/test_file_linkers.R @@ -0,0 +1,170 @@ +source('load_results.R') +source('R/filing_linkers.R') +source('R/processors.R') + +# load owner/parcel data +load_results("", load_boundaries=TRUE, summarize=TRUE) + +# plaintiffs - landlord bringing filing +# docket id if unique identifier from filings to plantiff +load_evic_results("EVICTION") + +# load places data +places <- load_places(munis = munis, zips, crs = 2249) + +# TEST FILING LINKING FUNCTIONS ------------------------------------------ + +# Get subset of each file - Just Somerville +filings_bos <- filings |> + dplyr::filter(city == "Boston") + +parcels_point_bos <- parcels_point |> + dplyr::filter(muni_id == "035") + +log_message("Linking assessors data to addresses") +assessor <- sites |> + tidylog::left_join( + addresses, + by = c("addr_id" = "id", + "muni_id" = "muni_id"), + na_matches = "never") + +log_message("Joining filings to parcels by address and city.") +# MATCHING BY BOTH CITY AND ZIP - DOUBLE CHECK THIS - DEFINITELY CAUSING TOO INCLUSIZE MATCH IF JUST CITY BUT MAYBE CAN HANDLE THAT OTHER WAY? +filings_clean <- filings_bos |> + process_filings() |> + tidylog::left_join( + dplyr::select(assessor, c(loc_id, addr, muni, postal)) |> dplyr::distinct(), + by = c("street" = "addr", "city" = "muni", "zip" = "postal"), + na_matches = "never" + ) + +# filter parcels to just points in assessor data +parcels <- parcels_point_bos |> + dplyr::filter(loc_id %in% dplyr::pull(assessor, loc_id)) + +# df of filings with direct address matches to assessors data +filings_address_match <- filings_clean |> + dplyr::filter(!is.na(loc_id)) |> + dplyr::mutate( + link_type = "address_city" + ) + +# if didn't match on addr, city, and zip - match just on addr and zip +filings_no_address <- filings_clean |> + dplyr::filter(is.na(loc_id)) |> + dplyr::select(-c(loc_id)) |> + tidylog::left_join( + dplyr::select(assessor, c(loc_id, addr, postal)), + by = c("street" = "addr", "zip" = "postal"), + na_matches = "never" + ) + +# filings that match on addr + zip +filings_zip_match <- filings_no_address |> + dplyr::filter(!is.na(loc_id)) |> + dplyr::mutate( + link_type = "address_zip" + ) + +# still unmatched filings +filings_unmatched <- filings_no_address |> + dplyr::filter(is.na(loc_id)) |> + dplyr::select(-c(loc_id)) + +# unmatchable filings because no parcel match potential - CHECK THIS +filings_unmatchable <- filings_unmatched |> + dplyr::filter( + !(match_type %in% c("building", "parcel", "rooftop")) + ) |> + dplyr::mutate( + link_type = NA_character_ + ) + +# For addresses not found - associate parcel loc_id to address +# Join parcel data to filings with buffer +log_message("Find assess parcels that contain filings") +filings_no_match <- filings_unmatched |> + dplyr::filter( + match_type %in% c("building", "parcel", "rooftop") + ) + + +# spatial join with parcel data +filings_spatial <- match_nearby_filings(parcel_points_df=parcels, filings_df =filings_no_match) |> + # filter unmatched rows + dplyr::filter(!is.na(loc_id)) |> + dplyr::mutate( + link_type = dplyr::case_when( + !is.na(loc_id) ~ "spatial" + ) + )|> + # join address data to be able to fuzzy match in next step + tidylog::left_join(addresses, by = c("loc_id"), suffix = c("_evic", "_assess")) + +# NEXT STEP - STRING FUZZY MATCH - OF THESE MATCHES DO ANY HAVE A STRING DISTANCE OF >.9 +filings_spatial_clean <- filings_spatial |> + # keep rows with perfect string match + dplyr::filter(start_evic == start_assess & end_evic == end_assess & body_evic == body_assess) |> + # CHECK - remove duplicates - in this case not keeping distinct loc_id because there are cases where the same address is in twice + dplyr::distinct(start_assess, end_assess, body_assess, docket_id, .keep_all = TRUE) |> + dplyr::mutate( + link_type = "spatial_direct") + +# list of unmatched filing ids +test_case <- filings_spatial_clean |> + dplyr::pull(docket_id) + +# Test edge cases +filings_spatial_edge <- filings_spatial |> + # joing w/ original unmatched filings to see which didn't work w/ the full filter + dplyr::filter(!docket_id %in% test_case) |> + # keep rows with perfect address name sting match and that fall in address range + dplyr::filter(body_evic == body_assess & start_evic >= start_assess & start_evic <= end_assess) |> # CHECK - remove duplicates - in this case not keeping distinct loc_id because there are cases where the same address is in twice + dplyr::distinct(start_assess, end_assess, body_assess, docket_id, .keep_all = TRUE) |> + dplyr::mutate( + link_type = "spatial_edge") + +# list of unmatched filing ids +test_case_2 <- filings_spatial_edge|> + dplyr::pull(docket_id) + +# now do fuzzy match if address range is the same +filings_spatial_fuzzy <- filings_spatial |> + # joing w/ original unmatched filings to see which didn't work w/ the full filter + dplyr::filter(!docket_id %in% test_case & !docket_id %in% test_case_2, + # make sure address numbers match + start_evic >= start_assess & end_evic<=end_assess) |> + # calculate stringdit using full Damerau-Levenshtein distance + dplyr::mutate( + dl_dist = stringdist::stringdist(body_evic, body_assess, method = "dl")) |> + # looking at the results, dl <= 5 are valid matches, more than that is not + dplyr::filter(dl_dist <6) |> + dplyr::distinct(loc_id, docket_id, .keep_all = TRUE) |> + dplyr::mutate( + link_type = "spatial_fuzzy") + + + +# CLEAN PLAINTIFF NAMES --------------------------------------------------- +proc_evic_plantiff <- function(df, type = "plantiff") { + + plaintiffs |> + std_uppercase("name") |> + std_remove_special("name") |> + dplyr::mutate(type = "plantiff") |> + proc_name_co_dba_attn( + "name", + "name", + retain = TRUE + ) |> + proc_name( + "name", + multiname = FALSE, + type="plantiff" + ) |> + dplyr::select(-id) + +} + + From 5bf11a3325f23a92c2669ebc100f708a8b831d68 Mon Sep 17 00:00:00 2001 From: Amy Rogin <32721672+rogin123@users.noreply.github.com> Date: Tue, 5 May 2026 12:09:19 -0400 Subject: [PATCH 5/7] Update filing_linkers.R --- R/filing_linkers.R | 111 ++++++++++++++++++++++++--------------------- 1 file changed, 59 insertions(+), 52 deletions(-) diff --git a/R/filing_linkers.R b/R/filing_linkers.R index 6bb1dc8..8e47d4d 100644 --- a/R/filing_linkers.R +++ b/R/filing_linkers.R @@ -2,23 +2,19 @@ source("R/standardizers.R") source("R/loaders.R") # OVERVIEW -# The goal of this script is to link eviction filings and their plaintiffs to property owners +# The goal of this script is to link eviction filings and their plaintiffs to assessor property records and their owners # STEP 1: Match eviction filing addresses to assessor records addresses -# STEP 1A: Match on address, city, zip, or address, city +# STEP 1A: Direct string match on address, city, zip, or address, city # STEP 1B: For evictions that don't match directly on address, do a # spatial nearest neighbor join (defined in match_nearby_filings() helper function) # to parcel data and then try to match addresses. # STEP 2: Match eviction plantiffs to owners +# STEP 2A: Direct string match eviction plantiff name to owner name +# STEP 2B: Cosine similarity fuzzy match eviction plantiff name to owner name +# STEP 2C: Match evictions and owners based on loc_id and if filing_date is within fy and ls_date of assessors data - -# NEW WORKFLOW - -# 1. tie address to address in addresses table (or append if not found). -# 2. if address not found (or if existing address has no location), associate parcel loc_id with address. -# 3. join to owners on name/address. -# 4. join to owners on name (cosine similarity)/address. -# 5. for those filings unmatched in 3-4, match by name within parcel. - +# TO DO: Add log messages # HELPER FUNCTIONS -------------------------------------------------------- @@ -94,7 +90,7 @@ process_link_filings <- function(assess_df, evic_df = filings, parcels_points, t "muni_id" = "muni_id"), na_matches = "never") - # PART 1A - MATCH EVICTION FILINGS TO ASSESSORS RECORDS ADDRESSES BASED ON A COMBINATION OF ADDRESS, CITY, ZIP + # STEP 1A - DIRECT STRING MATCH EVICTION FILINGS TO ASSESSORS RECORDS ADDRESSES BASED ON A COMBINATION OF ADDRESS, CITY, ZIP # Join eviction filings to assessors data by address, city, and zip code filings_clean <- filings |> # clean and standardize eviction filings addresses @@ -148,7 +144,7 @@ process_link_filings <- function(assess_df, evic_df = filings, parcels_points, t dplyr::filter(loc_id %in% dplyr::pull(assessor, loc_id)) - # PART 1B - FOR EVICTIONS WITHOUT A DIRECT ADDRESS MATCH, TRY TO SPATIALLY JOIN TO PARCEL DATA + # STEP 1B - FOR EVICTIONS WITHOUT A DIRECT ADDRESS MATCH, TRY TO SPATIALLY JOIN TO PARCEL DATA # AND THEN FUZZY MATCH ADDRESSES # Filter to eviction filings unmatched on address - 27,606 records @@ -221,73 +217,84 @@ process_link_filings <- function(assess_df, evic_df = filings, parcels_points, t # join assessors records to owners con <- duckdb::dbConnect(duckdb::duckdb()) - # Register your R dataframes as virtual tables in DuckDB + # Register owners and assessor dataframes as virtual tables in DuckDB duckdb::duckdb_register(con, "owners", owners) duckdb::duckdb_register(con, "assessor", assessor) + # NOTE: IT LOOKS LIKE CONDOS ARE STILL A PROBLEM - WHEN YOU JOIN ASSESSOR TO OWNERS BASED + # ON ADDR_ID, EVERY RECORD FOR A CONDO IS LINKED TO EACH OWNER - CAN'T DISCERN BETWEEN INDIVIDUAL + # SELL DATES + # Join them using dplyr syntax (executed via DuckDB) owners_assess <- dplyr::tbl(con, "owners") |> dplyr::distinct(name, addr_id, cosine_group, network_group) |> - tidylog::left_join(dplyr::tbl(con, "assessor")|> dplyr::select(c(loc_id, addr, addr_id, muni, postal)) |> dplyr::distinct(), by = "addr_id") |> + tidylog::left_join(dplyr::tbl(con, "assessor")|> dplyr::select(c(loc_id, ls_date, fy, addr, addr_id, muni, postal)) |> dplyr::distinct(), by = "addr_id") |> dplyr::collect() |># Pulls the final result back into R - dplyr::distinct(name, addr_id, cosine_group, network_group,.keep_all= TRUE) # get one record per name and address + dplyr::arrange(desc(ls_date)) |> # arrange so that only most recent assessor data is kept for each owner + dplyr::distinct(name, addr_id,cosine_group, network_group,.keep_all= TRUE) |> # get one record per name and address + dplyr::mutate(fy = lubridate::make_date(year = fy, month = 6, day = 30)) |> # turn fy to month day value + # filter out condo owners where it's not possible to directly link owner to eviction + dplyr::group_by(loc_id) |> + dplyr::mutate(has_multiple_owners = dplyr::n_distinct(name) > 1) # STEP 2A - DIRECT STRING MATCH TO OWNERS - 3,136 plaintiffs match owners + loc_id, 33,341 don't filings_spatial_owners <- filings_spatial_clean |> - tidylog::left_join(owners_assess, by = c("name", "loc_id")) + tidylog::left_join(owners_assess, by = c("name", "loc_id"), suffix = c("_evic", "_assess")) # TRYING FUZZY MATCH # data frame of matched filings_direct_own_match <- filings_spatial_owners |> dplyr::filter(!is.na(addr_id)) |> - dplyr::mutate(owner_link = "name_loc_id") + dplyr::mutate(owner_link = "name_loc_id", + name_evic = name) + + # list of matched loc_id + direct_match_loc_id <- filings_direct_own_match |> + dplyr::pull("loc_id") # STEP 2b: FUZZY MATCH USING COSINE SIMILARITY # Filter out plaintiffs that matched - 1,718 new matches fuzzy_match_plantiff <- filings_spatial_clean |> # filter evictions already matched - dplyr::anti_join(owners_assess, by = c("name")) |> - tidylog::left_join(owners_assess, by = c("loc_id")) |> + dplyr::filter(!loc_id %in% direct_match_loc_id) |> + tidylog::left_join(owners_assess, by = c("loc_id"),suffix = c("_evic", "_assess")) |> # calculate stringdit using cosine similarity and character level overlap dplyr::mutate( # 1. Cosine Similarity (q=2 captures "chunks" of names) # This prevents "Jon" and "Ian" from looking too similar - cosine_sim = stringdist::stringsim(name.x, name.y, method = "cosine", q = 2), + cosine_sim = stringdist::stringsim(name_evic, name_assess, method = "cosine", q = 2), # 2. Character-level overlap (how many characters they share) # Using q=1 makes it a "bag of characters" comparison - char_overlap = stringdist::stringsim(name.x, name.y, method = "cosine", q = 1) - ) |> - dplyr::filter(cosine_sim > 0.5) - - # STEP 2C - FOR REMAINING UNMATCHED - MATCH BY NAME WITHIN PARCEL - filin - - # match cleaned plantiff names toMatc eviction filings by docket_id - filings_spatial_names <- filings_spatial_clean |> - tidylog::left_join(plantiff_clean, by = c("docket_id")) - - # filings_spatial <- filings_no_match |> - # sf::st_as_sf() |> - # sf::st_transform(2249) |> - # sf::st_join(parcels, join=sf::st_intersects) |> - # dplyr::mutate( - # link_type = dplyr::case_when( - # !is.na(loc_id) ~ "spatial" - # ) - # ) |> - # dplyr::bind_rows(filings_address_match, filings_zip_match, filings_unmatchable) - # - # sf::st_drop_geometry() |> - # dplyr::select(docket_id, loc_id, city, zip, link_type) |> - # write_multi(FILINGS_OUT_NAME) - - filings_by_parcel <- filings |> - dplyr::group_by(loc_id) |> - dplyr::summarize( - filing_count = dplyr::n() + char_overlap = stringdist::stringsim(name_evic, name_assess, method = "cosine", q = 1) ) |> - write_multi("filings_per_parcel") - filings + dplyr::filter(cosine_sim > 0.5) |> #BUMP UP - CHECK + dplyr::mutate(owner_link = "fuzzy_loc_id") |> + dplyr::select(-c(cosine_sim, char_overlap)) + + # fuzzy matched filings - 1,487 match + filings_own_unmatched <- fuzzy_match_plantiff |> + dplyr::pull(loc_id) + + # STEP 2C - MATCH ON ADDRESS AND DATE RANGE + filings_own_date <- filings_spatial_clean |> + #filter out already matches names + dplyr::filter(!loc_id %in% direct_match_loc_id & !loc_id %in% filings_own_unmatched) |> + tidylog::left_join(owners_assess, by = c("loc_id"),suffix = c("_evic", "_assess")) |> + dplyr::filter(file_date <= fy & file_date >= ls_date) |># filing falls between last sell date and fy of assessor data as proxy for current ownership + dplyr::mutate(owner_link = "date_range") + + + # COMBINE INTO ONE DF - + evictions_owners <- dplyr::bind_rows(filings_own_date|> sf::st_as_sf() |> sf::st_transform(2249) , + fuzzy_match_plantiff |> sf::st_as_sf() |> sf::st_transform(2249) , + filings_direct_own_match |> sf::st_as_sf() |> sf::st_transform(2249)) + + # BEST WAY TO WRITE OUT? + + return(evictions_owners) + } +# test calling function +process_link_filings() From d62aa81054f216f54379d984d2d816005efee4a3 Mon Sep 17 00:00:00 2001 From: Amy Rogin <32721672+rogin123@users.noreply.github.com> Date: Wed, 13 May 2026 09:32:58 -0400 Subject: [PATCH 6/7] Update filing_linkers.R --- R/filing_linkers.R | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/R/filing_linkers.R b/R/filing_linkers.R index 8e47d4d..99f5205 100644 --- a/R/filing_linkers.R +++ b/R/filing_linkers.R @@ -1,5 +1,7 @@ source("R/standardizers.R") +source('load_results.R') source("R/loaders.R") +source("R/processors.R") # OVERVIEW # The goal of this script is to link eviction filings and their plaintiffs to assessor property records and their owners @@ -14,7 +16,18 @@ source("R/loaders.R") # STEP 2B: Cosine similarity fuzzy match eviction plantiff name to owner name # STEP 2C: Match evictions and owners based on loc_id and if filing_date is within fy and ls_date of assessors data -# TO DO: Add log messages + +# LOAD DATA --------------------------------------------------------------- + +# load owner/parcel data +load_results("", load_boundaries=TRUE, summarize=TRUE) + +# plaintiffs - landlord bringing filing +# docket id if unique identifier from filings to plantiff +load_evic_results("EVICTION") + +# load places data +places <- load_places(munis = munis, zips, crs = 2249) # HELPER FUNCTIONS -------------------------------------------------------- @@ -92,6 +105,8 @@ process_link_filings <- function(assess_df, evic_df = filings, parcels_points, t # STEP 1A - DIRECT STRING MATCH EVICTION FILINGS TO ASSESSORS RECORDS ADDRESSES BASED ON A COMBINATION OF ADDRESS, CITY, ZIP # Join eviction filings to assessors data by address, city, and zip code + message("LOG:Join eviction filings to assessors data by address, city, and zip code") + filings_clean <- filings |> # clean and standardize eviction filings addresses process_filings() |> @@ -108,6 +123,8 @@ process_link_filings <- function(assess_df, evic_df = filings, parcels_points, t ) # if didn't match on addr, city, and zip - match just on addr and zip + message("LOG:Join eviction filings to assessors data by address and zip code") + filings_no_address <- filings_clean |> dplyr::filter(is.na(loc_id)) |> dplyr::select(-c(loc_id)) |> @@ -154,6 +171,7 @@ process_link_filings <- function(assess_df, evic_df = filings, parcels_points, t ) # spatial join with parcel data + message("LOG:Spatial join fillings to assessors data") filings_spatial <- match_nearby_filings(parcel_points_df=parcels, filings_df =filings_no_match) |> # helper function defined above # filter out unmatched rows dplyr::filter(!is.na(loc_id)) |> @@ -202,6 +220,7 @@ process_link_filings <- function(assess_df, evic_df = filings, parcels_points, t link_type = "spatial_fuzzy") # COMBINE INTO ONE DF - 35,756 records - all values from filings match, but not all plaintiffs are in eviction records + message("LOG:Combine all eviction filings with assessors data into one dataframe") filings_spatial_clean <- dplyr::bind_rows(filings_spatial_direct |> sf::st_as_sf() |> sf::st_transform(2249) , filings_spatial_fuzzy |> sf::st_as_sf() |> sf::st_transform(2249), filings_address_match |> sf::st_as_sf() |> sf::st_transform(2249), @@ -226,6 +245,7 @@ process_link_filings <- function(assess_df, evic_df = filings, parcels_points, t # SELL DATES # Join them using dplyr syntax (executed via DuckDB) + message("LOG:Join owners and assessors data") owners_assess <- dplyr::tbl(con, "owners") |> dplyr::distinct(name, addr_id, cosine_group, network_group) |> tidylog::left_join(dplyr::tbl(con, "assessor")|> dplyr::select(c(loc_id, ls_date, fy, addr, addr_id, muni, postal)) |> dplyr::distinct(), by = "addr_id") |> @@ -237,6 +257,8 @@ process_link_filings <- function(assess_df, evic_df = filings, parcels_points, t dplyr::group_by(loc_id) |> dplyr::mutate(has_multiple_owners = dplyr::n_distinct(name) > 1) + message("LOG: Match evictions to owners") + # STEP 2A - DIRECT STRING MATCH TO OWNERS - 3,136 plaintiffs match owners + loc_id, 33,341 don't filings_spatial_owners <- filings_spatial_clean |> tidylog::left_join(owners_assess, by = c("name", "loc_id"), suffix = c("_evic", "_assess")) @@ -285,7 +307,8 @@ process_link_filings <- function(assess_df, evic_df = filings, parcels_points, t dplyr::mutate(owner_link = "date_range") - # COMBINE INTO ONE DF - + # COMBINE INTO ONE DF + message("LOG: Combine eviction filings and owners into one dataframe") evictions_owners <- dplyr::bind_rows(filings_own_date|> sf::st_as_sf() |> sf::st_transform(2249) , fuzzy_match_plantiff |> sf::st_as_sf() |> sf::st_transform(2249) , filings_direct_own_match |> sf::st_as_sf() |> sf::st_transform(2249)) @@ -297,4 +320,4 @@ process_link_filings <- function(assess_df, evic_df = filings, parcels_points, t } # test calling function -process_link_filings() +final_results <- process_link_filings() From 08eeea3dfdbcb434eced8e64ac8057752b659fec Mon Sep 17 00:00:00 2001 From: Amy Rogin <32721672+rogin123@users.noreply.github.com> Date: Mon, 18 May 2026 11:43:12 -0400 Subject: [PATCH 7/7] Update filing_linkers.R --- R/filing_linkers.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/filing_linkers.R b/R/filing_linkers.R index 99f5205..218efb3 100644 --- a/R/filing_linkers.R +++ b/R/filing_linkers.R @@ -275,7 +275,7 @@ process_link_filings <- function(assess_df, evic_df = filings, parcels_points, t dplyr::pull("loc_id") # STEP 2b: FUZZY MATCH USING COSINE SIMILARITY - # Filter out plaintiffs that matched - 1,718 new matches + # Filter out plaintiffs that matched - 1,487 new matches fuzzy_match_plantiff <- filings_spatial_clean |> # filter evictions already matched dplyr::filter(!loc_id %in% direct_match_loc_id) |>