diff --git a/.env.development b/.env.development index 254a937782c..58276007983 100644 --- a/.env.development +++ b/.env.development @@ -1,4 +1,5 @@ ATS_API_CLIENT_TESTING_API_KEY=a94272cb45956f2116f3026847ebb640dc60a2fe +DATABASE_URL=postgis://postgres:postgres@localhost DFE_SIGN_IN_REDIRECT_URL=http://localhost:3000/auth/dfe/callback DISABLE_EMAILS=false DOMAIN=localhost:3000 diff --git a/app/jobs/import_organisation_data_job.rb b/app/jobs/import_organisation_data_job.rb index 2fb62a57c28..1ae56f66024 100644 --- a/app/jobs/import_organisation_data_job.rb +++ b/app/jobs/import_organisation_data_job.rb @@ -4,10 +4,10 @@ class ImportOrganisationDataJob < ApplicationJob def perform SchoolGroupMembership.mark_all_records_for_deletion - Gias::ImportSchoolsAndLocalAuthorities.new.call + Gias::ImportSchoolsAndLocalAuthorities.call Gias::ImportTrusts.new.call - # NOTE: Gias::ImportSchoolsAndLocalAuthorities.new.call updates most SchoolGroupMemberships so they are no longer marked for deletion. Only + # NOTE: Gias::ImportSchoolsAndLocalAuthorities.call updates most SchoolGroupMemberships so they are no longer marked for deletion. Only # the SchoolGroupMemberships that don't exist in the new import will still be marked for deletion. SchoolGroupMembership.delete_records_marked_for_deletion end diff --git a/app/models/organisation.rb b/app/models/organisation.rb index 87663cc2913..7c13ee41d36 100644 --- a/app/models/organisation.rb +++ b/app/models/organisation.rb @@ -6,6 +6,8 @@ class Organisation < ApplicationRecord include PgSearch::Model extend FriendlyId + include Discard::Model + has_rich_text :description SPECIAL_SCHOOL_TYPES = ["Community special school", "Foundation special school", "Non-maintained special school", "Academy special converter", "Academy special sponsor led", "Free schools special"].freeze diff --git a/app/models/school.rb b/app/models/school.rb index 1f7cc91b077..b190f72c01d 100644 --- a/app/models/school.rb +++ b/app/models/school.rb @@ -2,7 +2,7 @@ class School < Organisation has_many :school_group_memberships, dependent: :destroy has_many :school_groups, through: :school_group_memberships - scope :not_excluded, -> { where.not(detailed_school_type: EXCLUDED_DETAILED_SCHOOL_TYPES) } + scope :not_excluded, -> { kept.where.not(detailed_school_type: EXCLUDED_DETAILED_SCHOOL_TYPES) } validates :urn, uniqueness: true @@ -10,13 +10,16 @@ class School < Organisation LA_SCHOOL_TYPE = "Local authority maintained schools".freeze FREE_SCHOOL_TYPE = "Free Schools".freeze INDEPENDENT_SCHOOL_TYPE = "Independent schools".freeze - VALID_SCHOOL_TYPES = [LA_SCHOOL_TYPE, INDEPENDENT_SCHOOL_TYPE, "Special schools", "Universities", ACADEMY_TYPE, FREE_SCHOOL_TYPE, "Welsh schools", "Other types", "Colleges", "Online provider"].freeze + COLLEGE_SCHOOL_TYPE = "Colleges".freeze + FE_DETAILED_SCHOOL_TYPE = "Further education".freeze + VALID_SCHOOL_TYPES = [LA_SCHOOL_TYPE, INDEPENDENT_SCHOOL_TYPE, "Special schools", "Universities", ACADEMY_TYPE, FREE_SCHOOL_TYPE, "Welsh schools", "Other types", COLLEGE_SCHOOL_TYPE, "Online provider"].freeze + EXCLUDED_SCHOOL_TYPES = ["Universities", "Welsh schools", "Online providers"].freeze - # This is direct from GIAS (with plurals removed via singularize) + # This is direct from GIAS validates :school_type, inclusion: { in: VALID_SCHOOL_TYPES } EXCLUDED_DETAILED_SCHOOL_TYPES = [ - "Further education", + FE_DETAILED_SCHOOL_TYPE, "Other independent school", "Online provider", "British schools overseas", @@ -51,6 +54,10 @@ class School < Organisation through: %i[early_years ks1 ks2 ks3 ks4 ks5], }.freeze + def excluded? + !kept? || detailed_school_type.in?(EXCLUDED_DETAILED_SCHOOL_TYPES) + end + def religious_character return if !respond_to?(:gias_data) || gias_data.nil? return if ["None", "Does not apply"].include?(gias_data["ReligiousCharacter (name)"]) diff --git a/app/models/school_group.rb b/app/models/school_group.rb index 1dcfd231026..af8206bd919 100644 --- a/app/models/school_group.rb +++ b/app/models/school_group.rb @@ -31,4 +31,8 @@ def all_organisation_ids def ats_interstitial_variant "non_faith" end + + def excluded? + false + end end diff --git a/app/models/school_group_membership.rb b/app/models/school_group_membership.rb index e1080873db6..bcc7f0ecffa 100644 --- a/app/models/school_group_membership.rb +++ b/app/models/school_group_membership.rb @@ -16,7 +16,7 @@ def self.delete_records_marked_for_deletion to_delete = marked_for_deletion.map { |m| m.school_group.name } .group_by { |x| x } .transform_values(&:length) - Sentry.capture_message("Memberships to delete, by SchoolGroup: #{to_delete}", level: :info) + Sentry.capture_message("Memberships to delete, by SchoolGroup: #{to_delete}", level: :info) if marked_for_deletion.any? if marked_for_deletion.count > MAX_RECORDS_TO_BULK_DELETE # If this error is raised investigate the school_groups associated with the marked for deletion school group memberships. if a school group, local authority, diff --git a/app/services/gias/import_schools_and_local_authorities.rb b/app/services/gias/import_schools_and_local_authorities.rb index 29e3c5ef7ce..f663534f36f 100644 --- a/app/services/gias/import_schools_and_local_authorities.rb +++ b/app/services/gias/import_schools_and_local_authorities.rb @@ -1,141 +1,158 @@ require "log_benchmark" +require "csv" class Gias::ImportSchoolsAndLocalAuthorities SCHOOLS_AND_LOCAL_AUTHORITIES_CSV = "edubasealldata".freeze BATCH_SIZE = 100 - include LogBenchmark - - def initialize - reset_data - end + extend LogBenchmark class ImportFailure < StandardError end - def call - log_benchmark("Importing schools and local authorities") do - import_errors = Gias::Data.new(SCHOOLS_AND_LOCAL_AUTHORITIES_CSV).each_slice(BATCH_SIZE).flat_map do |group| - group.each do |row| - local_authorities.add(group_data(row)) - schools.push(school_data(row)) - memberships.push(membership_data(row)) + class << self + def call + # This file is a list of colleges in-scope for FE vacancies on TVS + uk_colleges = CSV.read(Rails.root.join("config/data/colleges.csv"), headers: true).index_by { |r| r.fetch("UKPRN").to_i }.transform_values(&:to_h) + + log_benchmark("Importing schools and local authorities") do + import_errors = Gias::Data.new(SCHOOLS_AND_LOCAL_AUTHORITIES_CSV).each_slice(BATCH_SIZE).flat_map do |group| + import_group uk_colleges, group end - import_batch + raise ImportFailure, import_errors.map(&:errors) if import_errors.any? end - raise(ImportFailure, import_errors.map { |x| x.errors.full_messages }) if import_errors.any? end - end - private + private + + def import_group(uk_colleges, group) + local_authorities = Set.new # LAs are provided with every school so we can discard duplicates + schools = [] + memberships = [] + discarded = [] + + group.each do |row| + local_authorities.add(group_data(row)) + school_row = school_data(row) + schools.push(school_row) + if (school_row.fetch(:school_type) == School::COLLEGE_SCHOOL_TYPE && school_row.fetch(:detailed_school_type) == School::FE_DETAILED_SCHOOL_TYPE && uk_colleges.exclude?(school_row.fetch(:uk_prn))) + || school_row.fetch(:school_type).in?(School::EXCLUDED_SCHOOL_TYPES) + || school_row.fetch(:establishment_status) == "Closed" + discarded.push(school_row) + end + memberships.push(membership_data(row)) + end + import_batch(local_authorities, schools, memberships).tap do + discarded.each { |school_row| School.find_by!(urn: school_row.fetch(:urn)).discard } + end + end - attr_reader :local_authorities, :schools, :memberships + def import_batch(local_authorities, schools, memberships) + import_local_authorities(local_authorities).failed_instances + + import_schools(schools).failed_instances + + import_memberships(local_authorities, schools, memberships).failed_instances + end - def reset_data - @local_authorities = Set.new # LAs are provided with every school so we can discard duplicates - @schools = [] - @memberships = [] - end + def import_local_authorities(local_authorities) + SchoolGroup.import( + local_authorities.to_a, + on_duplicate_key_update: { + conflict_target: [:local_authority_code], + columns: local_authorities.first.keys, + }, + ) + end - # sum doesn't work on arrays the same way it works on Integers - # rubocop:disable Performance/Sum - def import_batch - [import_local_authorities, - import_schools, - import_memberships].map(&:failed_instances) - .reduce(:+).tap { reset_data } - end - # rubocop:enable Performance/Sum - - def import_local_authorities - SchoolGroup.import( - local_authorities.to_a, - on_duplicate_key_update: { - conflict_target: [:local_authority_code], - columns: local_authorities.first.keys, - }, - ) - end + def import_schools(schools) + imported_schools = schools.map { |s| s.except(:uk_prn, :religious_character).merge(discarded_at: nil) } + School.import( + imported_schools, + on_duplicate_key_update: { + conflict_target: [:urn], + columns: imported_schools.first.keys, + }, + ) + end - def import_schools - School.import( - schools, - on_duplicate_key_update: { - conflict_target: [:urn], - columns: schools.first.keys, - }, - ) - end + def import_memberships(local_authorities, schools, memberships) + school_ids = School.where(urn: schools.map { |s| s[:urn] }).pluck(:urn, :id).to_h + group_ids = SchoolGroup.where( + local_authority_code: local_authorities.map { |la| la[:local_authority_code] }, + ).pluck(:local_authority_code, :id).to_h + + # school_group_memberships = memberships.map do |m| + # { + # school_id: school_ids.fetch(m.fetch(:urn)), + # school_group_id: group_ids.fetch(m.fetch(:local_authority_code)), + # do_not_delete: true, + # } + # end + school_group_memberships = memberships.map do |m| + { + school_id: school_ids[m[:urn]], + school_group_id: group_ids[m[:local_authority_code]], + do_not_delete: true, + } + end - def import_memberships - school_ids = School.where(urn: schools.map { |s| s[:urn] }).pluck(:urn, :id).to_h - group_ids = SchoolGroup.where( - local_authority_code: local_authorities.map { |la| la[:local_authority_code] }, - ).pluck(:local_authority_code, :id).to_h + SchoolGroupMembership.import( + school_group_memberships, + on_duplicate_key_update: { + conflict_target: %i[school_id school_group_id], + columns: school_group_memberships.first.keys, + }, + ) + end - school_group_memberships = memberships.map do |m| + def group_data(row) { - school_id: school_ids[m[:urn]], - school_group_id: group_ids[m[:local_authority_code]], - do_not_delete: true, + local_authority_code: row["LA (code)"], + name: row["LA (name)"], + group_type: "local_authority", + gias_data: row.to_h.slice("LA (code)", "LA (name)"), } end - SchoolGroupMembership.import( - school_group_memberships, - on_duplicate_key_update: { - conflict_target: %i[school_id school_group_id], - columns: school_group_memberships.first.keys, - }, - ) - end - - def group_data(row) - { - local_authority_code: row["LA (code)"], - name: row["LA (name)"], - group_type: "local_authority", - gias_data: row.to_h.slice("LA (code)", "LA (name)"), - } - end - - def school_data(row) # rubocop:disable Metrics/MethodLength - { - urn: row["URN"], - address: row["Street"], - address3: row["Address3"], - county: row["County (name)"], - detailed_school_type: row["TypeOfEstablishment (name)"], - establishment_status: row["EstablishmentStatus (name)"], - local_authority_within: row["LA (name)"], - locality: row["Locality"], - maximum_age: row["StatutoryHighAge"], - minimum_age: row["StatutoryLowAge"], - name: row["EstablishmentName"], - postcode: row["Postcode"], - region: row["GOR (name)"], - school_type: row["EstablishmentTypeGroup (name)"], - town: row["Town"], - phase: row["PhaseOfEducation (code)"].to_i, - url: Addressable::URI.heuristic_parse(row["SchoolWebsite"]).to_s, - gias_data: row.to_h, - }.merge(school_location_data(row)).transform_values(&:presence) - end + def school_data(row) # rubocop:disable Metrics/MethodLength + { + urn: row["URN"], + address: row["Street"], + address3: row["Address3"], + county: row["County (name)"], + detailed_school_type: row["TypeOfEstablishment (name)"], + establishment_status: row["EstablishmentStatus (name)"], + local_authority_within: row["LA (name)"], + locality: row["Locality"], + maximum_age: row["StatutoryHighAge"], + minimum_age: row["StatutoryLowAge"], + name: row["EstablishmentName"], + postcode: row["Postcode"], + region: row["GOR (name)"], + school_type: row["EstablishmentTypeGroup (name)"], + town: row["Town"], + phase: row["PhaseOfEducation (code)"].to_i, + url: Addressable::URI.heuristic_parse(row["SchoolWebsite"]).to_s, + uk_prn: row["UKPRN"].to_i, + gias_data: row.to_h, + religious_character: row["ReligiousCharacter (name)"], + }.merge(school_location_data(row)).transform_values(&:presence) + end - def school_location_data(row) - return {} unless row["Easting"] && row["Northing"] + def school_location_data(row) + return {} unless row["Easting"] && row["Northing"] - uk27700 = GeoFactories::FACTORY_27700.point(row["Easting"].to_i, row["Northing"].to_i) - { - uk_geopoint: uk27700, - geopoint: GeoFactories.convert_sr27700_to_wgs84(uk27700), - } - end + uk27700 = GeoFactories::FACTORY_27700.point(row["Easting"].to_i, row["Northing"].to_i) + { + uk_geopoint: uk27700, + geopoint: GeoFactories.convert_sr27700_to_wgs84(uk27700), + } + end - def membership_data(row) - { - urn: row["URN"], - local_authority_code: row["LA (code)"], - } + def membership_data(row) + { + urn: row["URN"], + local_authority_code: row["LA (code)"], + } + end end end diff --git a/app/services/gias/import_trusts.rb b/app/services/gias/import_trusts.rb index bc44566d18d..c4500b776fd 100644 --- a/app/services/gias/import_trusts.rb +++ b/app/services/gias/import_trusts.rb @@ -3,7 +3,9 @@ class Gias::ImportTrusts TRUSTS_CSV = "allgroupsdata".freeze + # alllinksdata only seems to have type 5 and 6 links in it. TRUST_MEMBERSHIPS_CSV = "alllinksdata".freeze + # TRUST_MEMBERSHIPS_CSV = "links_edubasealldata".freeze MAT_GROUP_TYPE = 6 include LogBenchmark diff --git a/app/services/vacancies/import/shared.rb b/app/services/vacancies/import/shared.rb index 2bc1b0a4afc..bea0e19430b 100644 --- a/app/services/vacancies/import/shared.rb +++ b/app/services/vacancies/import/shared.rb @@ -2,9 +2,7 @@ module Vacancies::Import module Shared LEGACY_WORKING_PATTERNS = %w[flexible term_time job_share].freeze def vacancy_listed_at_excluded_school_type?(schools) - return false if schools.none? - - (schools.map(&:detailed_school_type) & School::EXCLUDED_DETAILED_SCHOOL_TYPES).present? + schools.any?(&:excluded?) end # Our system only imports MAT type trusts from GIAS DB. diff --git a/config/analytics.yml b/config/analytics.yml index 00fe4f57673..040f39f6827 100644 --- a/config/analytics.yml +++ b/config/analytics.yml @@ -272,6 +272,7 @@ shared: - group_type - local_authority_within - establishment_status + - discarded_at organisation_publisher_preferences: - id - organisation_id diff --git a/config/data/colleges.csv b/config/data/colleges.csv new file mode 100644 index 00000000000..6bf0ce6ee63 --- /dev/null +++ b/config/data/colleges.csv @@ -0,0 +1,211 @@ +UKPRN,Provider Name,Provider Type,HEI subsidiary +10000794,BOLTON COLLEGE,Statutory FE Sector - Designated Institution,HEI subsidiary +10008641,FIRCROFT COLLEGE OF ADULT EDUCATION,Statutory FE Sector - Designated Institution, +10080810,HARTPURY COLLEGE OF FURTHER EDUCATION,Statutory FE Sector - Designated Institution,HEI subsidiary +10007875,MARY WARD SETTLEMENT,Statutory FE Sector - Designated Institution, +10004432,MORLEY COLLEGE LIMITED,Statutory FE Sector - Designated Institution, +10003088,RICHMOND AND HILLCROFT ADULT AND COMMUNITY COLLEGE,Statutory FE Sector - Designated Institution, +10005583,RUSKIN COLLEGE,Statutory FE Sector - Designated Institution,HEI subsidiary +10003755,SOUTH BANK COLLEGES,Statutory FE Sector - Designated Institution,HEI subsidiary +10001463,THE CITY LITERARY INSTITUTE,Statutory FE Sector - Designated Institution, +10004204,THE MARINE SOCIETY AND SEA CADETS,Statutory FE Sector - Designated Institution, +10007364,WORKERS' EDUCATIONAL ASSOCIATION,Statutory FE Sector - Designated Institution, +10007636,WORKING MEN'S COLLEGE CORPORATION,Statutory FE Sector - Designated Institution, +10094574,WRITTLE COLLEGE LIMITED,Statutory FE Sector - Designated Institution,HEI subsidiary +10000055,ABINGDON AND WITNEY COLLEGE,Statutory FE Sector - General Further Education College, +10004927,ACTIVATE LEARNING,Statutory FE Sector - General Further Education College, +10057981,ADA NATIONAL COLLEGE FOR DIGITAL SKILLS,Statutory FE Sector - General Further Education College, +10000415,ASKHAM BRYAN COLLEGE,Statutory FE Sector - General Further Education College, +10000528,BARKING AND DAGENHAM COLLEGE,Statutory FE Sector - General Further Education College, +10000533,BARNET & SOUTHGATE COLLEGE,Statutory FE Sector - General Further Education College, +10000536,BARNSLEY COLLEGE,Statutory FE Sector - General Further Education College, +10000560,BASINGSTOKE COLLEGE OF TECHNOLOGY,Statutory FE Sector - General Further Education College, +10001465,BATH COLLEGE,Statutory FE Sector - General Further Education College, +10000610,BEDFORD COLLEGE,Statutory FE Sector - General Further Education College, +10006442,BIRMINGHAM METROPOLITAN COLLEGE,Statutory FE Sector - General Further Education College, +10000720,BISHOP AUCKLAND COLLEGE,Statutory FE Sector - General Further Education College, +10000721,BISHOP BURTON COLLEGE,Statutory FE Sector - General Further Education College, +10000747,BLACKBURN COLLEGE,Statutory FE Sector - General Further Education College, +10000754,BLACKPOOL AND THE FYLDE COLLEGE,Statutory FE Sector - General Further Education College, +10000812,BOSTON COLLEGE,Statutory FE Sector - General Further Education College, +10000820,"BOURNEMOUTH AND POOLE COLLEGE, THE",Statutory FE Sector - General Further Education College, +10000840,BRADFORD COLLEGE,Statutory FE Sector - General Further Education College, +10000944,BROCKENHURST COLLEGE,Statutory FE Sector - General Further Education College, +10000950,BROOKLANDS COLLEGE,Statutory FE Sector - General Further Education College, +10000473,BUCKINGHAMSHIRE COLLEGE GROUP,Statutory FE Sector - General Further Education College, +10001000,BURNLEY COLLEGE,Statutory FE Sector - General Further Education College, +10001004,BURTON AND SOUTH DERBYSHIRE COLLEGE,Statutory FE Sector - General Further Education College, +10001005,BURY COLLEGE,Statutory FE Sector - General Further Education College, +10001093,CALDERDALE COLLEGE,Statutory FE Sector - General Further Education College, +10001116,CAMBRIDGE REGIONAL COLLEGE,Statutory FE Sector - General Further Education College, +10001148,CAPEL MANOR COLLEGE,Statutory FE Sector - General Further Education College, +10007455,CAPITAL CITY COLLEGE GROUP,Statutory FE Sector - General Further Education College, +10001353,CHELMSFORD COLLEGE,Statutory FE Sector - General Further Education College, +10005972,CHESHIRE COLLEGE SOUTH AND WEST,Statutory FE Sector - General Further Education College, +10001378,CHESTERFIELD COLLEGE,Statutory FE Sector - General Further Education College, +10007817,CHICHESTER COLLEGE GROUP,Statutory FE Sector - General Further Education College, +10004772,CITY COLLEGE NORWICH,Statutory FE Sector - General Further Education College, +10005128,CITY COLLEGE PLYMOUTH,Statutory FE Sector - General Further Education College, +10001467,CITY OF BRISTOL COLLEGE,Statutory FE Sector - General Further Education College, +10007945,CITY OF PORTSMOUTH COLLEGE,Statutory FE Sector - General Further Education College, +10001475,CITY OF SUNDERLAND COLLEGE,Statutory FE Sector - General Further Education College, +10007578,CITY OF WOLVERHAMPTON COLLEGE,Statutory FE Sector - General Further Education College, +10001535,COLCHESTER INSTITUTE,Statutory FE Sector - General Further Education College, +10001696,CORNWALL COLLEGE,Statutory FE Sector - General Further Education College, +10003010,COVENTRY COLLEGE,Statutory FE Sector - General Further Education College, +10001743,CRAVEN COLLEGE,Statutory FE Sector - General Further Education College, +10001778,CROYDON COLLEGE,Statutory FE Sector - General Further Education College, +10001850,DARLINGTON COLLEGE,Statutory FE Sector - General Further Education College, +10001919,DCG,Statutory FE Sector - General Further Education College, +10001934,DERWENTSIDE COLLEGE,Statutory FE Sector - General Further Education College, +10004695,DN COLLEGES GROUP,Statutory FE Sector - General Further Education College, +10007924,DUDLEY COLLEGE OF TECHNOLOGY,Statutory FE Sector - General Further Education College, +10002094,"EALING, HAMMERSMITH & WEST LONDON COLLEGE",Statutory FE Sector - General Further Education College, +10004116,EAST COAST COLLEGE,Statutory FE Sector - General Further Education College, +10002111,EAST DURHAM COLLEGE,Statutory FE Sector - General Further Education College, +10004552,EAST LANCASHIRE LEARNING GROUP,Statutory FE Sector - General Further Education College, +10002130,EAST SURREY COLLEGE,Statutory FE Sector - General Further Education College, +10002923,EAST SUSSEX COLLEGE GROUP,Statutory FE Sector - General Further Education College, +10006570,EKC GROUP,Statutory FE Sector - General Further Education College, +10002370,EXETER AND NORTH DEVON COLLEGES GROUP,Statutory FE Sector - General Further Education College, +10002412,FARNBOROUGH COLLEGE OF TECHNOLOGY,Statutory FE Sector - General Further Education College, +10002599,FURNESS COLLEGE,Statutory FE Sector - General Further Education College, +10002638,GATESHEAD COLLEGE,Statutory FE Sector - General Further Education College, +10002696,GLOUCESTERSHIRE COLLEGE,Statutory FE Sector - General Further Education College, +10002743,GRANTHAM COLLEGE,Statutory FE Sector - General Further Education College, +10002852,HALESOWEN COLLEGE,Statutory FE Sector - General Further Education College, +10002899,HARLOW COLLEGE,Statutory FE Sector - General Further Education College, +10002917,HARTLEPOOL COLLEGE OF FURTHER EDUCATION,Statutory FE Sector - General Further Education College, +10005979,HAVANT AND SOUTH DOWNS COLLEGE,Statutory FE Sector - General Further Education College, +10007977,HEART OF WORCESTERSHIRE COLLEGE,Statutory FE Sector - General Further Education College, +10007289,HEART OF YORKSHIRE EDUCATION GROUP,Statutory FE Sector - General Further Education College, +10003022,HEREFORD COLLEGE OF ARTS,Statutory FE Sector - General Further Education College, +10003023,"HEREFORDSHIRE, LUDLOW, AND NORTH SHROPSHIRE COLLEGE",Statutory FE Sector - General Further Education College, +10003029,HEREWARD COLLEGE OF FURTHER EDUCATION,Statutory FE Sector - General Further Education College, +10003035,HERTFORD REGIONAL COLLEGE,Statutory FE Sector - General Further Education College, +10003146,HOPWOOD HALL COLLEGE,Statutory FE Sector - General Further Education College, +10007193,"HRUC (HARROW, RICHMOND AND UXBRIDGE COLLEGES).",Statutory FE Sector - General Further Education College, +10003193,HUGH BAIRD COLLEGE,Statutory FE Sector - General Further Education College, +10003200,HULL COLLEGE,Statutory FE Sector - General Further Education College, +10005077,INSPIRE EDUCATION GROUP,Statutory FE Sector - General Further Education College, +10003406,ISLE OF WIGHT COLLEGE,Statutory FE Sector - General Further Education College, +10003558,KENDAL COLLEGE,Statutory FE Sector - General Further Education College, +10003189,KIRKLEES COLLEGE,Statutory FE Sector - General Further Education College, +10003753,LAKES COLLEGE WEST CUMBRIA,Statutory FE Sector - General Further Education College, +10003768,LANCASTER AND MORECAMBE COLLEGE,Statutory FE Sector - General Further Education College, +10003855,LEEDS COLLEGE OF BUILDING,Statutory FE Sector - General Further Education College, +10003867,LEICESTER COLLEGE,Statutory FE Sector - General Further Education College, +10003928,LINCOLN COLLEGE,Statutory FE Sector - General Further Education College, +10000948,LONDON SOUTH EAST COLLEGES,Statutory FE Sector - General Further Education College, +10004112,LOUGHBOROUGH COLLEGE,Statutory FE Sector - General Further Education College, +10023139,LTE GROUP,Statutory FE Sector - General Further Education College, +10024962,LUMINATE EDUCATION GROUP,Statutory FE Sector - General Further Education College, +10004144,MACCLESFIELD COLLEGE,Statutory FE Sector - General Further Education College, +10004344,MIDDLESBROUGH COLLEGE,Statutory FE Sector - General Further Education College, +10004340,MID-KENT COLLEGE,Statutory FE Sector - General Further Education College, +10004375,MILTON KEYNES COLLEGE,Statutory FE Sector - General Further Education College, +10004442,MOULTON COLLEGE,Statutory FE Sector - General Further Education College, +10004478,MYERSCOUGH COLLEGE,Statutory FE Sector - General Further Education College, +10004599,NCG,Statutory FE Sector - General Further Education College, +10006963,NEW CITY COLLEGE,Statutory FE Sector - General Further Education College, +10004576,NEW COLLEGE DURHAM,Statutory FE Sector - General Further Education College, +10004579,NEW COLLEGE SWINDON,Statutory FE Sector - General Further Education College, +10004596,NEWBURY COLLEGE,Statutory FE Sector - General Further Education College, +10004603,NEWCASTLE AND STAFFORD COLLEGES GROUP,Statutory FE Sector - General Further Education College, +10004607,NEWHAM COLLEGE OF FURTHER EDUCATION,Statutory FE Sector - General Further Education College, +10004686,NORTH EAST SURREY COLLEGE OF TECHNOLOGY (NESCOT),Statutory FE Sector - General Further Education College, +10004690,NORTH HERTFORDSHIRE COLLEGE,Statutory FE Sector - General Further Education College, +10004721,NORTH KENT COLLEGE,Statutory FE Sector - General Further Education College, +10004718,NORTH WARWICKSHIRE AND SOUTH LEICESTERSHIRE COLLEGE,Statutory FE Sector - General Further Education College, +10007011,NORTHAMPTON COLLEGE,Statutory FE Sector - General Further Education College, +10004577,NOTTINGHAM COLLEGE,Statutory FE Sector - General Further Education College, +10004835,OAKLANDS COLLEGE,Statutory FE Sector - General Further Education College, +10005124,PLUMPTON COLLEGE,Statutory FE Sector - General Further Education College, +10005200,PRESTON COLLEGE,Statutory FE Sector - General Further Education College, +10005404,REASEHEATH COLLEGE,Statutory FE Sector - General Further Education College, +10002863,RIVERSIDE COLLEGE,Statutory FE Sector - General Further Education College, +10005534,RNN GROUP,Statutory FE Sector - General Further Education College, +10005575,RUNSHAW COLLEGE,Statutory FE Sector - General Further Education College, +10005032,SALFORD CITY COLLEGE,Statutory FE Sector - General Further Education College, +10005669,SANDWELL COLLEGE,Statutory FE Sector - General Further Education College, +10005788,"SHEFFIELD COLLEGE, THE",Statutory FE Sector - General Further Education College, +10005810,SHIPLEY COLLEGE,Statutory FE Sector - General Further Education College, +10005946,SOLIHULL COLLEGE AND UNIVERSITY CENTRE,Statutory FE Sector - General Further Education College, +10005967,SOUTH & CITY COLLEGE BIRMINGHAM,Statutory FE Sector - General Further Education College, +10005977,SOUTH DEVON COLLEGE,Statutory FE Sector - General Further Education College, +10005981,SOUTH ESSEX COLLEGE OF FURTHER AND HIGHER EDUCATION,Statutory FE Sector - General Further Education College, +10036143,SOUTH GLOUCESTERSHIRE AND STROUD COLLEGE,Statutory FE Sector - General Further Education College, +10007928,SOUTH HAMPSHIRE COLLEGE GROUP,Statutory FE Sector - General Further Education College, +10023526,SOUTH STAFFORDSHIRE COLLEGE,Statutory FE Sector - General Further Education College, +10003674,SOUTH THAMES COLLEGES GROUP,Statutory FE Sector - General Further Education College, +10006038,SOUTHPORT EDUCATION GROUP,Statutory FE Sector - General Further Education College, +10006050,SPARSHOLT COLLEGE,Statutory FE Sector - General Further Education College, +10006174,ST HELENS COLLEGE,Statutory FE Sector - General Further Education College, +10009439,STANMORE COLLEGE,Statutory FE Sector - General Further Education College, +10006349,STOKE ON TRENT COLLEGE,Statutory FE Sector - General Further Education College, +10006398,SUFFOLK NEW COLLEGE,Statutory FE Sector - General Further Education College, +10006494,TAMESIDE COLLEGE,Statutory FE Sector - General Further Education College, +10007938,TEC PARTNERSHIP,Statutory FE Sector - General Further Education College, +10006549,TELFORD COLLEGE,Statutory FE Sector - General Further Education College, +10003955,THE CITY OF LIVERPOOL COLLEGE,Statutory FE Sector - General Further Education College, +10007916,THE COLLEGE OF WEST ANGLIA,Statutory FE Sector - General Further Education College, +10006341,THE EDUCATION TRAINING COLLECTIVE,Statutory FE Sector - General Further Education College, +10001503,THE NORTHERN SCHOOL OF ART,Statutory FE Sector - General Further Education College, +10006770,THE OLDHAM COLLEGE,Statutory FE Sector - General Further Education College, +10005998,THE TRAFFORD AND STOCKPORT COLLEGE GROUP,Statutory FE Sector - General Further Education College, +10002107,THE WINDSOR FOREST COLLEGES GROUP,Statutory FE Sector - General Further Education College, +10007063,TRURO AND PENWITH COLLEGE,Statutory FE Sector - General Further Education College, +10005999,TYNE COAST COLLEGE,Statutory FE Sector - General Further Education College, +10005736,UNIFIED SEEVIC PALMER'S COLLEGE,Statutory FE Sector - General Further Education College, +10001476,UNITED COLLEGES GROUP,Statutory FE Sector - General Further Education College, +10000878,UNIVERSITY CENTRE SOMERSET COLLEGE GROUP,Statutory FE Sector - General Further Education College, +10007315,WALSALL COLLEGE,Statutory FE Sector - General Further Education College, +10007321,WALTHAM FOREST COLLEGE,Statutory FE Sector - General Further Education College, +10007339,WARRINGTON & VALE ROYAL COLLEGE,Statutory FE Sector - General Further Education College, +10007859,WARWICKSHIRE COLLEGE,Statutory FE Sector - General Further Education College, +10007417,WEST HERTS COLLEGE,Statutory FE Sector - General Further Education College, +10007427,WEST NOTTINGHAMSHIRE COLLEGE,Statutory FE Sector - General Further Education College, +10007431,WEST SUFFOLK COLLEGE,Statutory FE Sector - General Further Education College, +10007434,WEST THAMES COLLEGE,Statutory FE Sector - General Further Education College, +10007459,WESTON COLLEGE OF FURTHER AND HIGHER EDUCATION,Statutory FE Sector - General Further Education College, +10007469,WEYMOUTH AND KINGSTON MAURWARD COLLEGE,Statutory FE Sector - General Further Education College, +10007500,WIGAN AND LEIGH COLLEGE,Statutory FE Sector - General Further Education College, +10007527,WILTSHIRE COLLEGE AND UNIVERSITY CENTRE,Statutory FE Sector - General Further Education College, +10007553,WIRRAL METROPOLITAN COLLEGE,Statutory FE Sector - General Further Education College, +10007696,YEOVIL COLLEGE,Statutory FE Sector - General Further Education College, +10007709,YORK COLLEGE,Statutory FE Sector - General Further Education College, +10000330,AQUINAS COLLEGE,Statutory FE Sector - Sixth Form College, +10000552,BARTON PEVERIL SIXTH FORM COLLEGE,Statutory FE Sector - Sixth Form College, +10000670,BEXHILL COLLEGE,Statutory FE Sector - Sixth Form College, +10000796,BOLTON SIXTH FORM COLLEGE,Statutory FE Sector - Sixth Form College, +10000887,BRIGHTON HOVE AND SUSSEX SIXTH FORM COLLEGE,Statutory FE Sector - Sixth Form College, +10001165,CARDINAL NEWMAN COLLEGE,Statutory FE Sector - Sixth Form College, +10001201,CARMEL COLLEGE,Statutory FE Sector - Sixth Form College, +10001416,CHRIST THE KING SIXTH FORM COLLEGE,Statutory FE Sector - Sixth Form College, +10001446,CIRENCESTER COLLEGE,Statutory FE Sector - Sixth Form College, +10002770,GREENHEAD COLLEGE,Statutory FE Sector - Sixth Form College, +10003128,HOLY CROSS COLLEGE,Statutory FE Sector - Sixth Form College, +10003188,HUDDERSFIELD NEW COLLEGE,Statutory FE Sector - Sixth Form College, +10003427,ITCHEN COLLEGE,Statutory FE Sector - Sixth Form College, +10003491,JOHN LEGGOTT SIXTH FORM COLLEGE,Statutory FE Sector - Sixth Form College, +10003511,JOSEPH CHAMBERLAIN SIXTH FORM COLLEGE,Statutory FE Sector - Sixth Form College, +10003899,LEYTON SIXTH FORM COLLEGE,Statutory FE Sector - Sixth Form College, +10004108,LORETO COLLEGE,Statutory FE Sector - Sixth Form College, +10004125,LUTON SIXTH FORM COLLEGE,Statutory FE Sector - Sixth Form College, +10004785,NOTRE DAME CATHOLIC SIXTH FORM COLLEGE,Statutory FE Sector - Sixth Form College, +10005072,PETER SYMONDS COLLEGE,Statutory FE Sector - Sixth Form College, +10005687,SCARBOROUGH SIXTH FORM COLLEGE,Statutory FE Sector - Sixth Form College, +10005822,SHREWSBURY COLLEGE,Statutory FE Sector - Sixth Form College, +10005859,SIR GEORGE MONOUX COLLEGE,Statutory FE Sector - Sixth Form College, +10006130,ST BRENDAN'S SIXTH FORM COLLEGE,Statutory FE Sector - Sixth Form College, +10006135,ST CHARLES CATHOLIC SIXTH FORM COLLEGE,Statutory FE Sector - Sixth Form College, +10006148,ST DOMINIC'S SIXTH FORM COLLEGE,Statutory FE Sector - Sixth Form College, +10008007,ST FRANCIS XAVIER SIXTH FORM COLLEGE,Statutory FE Sector - Sixth Form College, +10006195,ST JOHN RIGBY RC SIXTH FORM COLLEGE,Statutory FE Sector - Sixth Form College, +10003011,THE HENLEY COLLEGE,Statutory FE Sector - Sixth Form College, +10007212,VARNDEAN COLLEGE,Statutory FE Sector - Sixth Form College, +10007503,WILBERFORCE COLLEGE,Statutory FE Sector - Sixth Form College, +10007546,WINSTANLEY COLLEGE,Statutory FE Sector - Sixth Form College, +10007671,WQE AND REGENT COLLEGE GROUP,Statutory FE Sector - Sixth Form College, +10007673,WYKE SIXTH FORM COLLEGE,Statutory FE Sector - Sixth Form College, +10007682,XAVERIAN COLLEGE,Statutory FE Sector - Sixth Form College, diff --git a/config/environments/development.rb b/config/environments/development.rb index d51b57fe9c4..0d550297b51 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -74,6 +74,9 @@ # Raises error for missing translations. # config.i18n.raise_on_missing_translations = true + # config.log_level = :info + config.log_level = :debug + # Annotate rendered view with file names. config.action_view.annotate_rendered_view_with_filenames = true diff --git a/config/environments/test.rb b/config/environments/test.rb index ca847154f57..577e1153a8c 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -69,6 +69,7 @@ config.active_record.verbose_query_logs = false config.active_record.query_log_tags_enabled = false config.log_level = :fatal + # config.log_level = :debug # we don't need strict mx validation (checking MX records etc) in test mode config.strict_mx_validation = false diff --git a/db/migrate/20260416154155_add_discarded_to_organisation.rb b/db/migrate/20260416154155_add_discarded_to_organisation.rb new file mode 100644 index 00000000000..e5164f705bf --- /dev/null +++ b/db/migrate/20260416154155_add_discarded_to_organisation.rb @@ -0,0 +1,5 @@ +class AddDiscardedToOrganisation < ActiveRecord::Migration[8.0] + def change + add_column :organisations, :discarded_at, :datetime + end +end diff --git a/db/schema.rb b/db/schema.rb index 3fb91dd0610..26974f19666 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -574,6 +574,7 @@ t.string "safeguarding_information", default: "Our organisation is committed to safeguarding and promoting the welfare of children, young people and vulnerable adults. We expect all staff, volunteers and trustees to share this commitment.\n\nOur recruitment process follows the keeping children safe in education guidance.\n\nOffers of employment may be subject to the following checks (where relevant):\nchildcare disqualification\nDisclosure and Barring Service (DBS)\nmedical\nonline and social media\nprohibition from teaching\nright to work\nsatisfactory references\nsuitability to work with children\n\nYou must tell us about any unspent conviction, cautions, reprimands or warnings under the Rehabilitation of Offenders Act 1974 (Exceptions) Order 1975." t.tsvector "searchable_content" t.geometry "uk_geopoint", limit: {srid: 27700, type: "st_point"} + t.datetime "discarded_at" t.index ["geopoint"], name: "index_organisations_on_geopoint", using: :gist t.index ["local_authority_code"], name: "index_organisations_on_local_authority_code", unique: true t.index ["searchable_content"], name: "index_organisations_on_searchable_content", using: :gin diff --git a/db/seeds.rb b/db/seeds.rb index c9db7ab23ec..2c82678ed35 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -3,7 +3,7 @@ require "faker" require "factory_bot_rails" -Gias::ImportSchoolsAndLocalAuthorities.new.call +Gias::ImportSchoolsAndLocalAuthorities.call Gias::ImportTrusts.new.call ImportPolygonDataJob.perform_now diff --git a/spec/factories/schools.rb b/spec/factories/schools.rb index b3f64051db3..2c6b2eb09b2 100644 --- a/spec/factories/schools.rb +++ b/spec/factories/schools.rb @@ -12,18 +12,18 @@ geopoint { "POINT(-1 51.5)" } gias_data do { - CloseDate: nil, - HeadFirstName: Faker::Name.first_name, - HeadLastName: Faker::Name.last_name.delete("'"), - HeadPreferredJobTitle: Faker::Name.prefix.delete("."), - DateOfLastInspectionVisit: Faker::Date.between(from: 999.days.ago, to: 5.days.ago), - NumberOfPupils: Faker::Number.number(digits: 3), - "OfstedRating (name)": factory_sample(ofsted_ratings), - OpenDate: Faker::Date.between(from: 10_000.days.ago, to: 1000.days.ago), - SchoolCapacity: Faker::Number.number(digits: 4), - TelephoneNum: Faker::Number.number(digits: 11).to_s, - "Trusts (name)": "#{Faker::Company.name.delete("'")} Trust", - "TypeOfEstablishment (code)": "02", + # CloseDate: nil, + # HeadFirstName: Faker::Name.first_name, + # HeadLastName: Faker::Name.last_name.delete("'"), + # HeadPreferredJobTitle: Faker::Name.prefix.delete("."), + # DateOfLastInspectionVisit: Faker::Date.between(from: 999.days.ago, to: 5.days.ago), + # NumberOfPupils: Faker::Number.number(digits: 3), + # "OfstedRating (name)": factory_sample(ofsted_ratings), + # OpenDate: Faker::Date.between(from: 10_000.days.ago, to: 1000.days.ago), + # SchoolCapacity: Faker::Number.number(digits: 4), + # TelephoneNum: Faker::Number.number(digits: 11).to_s, + # "Trusts (name)": "#{Faker::Company.name.delete("'")} Trust", + # "TypeOfEstablishment (code)": "02", } end detailed_school_type { "Voluntary aided school" } diff --git a/spec/fixtures/liverpool_schools.yml b/spec/fixtures/liverpool_schools.yml index cb869063a66..4a8f6af8dae 100644 --- a/spec/fixtures/liverpool_schools.yml +++ b/spec/fixtures/liverpool_schools.yml @@ -51,7 +51,7 @@ value_before_type_cast: '{"URN":"104921","LA (code)":"343","LA (name)":"Sefton","EstablishmentNumber":"3343","EstablishmentName":"St Luke''s Halsall Church of England Primary School","TypeOfEstablishment (code)":"02","TypeOfEstablishment (name)":"Voluntary aided school","EstablishmentTypeGroup (code)":"4","EstablishmentTypeGroup - (name)":"Local authority maintained schools","EstablishmentStatus (code)":"1","EstablishmentStatus + (name)":"Local authority maintained schoolss","EstablishmentStatus (code)":"1","EstablishmentStatus (name)":"Open","ReasonEstablishmentOpened (code)":"00","ReasonEstablishmentOpened (name)":"Not applicable","OpenDate":"","ReasonEstablishmentClosed (code)":"00","ReasonEstablishmentClosed (name)":"Not applicable","CloseDate":"","PhaseOfEducation (code)":"2","PhaseOfEducation diff --git a/spec/jobs/import_organisation_data_job_spec.rb b/spec/jobs/import_organisation_data_job_spec.rb index 8eb82b6f8ec..68c36c8355b 100644 --- a/spec/jobs/import_organisation_data_job_spec.rb +++ b/spec/jobs/import_organisation_data_job_spec.rb @@ -1,14 +1,12 @@ require "rails_helper" RSpec.describe ImportOrganisationDataJob do - let(:import_school_data) { instance_double(Gias::ImportSchoolsAndLocalAuthorities) } let(:import_trust_data) { instance_double(Gias::ImportTrusts) } it "executes the importers" do - expect(Gias::ImportSchoolsAndLocalAuthorities).to receive(:new).and_return(import_school_data) + expect(Gias::ImportSchoolsAndLocalAuthorities).to receive(:call) expect(Gias::ImportTrusts).to receive(:new).and_return(import_trust_data) - expect(import_school_data).to receive(:call) expect(import_trust_data).to receive(:call) described_class.perform_now diff --git a/spec/services/gias/import_schools_and_local_authorities_spec.rb b/spec/services/gias/import_schools_and_local_authorities_spec.rb index 68cf965549a..c0d2d9c1e78 100644 --- a/spec/services/gias/import_schools_and_local_authorities_spec.rb +++ b/spec/services/gias/import_schools_and_local_authorities_spec.rb @@ -1,7 +1,7 @@ require "rails_helper" RSpec.describe Gias::ImportSchoolsAndLocalAuthorities do - subject { described_class.new } + subject { described_class } describe "#call" do let(:csv) { File.read(test_file_path) }