diff --git a/README.md b/README.md index 31e75642..561859c4 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ import csv from jobspy import scrape_jobs jobs = scrape_jobs( - site_name=["indeed", "linkedin", "zip_recruiter", "google"], # "glassdoor", "bayt", "naukri", "bdjobs" + site_name=["indeed", "linkedin", "zip_recruiter", "google"], # "glassdoor", "bayt", "naukri", "bdjobs", "solidjobs" search_term="software engineer", google_search_term="software engineer jobs near San Francisco, CA since yesterday", location="San Francisco, CA", @@ -59,7 +59,7 @@ zip_recruiter Software Developer TEKsystems Phoenix ```plaintext Optional ├── site_name (list|str): -| linkedin, zip_recruiter, indeed, glassdoor, google, bayt, bdjobs +| linkedin, zip_recruiter, indeed, glassdoor, google, bayt, bdjobs, solidjobs | (default is all) │ ├── search_term (str) @@ -173,6 +173,10 @@ You can specify the following countries when searching on Indeed (use the exact Bayt only uses the search_term parameter currently and searches internationally +### **SolidJobs** + +SolidJobs is a Polish job board scraped through its public API (https://solid.jobs/api-ofert-pracy) (Documentation can be found there: https://github.com/solid-company/solid-jobs-client), which needs no API key. The API only supports filtering by category and pagination, so `search_term` (matched against title, company and skills), `location`, `is_remote` and `job_type` are all applied client-side. `job_type` is limited to `fulltime` / `parttime`. All jobs are located in Poland. + ## Notes diff --git a/jobspy/__init__.py b/jobspy/__init__.py index 76129e8b..32497fe2 100644 --- a/jobspy/__init__.py +++ b/jobspy/__init__.py @@ -12,6 +12,7 @@ from jobspy.indeed import Indeed from jobspy.linkedin import LinkedIn from jobspy.naukri import Naukri +from jobspy.solidjobs import SolidJobs from jobspy.model import JobType, Location, JobResponse, Country from jobspy.model import SalarySource, ScraperInput, Site from jobspy.util import ( @@ -64,6 +65,7 @@ def scrape_jobs( Site.BAYT: BaytScraper, Site.NAUKRI: Naukri, Site.BDJOBS: BDJobs, # Add BDJobs to the scraper mapping + Site.SOLIDJOBS: SolidJobs, } set_logger_level(verbose) job_type = get_enum_from_value(job_type) if job_type else None diff --git a/jobspy/exception.py b/jobspy/exception.py index 4fc8578e..2cf6eb8f 100644 --- a/jobspy/exception.py +++ b/jobspy/exception.py @@ -42,4 +42,4 @@ def __init__(self,message=None): class BDJobsException(Exception): def __init__(self, message=None): - super().__init__(message or "An error occurred with BDJobs") \ No newline at end of file + super().__init__(message or "An error occurred with BDJobs") diff --git a/jobspy/model.py b/jobspy/model.py index 9dba8261..10530752 100644 --- a/jobspy/model.py +++ b/jobspy/model.py @@ -293,6 +293,7 @@ class Site(Enum): BAYT = "bayt" NAUKRI = "naukri" BDJOBS = "bdjobs" # Add this line + SOLIDJOBS = "solidjobs" class SalarySource(Enum): diff --git a/jobspy/solidjobs/__init__.py b/jobspy/solidjobs/__init__.py new file mode 100644 index 00000000..212463e9 --- /dev/null +++ b/jobspy/solidjobs/__init__.py @@ -0,0 +1,237 @@ +from __future__ import annotations + +import re +from datetime import datetime + +from jobspy.model import ( + Scraper, + ScraperInput, + Site, + JobPost, + JobResponse, + Location, + Country, + JobType, + Compensation, + CompensationInterval, + DescriptionFormat, +) +from jobspy.solidjobs.constant import ( + api_url, + default_campaign, + divisions, + headers, + max_page_size, +) +from jobspy.util import ( + create_logger, + create_session, + markdown_converter, + plain_converter, +) + +log = create_logger("SolidJobs") + + +class SolidJobs(Scraper): + """Scraper for the Polish job board SOLID.Jobs (https://solid.jobs). + + Uses the public job offers API (https://solid.jobs/api-ofert-pracy) which + requires no authentication. SOLID.Jobs has no free text search endpoint, so + `search_term` is applied client-side against the title, company and skills. + """ + + def __init__( + self, + proxies: list[str] | str | None = None, + ca_cert: str | None = None, + user_agent: str | None = None, + ): + super().__init__( + Site.SOLIDJOBS, proxies=proxies, ca_cert=ca_cert, user_agent=user_agent + ) + self.scraper_input = None + self.session = None + + def scrape(self, scraper_input: ScraperInput) -> JobResponse: + self.scraper_input = scraper_input + self.session = create_session( + proxies=self.proxies, ca_cert=self.ca_cert, is_tls=False, has_retry=True + ) + self.session.headers.update(headers) + if self.user_agent: + self.session.headers["User-Agent"] = self.user_agent + + results_wanted = scraper_input.results_wanted or 15 + job_list: list[JobPost] = [] + seen_keys: set[str] = set() + + for division in divisions: + if len(job_list) >= results_wanted: + break + self._scrape_division(division, results_wanted, job_list, seen_keys) + + return JobResponse(jobs=job_list[:results_wanted]) + + def _scrape_division( + self, + division: str, + results_wanted: int, + job_list: list[JobPost], + seen_keys: set[str], + ) -> None: + page_index = 0 + while len(job_list) < results_wanted: + data = self._fetch_offers(division, page_index) + if not data: + break + offers = data.get("jobs") or [] + if not offers: + break + for offer in offers: + key = offer.get("jobOfferKey") + if not key or key in seen_keys: + continue + seen_keys.add(key) + if not self._matches_filters(offer): + continue + job_post = self._process_offer(offer) + if job_post: + job_list.append(job_post) + if len(job_list) >= results_wanted: + return + if page_index + 1 >= data.get("totalPages", 0): + break + page_index += 1 + + def _fetch_offers(self, division: str, page_index: int) -> dict | None: + # The public API ignores its documented filter params (city, minSalary, + # experienceLevel), so only division + pagination are sent and every + # other filter is applied client-side in _matches_filters. + params = { + "campaign": default_campaign, + "pageSize": max_page_size, + "pageIndex": page_index, + } + try: + response = self.session.get( + f"{api_url}/{division}", + params=params, + timeout=self.scraper_input.request_timeout, + ) + response.raise_for_status() + return response.json() + except Exception as e: + log.error(f"SolidJobs: Error fetching division {division} - {str(e)}") + return None + + def _matches_filters(self, offer: dict) -> bool: + if self.scraper_input.is_remote and not offer.get("isRemote"): + return False + if self.scraper_input.job_type: + offer_job_type = self._map_job_type(offer.get("contractTime")) + if offer_job_type != self.scraper_input.job_type: + return False + location = self.scraper_input.location + if location: + offer_locations = " ".join(offer.get("locations") or []).lower() + if location.lower() not in offer_locations: + return False + search_term = self.scraper_input.search_term + if search_term: + skills = offer.get("skills") or [] + haystack = " ".join( + [ + offer.get("title") or "", + offer.get("company") or "", + " ".join(skill.get("name", "") for skill in skills), + ] + ).lower() + # No free text endpoint, so require every search token to be present. + tokens = search_term.lower().split() + if not all(token in haystack for token in tokens): + return False + return True + + def _process_offer(self, offer: dict) -> JobPost | None: + title = offer.get("title") + job_url = offer.get("url") + if not title or not job_url: + return None + + locations = offer.get("locations") or [] + location = Location( + city=locations[0] if locations else None, + country=Country.POLAND, + ) + + job_type = self._map_job_type(offer.get("contractTime")) + + return JobPost( + id=f"solidjobs-{offer.get('jobOfferKey')}", + title=title, + company_name=offer.get("company"), + location=location, + job_url=job_url, + description=self._format_description(offer.get("description")), + company_logo=offer.get("companyLogoUrl"), + job_type=[job_type] if job_type else None, + compensation=self._map_compensation(offer.get("salary")), + date_posted=self._parse_date(offer.get("validFrom")), + is_remote=offer.get("isRemote"), + job_level=offer.get("experienceLevel"), + skills=[ + skill.get("name") + for skill in offer.get("skills") or [] + if skill.get("name") + ] + or None, + ) + + def _format_description(self, description: str | None) -> str | None: + if not description: + return None + if self.scraper_input.description_format == DescriptionFormat.MARKDOWN: + return markdown_converter(description) + if self.scraper_input.description_format == DescriptionFormat.PLAIN: + return plain_converter(description) + return description + + @staticmethod + def _map_job_type(contract_time: str | None) -> JobType | None: + return { + "full_time": JobType.FULL_TIME, + "part_time": JobType.PART_TIME, + }.get(contract_time) + + @staticmethod + def _map_compensation(salary: dict | None) -> Compensation | None: + if not salary: + return None + interval = ( + CompensationInterval.MONTHLY + if salary.get("period") == "Month" + else None + ) + return Compensation( + interval=interval, + min_amount=salary.get("from"), + max_amount=salary.get("to"), + currency=salary.get("currency") or "PLN", + ) + + @staticmethod + def _parse_date(value: str | None): + if not value: + return None + # The API returns 4-7 fractional-second digits and may use 'Z'; + # datetime.fromisoformat only accepts 3 or 6 digits (and no 'Z') on + # Python < 3.11, so pad/truncate fractional seconds to exactly 6. + value = value.replace("Z", "+00:00") + value = re.sub( + r"\.(\d+)", lambda m: "." + m.group(1)[:6].ljust(6, "0"), value + ) + try: + return datetime.fromisoformat(value).date() + except ValueError: + return None diff --git a/jobspy/solidjobs/constant.py b/jobspy/solidjobs/constant.py new file mode 100644 index 00000000..4a0c7177 --- /dev/null +++ b/jobspy/solidjobs/constant.py @@ -0,0 +1,26 @@ +# Headers for SOLID.Jobs public API requests +headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", + "Accept": "application/json", +} + +# Public API base url (no auth/key required, see https://solid.jobs/api-ofert-pracy) +api_url = "https://solid.jobs/public-api/offers" + +# Campaign identifier required by the public API (lowercase, digits, dashes; max 64 chars) +default_campaign = "jobspy" + +# Job categories (divisions) supported by the API, used as the path segment +divisions = [ + "IT", + "Engineering", + "Marketing", + "Sales", + "HR", + "Logistics", + "Finances", + "Other", +] + +# Max page size accepted by the API +max_page_size = 500 diff --git a/pyproject.toml b/pyproject.toml index 41d30f4c..bf69cbcf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ description = "Job scraper for LinkedIn, Indeed, Glassdoor, ZipRecruiter & Bayt" authors = ["Cullen Watson ", "Zachary Hampton "] homepage = "https://github.com/cullenwatson/JobSpy" readme = "README.md" -keywords = [ "jobs-scraper", "linkedin", "indeed", "glassdoor", "ziprecruiter", "bayt", "naukri"] +keywords = [ "jobs-scraper", "linkedin", "indeed", "glassdoor", "ziprecruiter", "bayt", "naukri", "solidjobs"] [[tool.poetry.packages]] include = "jobspy"