From 25608dd8eed487c462baebc964b06966c3c09124 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20O=C5=BAmina?= Date: Thu, 25 Jun 2026 13:15:22 +0200 Subject: [PATCH 1/2] Add SOLID.Jobs polish job board via its own public API --- README.md | 8 +- jobspy/__init__.py | 2 + jobspy/exception.py | 7 +- jobspy/model.py | 1 + jobspy/solidjobs/__init__.py | 226 +++++++++++++++++++++++++++++++++++ jobspy/solidjobs/constant.py | 27 +++++ pyproject.toml | 2 +- 7 files changed, 269 insertions(+), 4 deletions(-) create mode 100644 jobspy/solidjobs/__init__.py create mode 100644 jobspy/solidjobs/constant.py diff --git a/README.md b/README.md index 31e75642..561859c4 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ import csv from jobspy import scrape_jobs jobs = scrape_jobs( - site_name=["indeed", "linkedin", "zip_recruiter", "google"], # "glassdoor", "bayt", "naukri", "bdjobs" + site_name=["indeed", "linkedin", "zip_recruiter", "google"], # "glassdoor", "bayt", "naukri", "bdjobs", "solidjobs" search_term="software engineer", google_search_term="software engineer jobs near San Francisco, CA since yesterday", location="San Francisco, CA", @@ -59,7 +59,7 @@ zip_recruiter Software Developer TEKsystems Phoenix ```plaintext Optional ├── site_name (list|str): -| linkedin, zip_recruiter, indeed, glassdoor, google, bayt, bdjobs +| linkedin, zip_recruiter, indeed, glassdoor, google, bayt, bdjobs, solidjobs | (default is all) │ ├── search_term (str) @@ -173,6 +173,10 @@ You can specify the following countries when searching on Indeed (use the exact Bayt only uses the search_term parameter currently and searches internationally +### **SolidJobs** + +SolidJobs is a Polish job board scraped through its public API (https://solid.jobs/api-ofert-pracy) (Documentation can be found there: https://github.com/solid-company/solid-jobs-client), which needs no API key. The API only supports filtering by category and pagination, so `search_term` (matched against title, company and skills), `location`, `is_remote` and `job_type` are all applied client-side. `job_type` is limited to `fulltime` / `parttime`. All jobs are located in Poland. + ## Notes diff --git a/jobspy/__init__.py b/jobspy/__init__.py index 76129e8b..32497fe2 100644 --- a/jobspy/__init__.py +++ b/jobspy/__init__.py @@ -12,6 +12,7 @@ from jobspy.indeed import Indeed from jobspy.linkedin import LinkedIn from jobspy.naukri import Naukri +from jobspy.solidjobs import SolidJobs from jobspy.model import JobType, Location, JobResponse, Country from jobspy.model import SalarySource, ScraperInput, Site from jobspy.util import ( @@ -64,6 +65,7 @@ def scrape_jobs( Site.BAYT: BaytScraper, Site.NAUKRI: Naukri, Site.BDJOBS: BDJobs, # Add BDJobs to the scraper mapping + Site.SOLIDJOBS: SolidJobs, } set_logger_level(verbose) job_type = get_enum_from_value(job_type) if job_type else None diff --git a/jobspy/exception.py b/jobspy/exception.py index 4fc8578e..8491dcb2 100644 --- a/jobspy/exception.py +++ b/jobspy/exception.py @@ -42,4 +42,9 @@ def __init__(self,message=None): class BDJobsException(Exception): def __init__(self, message=None): - super().__init__(message or "An error occurred with BDJobs") \ No newline at end of file + super().__init__(message or "An error occurred with BDJobs") + + +class SolidJobsException(Exception): + def __init__(self, message=None): + super().__init__(message or "An error occurred with SolidJobs") \ No newline at end of file diff --git a/jobspy/model.py b/jobspy/model.py index 9dba8261..10530752 100644 --- a/jobspy/model.py +++ b/jobspy/model.py @@ -293,6 +293,7 @@ class Site(Enum): BAYT = "bayt" NAUKRI = "naukri" BDJOBS = "bdjobs" # Add this line + SOLIDJOBS = "solidjobs" class SalarySource(Enum): diff --git a/jobspy/solidjobs/__init__.py b/jobspy/solidjobs/__init__.py new file mode 100644 index 00000000..ecf9dd6e --- /dev/null +++ b/jobspy/solidjobs/__init__.py @@ -0,0 +1,226 @@ +from __future__ import annotations + +import re +from datetime import datetime + +from jobspy.model import ( + Scraper, + ScraperInput, + Site, + JobPost, + JobResponse, + Location, + Country, + JobType, + Compensation, + CompensationInterval, + DescriptionFormat, +) +from jobspy.solidjobs.constant import ( + api_url, + default_campaign, + divisions, + headers, + max_page_size, +) +from jobspy.util import ( + create_logger, + create_session, + markdown_converter, + plain_converter, +) + +log = create_logger("SolidJobs") + + +class SolidJobs(Scraper): + """Scraper for the Polish job board SOLID.Jobs (https://solid.jobs). + + Uses the public job offers API (https://solid.jobs/api-ofert-pracy) which + requires no authentication. SOLID.Jobs has no free text search endpoint, so + `search_term` is applied client-side against the title, company and skills. + """ + + def __init__( + self, + proxies: list[str] | str | None = None, + ca_cert: str | None = None, + user_agent: str | None = None, + ): + super().__init__(Site.SOLIDJOBS, proxies=proxies, ca_cert=ca_cert) + self.scraper_input = None + self.session = None + + def scrape(self, scraper_input: ScraperInput) -> JobResponse: + self.scraper_input = scraper_input + self.session = create_session( + proxies=self.proxies, ca_cert=self.ca_cert, is_tls=False, has_retry=True + ) + self.session.headers.update(headers) + + results_wanted = scraper_input.results_wanted or 15 + job_list: list[JobPost] = [] + seen_keys: set[str] = set() + + for division in divisions: + if len(job_list) >= results_wanted: + break + self._scrape_division(division, results_wanted, job_list, seen_keys) + + return JobResponse(jobs=job_list[:results_wanted]) + + def _scrape_division( + self, + division: str, + results_wanted: int, + job_list: list[JobPost], + seen_keys: set[str], + ) -> None: + page_index = 0 + while len(job_list) < results_wanted: + data = self._fetch_offers(division, page_index) + if not data: + break + offers = data.get("jobs") or [] + if not offers: + break + for offer in offers: + key = offer.get("jobOfferKey") + if key in seen_keys: + continue + seen_keys.add(key) + if not self._matches_filters(offer): + continue + job_post = self._process_offer(offer) + if job_post: + job_list.append(job_post) + if len(job_list) >= results_wanted: + return + if page_index + 1 >= data.get("totalPages", 0): + break + page_index += 1 + + def _fetch_offers(self, division: str, page_index: int) -> dict | None: + # The public API ignores its documented filter params (city, minSalary, + # experienceLevel), so only division + pagination are sent and every + # other filter is applied client-side in _matches_filters. + params = { + "campaign": default_campaign, + "pageSize": max_page_size, + "pageIndex": page_index, + } + try: + response = self.session.get( + f"{api_url}/{division}", + params=params, + timeout=self.scraper_input.request_timeout, + ) + response.raise_for_status() + return response.json() + except Exception as e: + log.error(f"SolidJobs: Error fetching division {division} - {str(e)}") + return None + + def _matches_filters(self, offer: dict) -> bool: + if self.scraper_input.is_remote and not offer.get("isRemote"): + return False + if self.scraper_input.job_type: + if self._map_job_type(offer.get("contractTime")) != self.scraper_input.job_type: + return False + location = self.scraper_input.location + if location: + offer_locations = " ".join(offer.get("locations") or []).lower() + if location.lower() not in offer_locations: + return False + search_term = self.scraper_input.search_term + if search_term: + haystack = " ".join( + [ + offer.get("title") or "", + offer.get("company") or "", + " ".join(skill.get("name", "") for skill in offer.get("skills") or []), + ] + ).lower() + # No free text endpoint, so require every search token to be present. + if not all(token in haystack for token in search_term.lower().split()): + return False + return True + + def _process_offer(self, offer: dict) -> JobPost | None: + title = offer.get("title") + job_url = offer.get("url") + if not title or not job_url: + return None + + locations = offer.get("locations") or [] + location = Location( + city=locations[0] if locations else None, + country=Country.POLAND, + ) + + job_type = self._map_job_type(offer.get("contractTime")) + + return JobPost( + id=f"solidjobs-{offer.get('jobOfferKey')}", + title=title, + company_name=offer.get("company"), + location=location, + job_url=job_url, + description=self._format_description(offer.get("description")), + company_logo=offer.get("companyLogoUrl"), + job_type=[job_type] if job_type else None, + compensation=self._map_compensation(offer.get("salary")), + date_posted=self._parse_date(offer.get("validFrom")), + is_remote=offer.get("isRemote"), + job_level=offer.get("experienceLevel"), + skills=[skill.get("name") for skill in offer.get("skills") or [] if skill.get("name")] + or None, + ) + + def _format_description(self, description: str | None) -> str | None: + if not description: + return None + if self.scraper_input.description_format == DescriptionFormat.MARKDOWN: + return markdown_converter(description) + if self.scraper_input.description_format == DescriptionFormat.PLAIN: + return plain_converter(description) + return description + + @staticmethod + def _map_job_type(contract_time: str | None) -> JobType | None: + return { + "full_time": JobType.FULL_TIME, + "part_time": JobType.PART_TIME, + }.get(contract_time) + + @staticmethod + def _map_compensation(salary: dict | None) -> Compensation | None: + if not salary: + return None + interval = ( + CompensationInterval.MONTHLY + if salary.get("period") == "Month" + else None + ) + return Compensation( + interval=interval, + min_amount=salary.get("from"), + max_amount=salary.get("to"), + currency=salary.get("currency", "PLN"), + ) + + @staticmethod + def _parse_date(value: str | None): + if not value: + return None + # The API returns 4-7 fractional-second digits and may use 'Z'; + # datetime.fromisoformat only accepts 3 or 6 digits (and no 'Z') on + # Python < 3.11, so pad/truncate fractional seconds to exactly 6. + value = value.replace("Z", "+00:00") + value = re.sub( + r"\.(\d+)", lambda m: "." + m.group(1)[:6].ljust(6, "0"), value + ) + try: + return datetime.fromisoformat(value).date() + except ValueError: + return None diff --git a/jobspy/solidjobs/constant.py b/jobspy/solidjobs/constant.py new file mode 100644 index 00000000..6f37ef4f --- /dev/null +++ b/jobspy/solidjobs/constant.py @@ -0,0 +1,27 @@ +#constant.py +# Headers for SOLID.Jobs public API requests +headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", + "Accept": "application/json", +} + +# Public API base url (no auth/key required, see https://solid.jobs/api-ofert-pracy) +api_url = "https://solid.jobs/public-api/offers" + +# Campaign identifier required by the public API (lowercase, digits, dashes; max 64 chars) +default_campaign = "jobspy" + +# Job categories (divisions) supported by the API, used as the path segment +divisions = [ + "IT", + "Engineering", + "Marketing", + "Sales", + "HR", + "Logistics", + "Finances", + "Other", +] + +# Max page size accepted by the API +max_page_size = 500 diff --git a/pyproject.toml b/pyproject.toml index 41d30f4c..bf69cbcf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ description = "Job scraper for LinkedIn, Indeed, Glassdoor, ZipRecruiter & Bayt" authors = ["Cullen Watson ", "Zachary Hampton "] homepage = "https://github.com/cullenwatson/JobSpy" readme = "README.md" -keywords = [ "jobs-scraper", "linkedin", "indeed", "glassdoor", "ziprecruiter", "bayt", "naukri"] +keywords = [ "jobs-scraper", "linkedin", "indeed", "glassdoor", "ziprecruiter", "bayt", "naukri", "solidjobs"] [[tool.poetry.packages]] include = "jobspy" From 61e6937bef8b8292e6548cc76a1ff712339160c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20O=C5=BAmina?= Date: Thu, 25 Jun 2026 13:52:00 +0200 Subject: [PATCH 2/2] Fix minor bugs, refactor of code --- jobspy/exception.py | 5 ----- jobspy/solidjobs/__init__.py | 25 ++++++++++++++++++------- jobspy/solidjobs/constant.py | 1 - 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/jobspy/exception.py b/jobspy/exception.py index 8491dcb2..2cf6eb8f 100644 --- a/jobspy/exception.py +++ b/jobspy/exception.py @@ -43,8 +43,3 @@ def __init__(self,message=None): class BDJobsException(Exception): def __init__(self, message=None): super().__init__(message or "An error occurred with BDJobs") - - -class SolidJobsException(Exception): - def __init__(self, message=None): - super().__init__(message or "An error occurred with SolidJobs") \ No newline at end of file diff --git a/jobspy/solidjobs/__init__.py b/jobspy/solidjobs/__init__.py index ecf9dd6e..212463e9 100644 --- a/jobspy/solidjobs/__init__.py +++ b/jobspy/solidjobs/__init__.py @@ -47,7 +47,9 @@ def __init__( ca_cert: str | None = None, user_agent: str | None = None, ): - super().__init__(Site.SOLIDJOBS, proxies=proxies, ca_cert=ca_cert) + super().__init__( + Site.SOLIDJOBS, proxies=proxies, ca_cert=ca_cert, user_agent=user_agent + ) self.scraper_input = None self.session = None @@ -57,6 +59,8 @@ def scrape(self, scraper_input: ScraperInput) -> JobResponse: proxies=self.proxies, ca_cert=self.ca_cert, is_tls=False, has_retry=True ) self.session.headers.update(headers) + if self.user_agent: + self.session.headers["User-Agent"] = self.user_agent results_wanted = scraper_input.results_wanted or 15 job_list: list[JobPost] = [] @@ -86,7 +90,7 @@ def _scrape_division( break for offer in offers: key = offer.get("jobOfferKey") - if key in seen_keys: + if not key or key in seen_keys: continue seen_keys.add(key) if not self._matches_filters(offer): @@ -125,7 +129,8 @@ def _matches_filters(self, offer: dict) -> bool: if self.scraper_input.is_remote and not offer.get("isRemote"): return False if self.scraper_input.job_type: - if self._map_job_type(offer.get("contractTime")) != self.scraper_input.job_type: + offer_job_type = self._map_job_type(offer.get("contractTime")) + if offer_job_type != self.scraper_input.job_type: return False location = self.scraper_input.location if location: @@ -134,15 +139,17 @@ def _matches_filters(self, offer: dict) -> bool: return False search_term = self.scraper_input.search_term if search_term: + skills = offer.get("skills") or [] haystack = " ".join( [ offer.get("title") or "", offer.get("company") or "", - " ".join(skill.get("name", "") for skill in offer.get("skills") or []), + " ".join(skill.get("name", "") for skill in skills), ] ).lower() # No free text endpoint, so require every search token to be present. - if not all(token in haystack for token in search_term.lower().split()): + tokens = search_term.lower().split() + if not all(token in haystack for token in tokens): return False return True @@ -173,7 +180,11 @@ def _process_offer(self, offer: dict) -> JobPost | None: date_posted=self._parse_date(offer.get("validFrom")), is_remote=offer.get("isRemote"), job_level=offer.get("experienceLevel"), - skills=[skill.get("name") for skill in offer.get("skills") or [] if skill.get("name")] + skills=[ + skill.get("name") + for skill in offer.get("skills") or [] + if skill.get("name") + ] or None, ) @@ -206,7 +217,7 @@ def _map_compensation(salary: dict | None) -> Compensation | None: interval=interval, min_amount=salary.get("from"), max_amount=salary.get("to"), - currency=salary.get("currency", "PLN"), + currency=salary.get("currency") or "PLN", ) @staticmethod diff --git a/jobspy/solidjobs/constant.py b/jobspy/solidjobs/constant.py index 6f37ef4f..4a0c7177 100644 --- a/jobspy/solidjobs/constant.py +++ b/jobspy/solidjobs/constant.py @@ -1,4 +1,3 @@ -#constant.py # Headers for SOLID.Jobs public API requests headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",