diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 6c6db3fe6..6110ceb5a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,11 +39,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v5 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -54,7 +54,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v4 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -68,4 +68,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index f71000bae..fe87820e6 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -34,9 +34,10 @@ jobs: - name: Install dependencies run: | python${{ matrix.python-version }} -m pip install --upgrade "pip<25.3" build pip-tools pre-commit - python${{ matrix.python-version }} -m piptools compile --extra dev -o requirements.txt mpcontribs-client/setup.py mpcontribs-api/setup.py + python${{ matrix.python-version }} -m piptools compile --extra dev -o requirements.txt mpcontribs-client/pyproject.toml mpcontribs-api/pyproject.toml python${{ matrix.python-version }} -m pip install -r requirements.txt - cd mpcontribs-api && python${{ matrix.python-version }} -m pip install --no-deps . + cd mpcontribs-api + python${{ matrix.python-version }} -m pip install --no-deps . - name: Set SSL_CERT_FILE (Linux) if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' run: | @@ -50,7 +51,6 @@ jobs: - name: Run pre-commit run: | - python${{ matrix.python-version }} -m pip install pre-commit pre-commit install pre-commit run --all-files @@ -61,9 +61,10 @@ jobs: shell: bash run: | cd mpcontribs-client + python${{ matrix.python-version }} -m pip install --no-deps . python${{ matrix.python-version }} -m flake8 --max-line-length 100 python${{ matrix.python-version }} -m pycodestyle --max-line-length 100 . - python${{ matrix.python-version }} -m pytest -v -s --cov=mpcontribs/client --cov-report=term-missing --cov-report=xml --ignore=bravado + python${{ matrix.python-version }} -m pytest -n auto -v -s --cov=mpcontribs/client --cov-report=term-missing --cov-report=xml --ignore=bravado python${{ matrix.python-version }} -m build --outdir ../dist - name: Install lux and test with pytest diff --git a/.gitignore b/.gitignore index af8e9312a..249cd86aa 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ mpcontribs-api/mpcontribs/api/swagger*/ .ipython/ .jupyter/ .local/ +.vscode/ mpcontribs-portal/mpcontribs/portal/templates/notebooks coverage.xml mpcontribs-api/supervisord.conf diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3f37230e8..50a0eb904 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,12 +28,6 @@ repos: files: cloudformation/.*\.(json|yml|yaml)$ args: ["--ignore-checks=E3030,E3001,E3002,E3012"] - #- repo: https://gitlab.com/pycqa/flake8 - # rev: 3.7.9 - # hooks: - # - id: flake8 - # args: [--max-line-length=99] - - repo: https://github.com/psf/black rev: 25.9.0 hooks: @@ -47,3 +41,8 @@ repos: - --drop-empty-cells - --strip-init-cells - --extra-keys=metadata.kernelspec + + - repo: https://github.com/asottile/pyupgrade + rev: v3.15.1 + hooks: + - id: pyupgrade diff --git a/mpcontribs-api/Dockerfile b/mpcontribs-api/Dockerfile index 5986dfd7d..10513c25c 100644 --- a/mpcontribs-api/Dockerfile +++ b/mpcontribs-api/Dockerfile @@ -7,7 +7,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends gcc git g++ lib ENV PIP_FLAGS "--no-cache-dir --compile" COPY requirements/deployment.txt ./requirements.txt RUN pip install $PIP_FLAGS -r requirements.txt -COPY setup.py . +COPY pyproject.toml . COPY mpcontribs mpcontribs RUN pip install $PIP_FLAGS --no-deps . #ENV SETUPTOOLS_SCM_PRETEND_VERSION 0.0.0 diff --git a/mpcontribs-api/mpcontribs/api/__init__.py b/mpcontribs-api/mpcontribs/api/__init__.py index 515d2b43f..e61583b4b 100644 --- a/mpcontribs-api/mpcontribs/api/__init__.py +++ b/mpcontribs-api/mpcontribs/api/__init__.py @@ -8,6 +8,7 @@ from email.message import EmailMessage from importlib import import_module +from importlib.metadata import version from websocket import create_connection from flask import Flask, current_app, request, jsonify from flask_marshmallow import Marshmallow @@ -26,6 +27,11 @@ from notebook.gateway.managers import GatewayClient from requests.exceptions import ConnectionError, Timeout +try: + __version__ = version("mpcontribs-api") +except Exception: + # package is not installed + pass delimiter, max_depth = ".", 7 # = MAX_NESTING + 2 from client invalidChars = set(punctuation.replace("*", "").replace("|", "") + whitespace) diff --git a/mpcontribs-api/mpcontribs/api/config.py b/mpcontribs-api/mpcontribs/api/config.py index 117d8ffc9..17732a602 100644 --- a/mpcontribs-api/mpcontribs/api/config.py +++ b/mpcontribs-api/mpcontribs/api/config.py @@ -2,11 +2,10 @@ """configuration module for MPContribs Flask API""" import os -import datetime import json import gzip -from semantic_version import Version +from mpcontribs.api import __version__ formulae_path = os.path.join( os.path.dirname(__file__), "contributions", "formulae.json.gz" @@ -15,12 +14,6 @@ with gzip.open(formulae_path) as f: FORMULAE = json.load(f) -now = datetime.datetime.now() -VERSION = str(Version( - major=now.year, minor=now.month, patch=now.day, - prerelease=(str(now.hour), str(now.minute)) -)) - JSON_ADD_STATUS = False SECRET_KEY = "super-secret" # TODO in local prod config @@ -65,7 +58,7 @@ "title": "MPContribs API", "description": "Operations to contribute, update and retrieve materials data on Materials Project", "termsOfService": "https://materialsproject.org/terms", - "version": VERSION, + "version": __version__, "contact": { "name": "MPContribs", "email": "contribs@materialsproject.org", diff --git a/mpcontribs-api/pyproject.toml b/mpcontribs-api/pyproject.toml new file mode 100644 index 000000000..c9f6abbb0 --- /dev/null +++ b/mpcontribs-api/pyproject.toml @@ -0,0 +1,103 @@ +[build-system] +requires = ["setuptools>=80.0.0","setuptools-scm>=8"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +root = ".." +relative_to = "__file__" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +where = ["."] +exclude = ["scripts","supervisord"] +include = ["mpcontribs.api"] + +[project] +name = "mpcontribs-api" +dynamic = ["version"] +requires-python = ">=3.11" +description="API for community-contributed Materials Project data" +license = {text = "MIT"} +authors = [ + {name = "Patrick Huck", email = "phuck@lbl.gov"}, + {name = "The Materials Project", email="feedback@materialsproject.org"}, +] +dependencies = [ + "numpy", + "apispec<6", + "asn1crypto", + "blinker", + "boltons", + "css-html-js-minify", + "dateparser", + "ddtrace", + "dnspython", + "filetype", + "flasgger-tschaume>=0.9.7", + "flask-compress", + "flask-marshmallow", + "flask-mongorest-mpcontribs>=3.2.1", + "Flask-RQ2", + "gunicorn[gevent]", + "jinja2", + "json2html", + "marshmallow<4", + "more-itertools", + "nbformat", + "notebook<7", + "pint>=0.24", + "psycopg2-binary", + "pymatgen", + "pyopenssl", + "python-snappy", + "rq<=2.3.2", # see https://github.com/rq/Flask-RQ2/issues/620 + "supervisor", + "setproctitle", + "uncertainties", + "websocket_client", + "zstandard", +] + +[project.urls] +Homepage = "https://github.com/materialsproject/MPContribs" +Documentation = "https://docs.materialsproject.org/services/mpcontribs" + +[project.optional-dependencies] +dev = [ + "flake8", + "pytest", + "pytest-flake8", + "pytest-pycodestyle", + "pytest-xdist", +] +all = [ + "mpcontribs-api[dev]" +] + +[tool.pytest] +markers = [ + "base: basic resource testing", + "extra: all extra views", +] + +[tool.pycodestyle] +count = true +ignore = ["E121","E123","E126","E133","E226","E241","E242","E704","W503","W504","W505","E741","W605"] +max-line-length = 120 +statistics = true +exclude = ["flasgger","flask-mongorest"] + +[tool.flake8] +exclude = [".git","__pycache__","tests","flasgger","flask-mongorest"] +extend-ignore = ["E741",] +max-line-length = 120 + +[tool.pydocstyle] +ignore = ["D105","D2","D4"] + +[tool.mypy] +ignore_missing_imports = true +namespace_packages = true +python_version = 3.11 diff --git a/mpcontribs-api/setup.cfg b/mpcontribs-api/setup.cfg deleted file mode 100644 index 4f3ee2f11..000000000 --- a/mpcontribs-api/setup.cfg +++ /dev/null @@ -1,25 +0,0 @@ -[tool:pytest] -markers = - base: basic resource testing - extra: all extra views - -[pycodestyle] -count = True -ignore = E121,E123,E126,E133,E226,E241,E242,E704,W503,W504,W505,E741,W605 -max-line-length = 120 -statistics = True -exclude = flasgger,flask-mongorest - -[flake8] -exclude = .git,__pycache__,tests,flasgger,flask-mongorest -# max-complexity = 10 -extend-ignore = E741 -max-line-length = 120 - -[pydocstyle] -ignore = D105,D2,D4 - -[mypy] -ignore_missing_imports = True -namespace_packages = True -python_version = 3.7 diff --git a/mpcontribs-api/setup.py b/mpcontribs-api/setup.py deleted file mode 100644 index b343d13be..000000000 --- a/mpcontribs-api/setup.py +++ /dev/null @@ -1,52 +0,0 @@ -# -*- coding: utf-8 -*- -import datetime -from setuptools import setup - -setup( - name="mpcontribs-api", - version=datetime.datetime.today().strftime("%Y.%m.%d"), - description="API for community-contributed Materials Project data", - author="Patrick Huck", - author_email="phuck@lbl.gov", - url="https://mpcontribs.org", - packages=["mpcontribs.api"], - install_requires=[ - "numpy", - "apispec<6", - "asn1crypto", - "blinker", - "boltons", - "css-html-js-minify", - "dateparser", - "ddtrace", - "dnspython", - "filetype", - "flasgger-tschaume>=0.9.7", - "flask-compress", - "flask-marshmallow", - "flask-mongorest-mpcontribs>=3.2.1", - "Flask-RQ2", - "gunicorn[gevent]", - "jinja2", - "json2html", - "marshmallow<4", - "more-itertools", - "nbformat", - "notebook<7", - "pint>=0.24", - "psycopg2-binary", - "pymatgen", - "pyopenssl", - "python-snappy", - "rq<=2.3.2", # see https://github.com/rq/Flask-RQ2/issues/620 - "semantic-version", - "supervisor", - "setproctitle", - "uncertainties", - "websocket_client", - "zstandard", - ], - extras_require={"dev": ["pytest", "flake8"]}, - license="MIT", - zip_safe=False, -) diff --git a/mpcontribs-client/mpcontribs/client/__init__.py b/mpcontribs-client/mpcontribs/client/__init__.py index 16eb8d7b2..8b8cfcdaa 100644 --- a/mpcontribs-client/mpcontribs/client/__init__.py +++ b/mpcontribs-client/mpcontribs/client/__init__.py @@ -1,5 +1,5 @@ -# -*- coding: utf-8 -*- import io +import importlib.metadata import sys import os import ujson @@ -13,14 +13,12 @@ import functools import requests import logging -import datetime from inspect import getfullargspec from math import isclose -from semantic_version import Version from requests.exceptions import RequestException from bson.objectid import ObjectId -from typing import Union, Type, Optional +from typing import Type from tqdm.auto import tqdm from hashlib import md5 from pathlib import Path @@ -59,7 +57,12 @@ from plotly.express._chart_types import line as line_chart from cachetools import cached, LRUCache from cachetools.keys import hashkey -from pymatgen.core import SETTINGS + +try: + __version__ = importlib.metadata.version("mpcontribs-client") +except Exception: + # package is not installed + pass RETRIES = 3 MAX_WORKERS = 3 @@ -88,6 +91,7 @@ SUPPORTED_FILETYPES = (Gz, Jpeg, Png, Gif, Tiff) SUPPORTED_MIMES = [t().mime for t in SUPPORTED_FILETYPES] DEFAULT_DOWNLOAD_DIR = Path.home() / "mpcontribs-downloads" +VALID_API_KEY_ALIASES = ["MPCONTRIBS_API_KEY", "MP_API_KEY", "PMG_MAPI_KEY"] j2h = Json2Html() pd.options.plotting.backend = "plotly" @@ -126,7 +130,7 @@ class LogFilter(logging.Filter): def __init__(self, level, *args, **kwargs): self.level = level - super(LogFilter, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def filter(self, record): return record.levelno < self.level @@ -144,7 +148,7 @@ class TqdmToLogger(io.StringIO): buf = "" def __init__(self, logger, level=None): - super(TqdmToLogger, self).__init__() + super().__init__() self.logger = logger self.level = level or logging.INFO @@ -177,7 +181,7 @@ class MPContribsClientError(ValueError): """custom error for mpcontribs-client""" -def get_md5(d): +def get_md5(d) -> str: s = ujson.dumps(d, sort_keys=True).encode("utf-8") return md5(s).hexdigest() @@ -478,7 +482,7 @@ def unpack(self) -> str: return unpacked - def write(self, outdir: Optional[Union[str, Path]] = None) -> Path: + def write(self, outdir: str | Path | None = None) -> Path: """Write attachment to file using its name Args: @@ -490,7 +494,7 @@ def write(self, outdir: Optional[Union[str, Path]] = None) -> Path: path.write_bytes(content) return path - def display(self, outdir: Optional[Union[str, Path]] = None): + def display(self, outdir: str | Path | None = None): """Display Image/FileLink for attachment if in IPython/Jupyter Args: @@ -519,7 +523,7 @@ def name(self) -> str: return self["name"] @classmethod - def from_data(cls, data: Union[list, dict], name: str = "attachment"): + def from_data(cls, data: list | dict, name: str = "attachment"): """Construct attachment from data dict or list Args: @@ -539,7 +543,7 @@ def from_data(cls, data: Union[list, dict], name: str = "attachment"): ) @classmethod - def from_file(cls, path: Union[Path, str]): + def from_file(cls, path: str | Path): """Construct attachment from file Args: @@ -616,7 +620,7 @@ def from_list(cls, elements: list): return attachments @classmethod - def from_data(cls, data: Union[list, dict], prefix: str = "attachment"): + def from_data(cls, data: list | dict, prefix: str = "attachment"): """Construct list of attachments from data dict or list Args: @@ -830,32 +834,24 @@ def _expand_params(protocol, host, version, projects_json, apikey=None): def _version(url): retries, max_retries = 0, 3 protocol = urlparse(url).scheme - is_mock_test = "pytest" in sys.modules and protocol == "http" - - if is_mock_test: - now = datetime.datetime.now() - return Version( - major=now.year, - minor=now.month, - patch=now.day, - prerelease=(str(now.hour), str(now.minute)), - ) - else: - while retries < max_retries: - try: - r = requests.get(f"{url}/healthcheck", timeout=5) - if r.status_code in {200, 403}: - return r.json().get("version") - else: - retries += 1 - logger.warning( - f"Healthcheck for {url} failed ({r.status_code})! Wait 30s." - ) - time.sleep(30) - except RequestException as ex: + if "pytest" in sys.modules and protocol == "http": + return __version__ + + while retries < max_retries: + try: + r = requests.get(f"{url}/healthcheck", timeout=5) + if r.status_code in {200, 403}: + return r.json().get("version") + else: retries += 1 - logger.warning(f"Could not connect to {url} ({ex})! Wait 30s.") + logger.warning( + f"Healthcheck for {url} failed ({r.status_code})! Wait 30s." + ) time.sleep(30) + except RequestException as ex: + retries += 1 + logger.warning(f"Could not connect to {url} ({ex})! Wait 30s.") + time.sleep(30) class Client(SwaggerClient): @@ -870,11 +866,11 @@ class Client(SwaggerClient): def __init__( self, - apikey: Optional[str] = None, - headers: Optional[dict] = None, - host: Optional[str] = None, - project: Optional[str] = None, - session: Optional[requests.Session] = None, + apikey: str | None = None, + headers: dict | None = None, + host: str | None = None, + project: str | None = None, + session: requests.Session | None = None, ): """Initialize the client - only reloads API spec from server as needed @@ -892,7 +888,17 @@ def __init__( host = os.environ.get("MPCONTRIBS_API_HOST", DEFAULT_HOST) if not apikey: - apikey = os.environ.get("MPCONTRIBS_API_KEY", SETTINGS.get("PMG_MAPI_KEY")) + try: + apikey = next( + os.environ.get(kalias) + for kalias in VALID_API_KEY_ALIASES + if kalias is not None + ) + except StopIteration: + from pymatgen.core import SETTINGS + + apikey = SETTINGS.get("PMG_MAPI_KEY") + if apikey and len(apikey) != 32: raise MPContribsClientError(f"Invalid API key: {apikey}") @@ -925,7 +931,7 @@ def __init__( def __enter__(self): return self - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__(self, exc_type, exc_val, exc_tb) -> None: return None @property @@ -934,17 +940,18 @@ def cached_swagger_spec(self): self.protocol, self.host, self.headers_json, self.project, self.version ) - def __dir__(self): + def __dir__(self) -> set[str]: members = set(self.swagger_spec.resources.keys()) - members |= set(k for k in self.__dict__.keys() if not k.startswith("_")) - members |= set(k for k in dir(self.__class__) if not k.startswith("_")) + members |= {k for k in self.__dict__.keys() if not k.startswith("_")} + members |= {k for k in dir(self.__class__) if not k.startswith("_")} return members def _reinit(self): _load.cache_clear() super().__init__(self.cached_swagger_spec) - def _is_valid_payload(self, model: str, data: dict): + def _is_valid_payload(self, model: str, data: dict) -> None: + """Raise an error if a payload is invalid.""" model_spec = deepcopy(self.get_model(f"{model}sSchema")._model_spec) model_spec.pop("required") model_spec["additionalProperties"] = False @@ -952,17 +959,20 @@ def _is_valid_payload(self, model: str, data: dict): try: validate_object(self.swagger_spec, model_spec, data) except ValidationError as ex: - return False, str(ex) - - return True, None - - def _is_serializable_dict(self, dct): - for k, v in flatten(dct, reducer="dot").items(): - if v is not None and not isinstance(v, (str, int, float)): - error = f"Value {v} of {type(v)} for key {k} not supported." - return False, error + raise MPContribsClientError(str(ex)) - return True, None + def _is_serializable_dict(self, dct: dict) -> None: + """Raise an error if an input dict is not JSON serializable.""" + try: + raise MPContribsClientError( + next( + f"Value {v} of {type(v)} for key {k} not supported." + for k, v in flatten(dct, reducer="dot").items() + if v is not None and not isinstance(v, (str, int, float)) + ) + ) + except StopIteration: + pass def _get_per_page_default_max( self, op: str = "query", resource: str = "contributions" @@ -1030,14 +1040,10 @@ def _split_query( for q in queries: # copy over missing parameters - for k, v in query.items(): - if k not in q: - q[k] = v + q.update({k: v for k, v in query.items() if k not in q}) # comma-separated lists - for k, v in q.items(): - if isinstance(v, list): - q[k] = ",".join(v) + q.update({k: ",".join(v) for k, v in q.items() if isinstance(v, list)}) return queries @@ -1047,7 +1053,7 @@ def _get_future( params: dict, rel_url: str = "contributions", op: str = "query", - data: Optional[dict] = None, + data: dict | None = None, ): rname = rel_url.split("/", 1)[0] resource = self.swagger_spec.resources[rname] @@ -1066,7 +1072,7 @@ def _get_future( def available_query_params( self, - startswith: Optional[tuple] = None, + startswith: tuple | None = None, resource: str = "contributions", ) -> list: resources = self.swagger_spec.resources @@ -1083,9 +1089,7 @@ def available_query_params( return [param for param in params if param.startswith(startswith)] - def get_project( - self, name: Optional[str] = None, fields: Optional[list] = None - ) -> Dict: + def get_project(self, name: str | None = None, fields: list | None = None) -> Dict: """Retrieve a project entry Args: @@ -1103,10 +1107,10 @@ def get_project( def query_projects( self, - query: Optional[dict] = None, - term: Optional[str] = None, - fields: Optional[list] = None, - sort: Optional[str] = None, + query: dict | None = None, + term: str | None = None, + fields: list | None = None, + sort: str | None = None, timeout: int = -1, ) -> list[dict]: """Query projects by query and/or term (Atlas Search) @@ -1157,9 +1161,13 @@ def search_future(search_term): if total_pages < 2: return ret["data"] - for field in ["name__in", "_fields"]: - if field in query: - query[field] = ",".join(query[field]) + query.update( + { + field: ",".join(query[field]) + for field in ["name__in", "_fields"] + if field in query + } + ) queries = [] @@ -1172,8 +1180,7 @@ def search_future(search_term): ] responses = _run_futures(futures, total=total_count, timeout=timeout) - for resp in responses.values(): - ret["data"] += resp["result"]["data"] + ret["data"].extend([resp["result"]["data"] for resp in responses.values()]) return ret["data"] @@ -1210,7 +1217,7 @@ def create_project( else: raise MPContribsClientError(resp) - def update_project(self, update: dict, name: Optional[str] = None): + def update_project(self, update: dict, name: str | None = None): """Update project info Args: @@ -1268,15 +1275,12 @@ def update_project(self, update: dict, name: Optional[str] = None): logger.warning("nothing to update") return - valid, error = self._is_valid_payload("Project", payload) - if valid: - resp = self.projects.updateProjectByName(pk=name, project=payload).result() - if not resp.get("count", 0): - raise MPContribsClientError(resp) - else: - raise MPContribsClientError(error) + self._is_valid_payload("Project", payload) + resp = self.projects.updateProjectByName(pk=name, project=payload).result() + if not resp.get("count", 0): + raise MPContribsClientError(resp) - def delete_project(self, name: Optional[str] = None): + def delete_project(self, name: str | None = None): """Delete a project Args: @@ -1295,7 +1299,7 @@ def delete_project(self, name: Optional[str] = None): if resp and "error" in resp: raise MPContribsClientError(resp["error"]) - def get_contribution(self, cid: str, fields: Optional[list] = None) -> Dict: + def get_contribution(self, cid: str, fields: list | None = None) -> Dict: """Retrieve a contribution Args: @@ -1408,7 +1412,7 @@ def get_attachment(self, aid_or_md5: str) -> Attachment: ) def init_columns( - self, columns: Optional[dict] = None, name: Optional[str] = None + self, columns: dict | None = None, name: str | None = None ) -> dict: """initialize columns for a project to set their order and desired units @@ -1532,20 +1536,23 @@ def init_columns( new_unit = new_column.get("unit", "NaN") existing_unit = existing_column.get("unit") if existing_unit != new_unit: - conv_args = [] - for u in [existing_unit, new_unit]: + if existing_unit == "NaN" and new_unit == "": + factor = 1 + else: + conv_args = [] + for u in [existing_unit, new_unit]: + try: + conv_args.append(ureg.Unit(u)) + except ValueError: + raise MPContribsClientError( + f"Can't convert {existing_unit} to {new_unit} for {path}" + ) try: - conv_args.append(ureg.Unit(u)) - except ValueError: + factor = ureg.convert(1, *conv_args) + except DimensionalityError: raise MPContribsClientError( f"Can't convert {existing_unit} to {new_unit} for {path}" ) - try: - factor = ureg.convert(1, *conv_args) - except DimensionalityError: - raise MPContribsClientError( - f"Can't convert {existing_unit} to {new_unit} for {path}" - ) if not isclose(factor, 1): logger.info( @@ -1560,13 +1567,11 @@ def init_columns( new_columns.append(new_column) payload = {"columns": new_columns} - valid, error = self._is_valid_payload("Project", payload) - if not valid: - raise MPContribsClientError(error) + self._is_valid_payload("Project", payload) return self.projects.updateProjectByName(pk=name, project=payload).result() - def delete_contributions(self, query: Optional[dict] = None, timeout: int = -1): + def delete_contributions(self, query: dict | None = None, timeout: int = -1): """Remove all contributions for a query Args: @@ -1612,7 +1617,7 @@ def delete_contributions(self, query: Optional[dict] = None, timeout: int = -1): def get_totals( self, - query: Optional[dict] = None, + query: dict | None = None, timeout: int = -1, resource: str = "contributions", op: str = "query", @@ -1641,23 +1646,23 @@ def get_totals( query = {k: v for k, v in query.items() if k not in skip_keys} query["_fields"] = [] # only need totals -> explicitly request no fields queries = self._split_query(query, resource=resource, op=op) # don't paginate - result = {"total_count": 0, "total_pages": 0} futures = [ self._get_future(i, q, rel_url=resource) for i, q in enumerate(queries) ] responses = _run_futures(futures, timeout=timeout, desc="Totals") - for resp in responses.values(): - for k in result: - result[k] += resp.get("result", {}).get(k, 0) + result = { + k: sum(resp.get("result", {}).get(k, 0) for resp in responses.values()) + for k in ("total_count", "total_pages") + } return result["total_count"], result["total_pages"] - def count(self, query: Optional[dict] = None) -> int: + def count(self, query: dict | None = None) -> int: """shortcut for get_totals()""" return self.get_totals(query=query)[0] - def get_unique_identifiers_flags(self, query: Optional[dict] = None) -> dict: + def get_unique_identifiers_flags(self, query: dict | None = None) -> dict: """Retrieve values for `unique_identifiers` flags. See `client.available_query_params(resource="projects")` for available query parameters. @@ -1677,10 +1682,10 @@ def get_unique_identifiers_flags(self, query: Optional[dict] = None) -> dict: def get_all_ids( self, - query: Optional[dict] = None, - include: Optional[list[str]] = None, + query: dict | None = None, + include: list[str] | None = None, timeout: int = -1, - data_id_fields: Optional[dict] = None, + data_id_fields: dict | None = None, fmt: str = "sets", op: str = "query", ) -> dict: @@ -1731,7 +1736,7 @@ def get_all_ids( }, ...} """ include = include or [] - components = set(x for x in include if x in COMPONENTS) + components = {x for x in include if x in COMPONENTS} if include and not components: raise MPContribsClientError(f"`include` must be subset of {COMPONENTS}!") @@ -1745,9 +1750,13 @@ def get_all_ids( unique_identifiers = self.get_unique_identifiers_flags() data_id_fields = data_id_fields or {} - for k, v in data_id_fields.items(): - if k in unique_identifiers and isinstance(v, str): - data_id_fields[k] = v + data_id_fields.update( + { + k: v + for k, v in data_id_fields.items() + if k in unique_identifiers and isinstance(v, str) + } + ) ret = {} query = query or {} @@ -1810,34 +1819,40 @@ def get_all_ids( if data_id_field and data_id_field_val: ret[project][identifier][data_id_field] = data_id_field_val - for component in components: - if component in contrib: - ret[project][identifier][component] = { + ret[project][identifier].update( + { + component: { d["name"]: {"id": d["id"], "md5": d["md5"]} for d in contrib[component] } + for component in components + if component in contrib + } + ) elif data_id_field and data_id_field_val: ret[project][identifier] = { data_id_field_val: {"id": contrib["id"]} } - for component in components: - if component in contrib: - ret[project][identifier][data_id_field_val][ - component - ] = { + ret[project][identifier][data_id_field_val].update( + { + component: { d["name"]: {"id": d["id"], "md5": d["md5"]} for d in contrib[component] } + for component in components + if component in contrib + } + ) return ret def query_contributions( self, - query: Optional[dict] = None, - fields: Optional[list] = None, - sort: Optional[str] = None, + query: dict | None = None, + fields: list | None = None, + sort: str | None = None, paginate: bool = False, timeout: int = -1, ) -> dict: @@ -1861,12 +1876,11 @@ def query_contributions( query["project"] = self.project if paginate: - cids = [] - - for v in self.get_all_ids(query).values(): - cids_project = v.get("ids") - if cids_project: - cids.extend(cids_project) + cids = [ + idx + for v in self.get_all_ids(query).values() + for idx in (v.get("ids") or []) + ] if not cids: raise MPContribsClientError("No contributions match the query.") @@ -1891,7 +1905,7 @@ def query_contributions( return ret def update_contributions( - self, data: dict, query: Optional[dict] = None, timeout: int = -1 + self, data: dict, query: dict | None = None, timeout: int = -1 ) -> dict: """Apply the same update to all contributions in a project (matching query) @@ -1906,14 +1920,10 @@ def update_contributions( raise MPContribsClientError("Nothing to update.") tic = time.perf_counter() - valid, error = self._is_valid_payload("Contribution", data) - if not valid: - raise MPContribsClientError(error) + self._is_valid_payload("Contribution", data) if "data" in data: - serializable, error = self._is_serializable_dict(data["data"]) - if not serializable: - raise MPContribsClientError(error) + self._is_serializable_dict(data["data"]) query = query or {} @@ -1939,7 +1949,7 @@ def update_contributions( # get current list of data columns to decide if swagger reload is needed resp = self.projects.getProjectByName(pk=name, _fields=["columns"]).result() - old_paths = set(c["path"] for c in resp["columns"]) + old_paths = {c["path"] for c in resp["columns"]} total = len(cids) cids_query = {"id__in": cids} @@ -1954,7 +1964,7 @@ def update_contributions( if updated: resp = self.projects.getProjectByName(pk=name, _fields=["columns"]).result() - new_paths = set(c["path"] for c in resp["columns"]) + new_paths = {c["path"] for c in resp["columns"]} if new_paths != old_paths: self.init_columns(name=name) @@ -1964,7 +1974,7 @@ def update_contributions( return {"updated": updated, "total": total, "seconds_elapsed": toc - tic} def make_public( - self, query: Optional[dict] = None, recursive: bool = False, timeout: int = -1 + self, query: dict | None = None, recursive: bool = False, timeout: int = -1 ) -> dict: """Publish a project and optionally its contributions @@ -1977,7 +1987,7 @@ def make_public( ) def make_private( - self, query: Optional[dict] = None, recursive: bool = False, timeout: int = -1 + self, query: dict | None = None, recursive: bool = False, timeout: int = -1 ) -> dict: """Make a project and optionally its contributions private @@ -1992,7 +2002,7 @@ def make_private( def _set_is_public( self, is_public: bool, - query: Optional[dict] = None, + query: dict | None = None, recursive: bool = False, timeout: int = -1, ) -> dict: @@ -2124,9 +2134,13 @@ def submit_contributions( resp = self.get_all_ids(dict(id__in=collect_ids), timeout=timeout) project_names |= set(resp.keys()) - for project_name, values in resp.items(): - for cid in values["ids"]: - id2project[cid] = project_name + id2project.update( + { + cid: project_name + for project_name, values in resp.items() + for cid in values["ids"] + } + ) existing = defaultdict(dict) unique_identifiers = defaultdict(dict) @@ -2160,9 +2174,7 @@ def submit_contributions( for contrib in tqdm(contributions, desc="Prepare"): if "data" in contrib: contrib["data"] = unflatten(contrib["data"], splitter="dot") - serializable, error = self._is_serializable_dict(contrib["data"]) - if not serializable: - raise MPContribsClientError(error) + self._is_serializable_dict(contrib["data"]) update = "id" in contrib project_name = id2project[contrib["id"]] if update else contrib["project"] @@ -2282,13 +2294,7 @@ def submit_contributions( digests[project_name][component].add(digest) contribs[project_name][-1][component].append(dct) - valid, error = self._is_valid_payload( - "Contribution", contribs[project_name][-1] - ) - if not valid: - raise MPContribsClientError( - f"{contrib['identifier']} invalid: {error}!" - ) + self._is_valid_payload("Contribution", contribs[project_name][-1]) # submit contributions if contribs: @@ -2425,10 +2431,10 @@ def put_future(pk, payload): def download_contributions( self, - query: Optional[dict] = None, - outdir: Union[str, Path] = DEFAULT_DOWNLOAD_DIR, + query: dict | None = None, + outdir: str | Path = DEFAULT_DOWNLOAD_DIR, overwrite: bool = False, - include: Optional[list[str]] = None, + include: list[str] | None = None, timeout: int = -1, ) -> list: """Download a list of contributions as .json.gz file(s) @@ -2448,7 +2454,7 @@ def download_contributions( include = include or [] outdir = Path(outdir) or Path(".") outdir.mkdir(parents=True, exist_ok=True) - components = set(x for x in include if x in COMPONENTS) + components = {x for x in include if x in COMPONENTS} if include and not components: raise MPContribsClientError(f"`include` must be subset of {COMPONENTS}!") @@ -2527,7 +2533,7 @@ def download_contributions( def download_structures( self, ids: list[str], - outdir: Union[str, Path] = DEFAULT_DOWNLOAD_DIR, + outdir: str | Path = DEFAULT_DOWNLOAD_DIR, overwrite: bool = False, timeout: int = -1, fmt: str = "json", @@ -2556,7 +2562,7 @@ def download_structures( def download_tables( self, ids: list[str], - outdir: Union[str, Path] = DEFAULT_DOWNLOAD_DIR, + outdir: str | Path = DEFAULT_DOWNLOAD_DIR, overwrite: bool = False, timeout: int = -1, fmt: str = "json", @@ -2585,7 +2591,7 @@ def download_tables( def download_attachments( self, ids: list[str], - outdir: Union[str, Path] = DEFAULT_DOWNLOAD_DIR, + outdir: str | Path = DEFAULT_DOWNLOAD_DIR, overwrite: bool = False, timeout: int = -1, fmt: str = "json", @@ -2615,7 +2621,7 @@ def _download_resource( self, resource: str, ids: list[str], - outdir: Union[str, Path] = DEFAULT_DOWNLOAD_DIR, + outdir: str | Path = DEFAULT_DOWNLOAD_DIR, overwrite: bool = False, timeout: int = -1, fmt: str = "json", diff --git a/mpcontribs-client/pyproject.toml b/mpcontribs-client/pyproject.toml new file mode 100644 index 000000000..e7a4fd7e2 --- /dev/null +++ b/mpcontribs-client/pyproject.toml @@ -0,0 +1,64 @@ +[build-system] +requires = ["setuptools>=80.0.0","setuptools-scm>=8"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +root = ".." +relative_to = "__file__" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +where = ["."] +exclude = ["tests"] +include = ["mpcontribs.client"] + +[project] +name = "mpcontribs-client" +dynamic = ["version"] +requires-python = ">=3.11" +description="Client library for MPContribs API" +license = {text = "MIT"} +authors = [ + {name = "Patrick Huck", email = "phuck@lbl.gov"}, + {name = "The Materials Project", email="feedback@materialsproject.org"}, +] +dependencies = [ + "numpy", + "boltons", + "bravado", + "filetype", + "flatten-dict", + "ipython", + "json2html", + "pandas", + "pint", + "plotly", + "pyIsEmail", + "pymatgen", + "pymongo", + "requests-futures", + "swagger-spec-validator", + "tqdm", + "ujson", + "cachetools", +] + +[project.urls] +Homepage = "https://github.com/materialsproject/MPContribs" +Documentation = "https://docs.materialsproject.org/services/mpcontribs" + +[project.optional-dependencies] +dev = [ + "flake8", + "pytest", + "pytest-flake8", + "pytest-pycodestyle", + "pytest-cov", + "pytest-xdist", + "py", +] +all = [ + "mpcontribs-client[dev]" +] diff --git a/mpcontribs-client/setup.py b/mpcontribs-client/setup.py deleted file mode 100644 index 9ad2f515e..000000000 --- a/mpcontribs-client/setup.py +++ /dev/null @@ -1,63 +0,0 @@ -# -*- coding: utf-8 -*- -from setuptools import setup - - -def local_version(version): - # https://github.com/pypa/setuptools_scm/issues/342 - return "" - - -with open("README.md", "r") as fh: - long_description = fh.read() - -setup( - name="mpcontribs-client", - python_requires=">=3.8", - author="Patrick Huck", - author_email="phuck@lbl.gov", - description="client library for MPContribs API", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/materialsproject/MPContribs/tree/master/mpcontribs-client", - packages=["mpcontribs.client"], - install_requires=[ - "numpy", - "boltons", - "bravado", - "filetype", - "flatten-dict", - "ipython", - "json2html", - "pandas", - "pint", - "plotly", - "pyIsEmail", - "pymatgen", - "pymongo", - "requests-futures", - "swagger-spec-validator", - "tqdm", - "ujson", - "semantic-version", - "cachetools", - ], - extras_require={ - "dev": [ - "flake8", - "pytest", - "pytest-flake8", - "pytest-pycodestyle", - "pytest-cov", - "py", - ] - }, - license="MIT", - zip_safe=False, - include_package_data=True, - use_scm_version={ - "root": "..", - "relative_to": __file__, - "local_scheme": local_version, - }, - setup_requires=["setuptools_scm"], -) diff --git a/mpcontribs-client/tests/test_client.py b/mpcontribs-client/tests/test_client.py index 45ded807c..8dd7c344c 100644 --- a/mpcontribs-client/tests/test_client.py +++ b/mpcontribs-client/tests/test_client.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import pytest import logging diff --git a/mpcontribs-io/mpcontribs/io/__init__.py b/mpcontribs-io/mpcontribs/io/__init__.py new file mode 100644 index 000000000..b159f13a5 --- /dev/null +++ b/mpcontribs-io/mpcontribs/io/__init__.py @@ -0,0 +1,9 @@ +"""Set version for mpcontribs-io.""" + +from importlib.metadata import version + +try: + __version__ = version("mpcontribs-io") +except Exception: + # package is not installed + pass diff --git a/mpcontribs-io/pyproject.toml b/mpcontribs-io/pyproject.toml new file mode 100644 index 000000000..295eb1163 --- /dev/null +++ b/mpcontribs-io/pyproject.toml @@ -0,0 +1,74 @@ +[build-system] +requires = ["setuptools>=80.0.0","setuptools-scm>=8"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +root = ".." +relative_to = "__file__" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +where = ["."] +exclude = ["scripts","supervisord"] +include = ["mpcontribs.io"] + +[project] +name = "mpcontribs-io" +dynamic = ["version"] +requires-python = ">=3.11" +description="MPContribs I/O Library" +license = {text = "MIT"} +authors = [ + {name = "Patrick Huck", email = "phuck@lbl.gov"}, + {name = "The Materials Project", email="feedback@materialsproject.org"}, +] +dependencies = [ + "archieml", + "ipython", + "pandas", + "plotly", + "pymatgen" +] + +[project.urls] +Homepage = "https://github.com/materialsproject/MPContribs/tree/master/mpcontribs-io" +Documentation = "https://docs.materialsproject.org/services/mpcontribs" + +[tool.tox] +envlist = ["clean","py311","py312"] + +[tool.tox.gh-actions] +python = ["3.11: py312","3.12: py312"] + +[tool.tox.pycodestyle] +max-line-length = 120 + +[tool.tox.pytest] +flake8-max-line-length = 120 +python_files = ["test_*.py",] + +[tool.tox.coverage.paths] +source = "mpcontribs/io" + +[tool.tox.coverage.run] +source = "mpcontribs/io" +omit = ["*test_*.py",] + +[tool.tox.testenv] +deps = [ + "pytest", + "pytest-flake8", + "pytest-pycodestyle", + "pytest-cov", + "-rrequirements.txt", +] +commands = [ + "pytest -v -s --flake8 --pycodestyle --cov={envsitepackagesdir}/mpcontribs/io --cov-report=term-missing --cov-report=xml" +] + +[tool.tox.testenv.clean] +deps = ["coverage"] +skip_install = true +commands = ["coverage", "erase"] diff --git a/mpcontribs-io/setup.py b/mpcontribs-io/setup.py deleted file mode 100644 index 7e1546efc..000000000 --- a/mpcontribs-io/setup.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- -from setuptools import setup - - -def local_version(version): - # https://github.com/pypa/setuptools_scm/issues/342 - return "" - - -with open("README.md", "r") as fh: - long_description = fh.read() - -setup( - name="mpcontribs-io", - author="Patrick Huck", - author_email="phuck@lbl.gov", - description="MPContribs I/O Library", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/materialsproject/MPContribs/tree/master/mpcontribs-io", - packages=["mpcontribs.io.core", "mpcontribs.io.archie"], - install_requires=["archieml", "ipython", "pandas", "plotly", "pymatgen"], - license="MIT", - zip_safe=False, - include_package_data=True, - use_scm_version={ - "root": "..", - "relative_to": __file__, - "local_scheme": local_version, - }, - setup_requires=["setuptools_scm"], -) diff --git a/mpcontribs-io/tox.ini b/mpcontribs-io/tox.ini deleted file mode 100644 index 57c931167..000000000 --- a/mpcontribs-io/tox.ini +++ /dev/null @@ -1,35 +0,0 @@ -[tox] -envlist = clean,py37,py38 - -[gh-actions] -python = - 3.7: py37 - 3.8: py38 - -[pycodestyle] -max-line-length = 120 - -[pytest] -flake8-max-line-length = 120 -python_files = test_*.py - -[coverage:paths] -source = mpcontribs/io - -[coverage:run] -source = mpcontribs/io -omit = *test_*.py - -[testenv] -deps = - pytest - pytest-flake8 - pytest-pycodestyle - pytest-cov - -rrequirements.txt -commands = pytest -v -s --flake8 --pycodestyle --cov={envsitepackagesdir}/mpcontribs/io --cov-report=term-missing --cov-report=xml - -[testenv:clean] -deps = coverage -skip_install = true -commands = coverage erase diff --git a/mpcontribs-lux/mpcontribs/lux/_types.py b/mpcontribs-lux/mpcontribs/lux/_types.py index 5da8cdda9..903c52866 100644 --- a/mpcontribs-lux/mpcontribs/lux/_types.py +++ b/mpcontribs-lux/mpcontribs/lux/_types.py @@ -9,5 +9,7 @@ ) ComplexType = Annotated[tuple[float, float], _complex_type_validator] +"""Type annotation for complex numbers which are required fields.""" NullableComplexType = Annotated[tuple[float, float] | None, _complex_type_validator] +"""Type annotation for complex numbers which are nullable fields.""" diff --git a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/__init__.py b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/__init__.py index e30cfc64a..a60599183 100644 --- a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/__init__.py +++ b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/__init__.py @@ -33,4 +33,3 @@ "XRDRefinement", "XRDPhase", ] - diff --git a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/base.py b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/base.py index 5bae46d12..d514463dd 100644 --- a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/base.py +++ b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/base.py @@ -4,21 +4,17 @@ Provides common types, validators, and field utilities. """ -from typing import Any, Dict +from typing import Any from pydantic import Field -def ExcludeFromUpload( - default: Any = None, - description: str = "", - **kwargs -) -> Any: +def ExcludeFromUpload(default: Any = None, description: str = "", **kwargs) -> Any: """ Field that should NOT be uploaded to MPContribs. - + Use this for sensitive data that must remain private until publication. Examples: weight_collected, mass measurements that are embargoed. - + Usage: weight_collected: float | None = ExcludeFromUpload( description="Weight of powder collected (embargoed)" @@ -35,10 +31,10 @@ def ExcludeFromUpload( def get_uploadable_fields(model_class) -> list[str]: """ Get list of fields that should be uploaded to MPContribs. - + Args: model_class: Pydantic model class - + Returns: List of field names that are NOT marked with exclude_from_upload """ @@ -53,10 +49,10 @@ def get_uploadable_fields(model_class) -> list[str]: def get_excluded_fields(model_class) -> list[str]: """ Get list of fields that should NOT be uploaded to MPContribs. - + Args: model_class: Pydantic model class - + Returns: List of field names that ARE marked with exclude_from_upload """ @@ -66,4 +62,3 @@ def get_excluded_fields(model_class) -> list[str]: if extra.get("exclude_from_upload", False): excluded.append(field_name) return excluded - diff --git a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/experiment_elements.py b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/experiment_elements.py index ad90a56dd..8c70b7f43 100644 --- a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/experiment_elements.py +++ b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/experiment_elements.py @@ -11,20 +11,14 @@ class ExperimentElement(BaseModel, extra="forbid"): """ Element present in an experiment. - + Each experiment can have multiple elements (1:N relationship). """ - - experiment_id: str = Field( - description="Reference to parent experiment" - ) - - element_symbol: str = Field( - description="Element symbol (e.g., Na, Mg, O)" - ) - + + experiment_id: str = Field(description="Reference to parent experiment") + + element_symbol: str = Field(description="Element symbol (e.g., Na, Mg, O)") + target_atomic_percent: float | None = Field( - default=None, - description="Target atomic percentage of this element" + default=None, description="Target atomic percentage of this element" ) - diff --git a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/experiments.py b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/experiments.py index 5ca1ffdd3..d242ad272 100644 --- a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/experiments.py +++ b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/experiments.py @@ -18,6 +18,7 @@ # When imported dynamically, use absolute import import sys from pathlib import Path + sys.path.insert(0, str(Path(__file__).parent)) from base import ExcludeFromUpload @@ -25,203 +26,157 @@ class Experiment(BaseModel, extra="forbid"): """ Main experiment schema with all 1:1 data consolidated. - + This is the primary table - one row per experiment. All related 1:1 data (heating, recovery, xrd, finalization, dosing) is merged here. """ - + # === Core experiment fields === - experiment_id: str = Field( - description="Unique experiment identifier (MongoDB _id)" - ) - - name: str = Field( - description="Experiment name (e.g., NSC_249, MINES_12)" - ) - + experiment_id: str = Field(description="Unique experiment identifier (MongoDB _id)") + + name: str = Field(description="Experiment name (e.g., NSC_249, MINES_12)") + experiment_type: str = Field( description="Root experiment type (NSC, Na, PG, MINES, TRI)" ) - + experiment_subgroup: str | None = Field( - default=None, - description="Experiment subgroup (e.g., NSC_249, Na_123)" + default=None, description="Experiment subgroup (e.g., NSC_249, Na_123)" ) - - target_formula: str = Field( - description="Target chemical formula" - ) - - last_updated: datetime = Field( - description="Last modification timestamp" - ) - + + target_formula: str = Field(description="Target chemical formula") + + last_updated: datetime = Field(description="Last modification timestamp") + status: Literal["completed", "error", "active", "unknown"] = Field( description="Workflow status" ) - + notes: str | None = Field( - default=None, - description="Optional notes about the experiment" + default=None, description="Optional notes about the experiment" ) # === Heating fields (prefix: heating_) === heating_method: Literal["standard", "atmosphere", "manual", "none"] | None = Field( - default=None, - description="Heating method used" + default=None, description="Heating method used" ) - + heating_temperature: float | None = Field( - default=None, - description="Target heating temperature in °C" + default=None, description="Target heating temperature in °C" ) - + heating_time: float | None = Field( - default=None, - description="Heating duration in minutes" + default=None, description="Heating duration in minutes" ) - + heating_cooling_rate: float | None = Field( - default=None, - description="Cooling rate in °C/min" + default=None, description="Cooling rate in °C/min" ) - + heating_atmosphere: str | None = Field( - default=None, - description="Atmosphere used during heating (e.g., N2, Ar, Air)" + default=None, description="Atmosphere used during heating (e.g., N2, Ar, Air)" ) - + heating_flow_rate_ml_min: float | None = Field( - default=None, - description="Gas flow rate during heating in mL/min" + default=None, description="Gas flow rate during heating in mL/min" ) - + heating_low_temp_calcination: bool | None = Field( - default=None, - description="Whether low temperature calcination was used" + default=None, description="Whether low temperature calcination was used" ) # === Recovery fields (prefix: recovery_) === recovery_total_dosed_mass_mg: float | None = Field( - default=None, - description="Total mass of all powders dosed in mg" + default=None, description="Total mass of all powders dosed in mg" ) - + # EXCLUDED FROM UPLOAD per team request recovery_weight_collected_mg: float | None = ExcludeFromUpload( description="Weight of powder collected after heating in mg (EMBARGOED)" ) - + recovery_yield_percent: float | None = Field( - default=None, - description="Recovery yield (collected / dosed * 100)" + default=None, description="Recovery yield (collected / dosed * 100)" ) - + recovery_initial_crucible_weight_mg: float | None = Field( - default=None, - description="Initial crucible weight before experiment in mg" + default=None, description="Initial crucible weight before experiment in mg" ) - + recovery_failure_classification: str | None = Field( - default=None, - description="Classification of any failure during recovery" + default=None, description="Classification of any failure during recovery" ) # === XRD measurement fields (prefix: xrd_) === xrd_sampleid_in_aeris: str | None = Field( - default=None, - description="Sample ID in Aeris XRD system" + default=None, description="Sample ID in Aeris XRD system" ) - + xrd_holder_index: int | None = Field( - default=None, - description="XRD sample holder position index" + default=None, description="XRD sample holder position index" ) - + # EXCLUDED FROM UPLOAD per team request xrd_total_mass_dispensed_mg: float | None = ExcludeFromUpload( description="Mass dispensed for XRD measurement in mg (EMBARGOED)" ) - + xrd_met_target_mass: bool | None = Field( - default=None, - description="Whether target mass was achieved for XRD" + default=None, description="Whether target mass was achieved for XRD" ) # === Finalization fields (prefix: finalization_) === finalization_decoded_sample_id: str | None = Field( - default=None, - description="Decoded sample ID from barcode" + default=None, description="Decoded sample ID from barcode" ) - + finalization_successful_labeling: bool | None = Field( - default=None, - description="Whether sample was successfully labeled" + default=None, description="Whether sample was successfully labeled" ) - + finalization_storage_location: str | None = Field( - default=None, - description="Final storage location of sample" + default=None, description="Final storage location of sample" ) # === Dosing fields (prefix: dosing_) === dosing_crucible_position: int | None = Field( - default=None, - description="Crucible position in rack", - ge=1, - le=4 - ) - - dosing_crucible_sub_rack: Literal["SubRackA", "SubRackB", "SubRackC", "SubRackD"] | None = Field( - default=None, - description="Sub-rack identifier" + default=None, description="Crucible position in rack", ge=1, le=4 ) - + + dosing_crucible_sub_rack: ( + Literal["SubRackA", "SubRackB", "SubRackC", "SubRackD"] | None + ) = Field(default=None, description="Sub-rack identifier") + dosing_mixing_pot_position: int | None = Field( - default=None, - description="Mixing pot position", - ge=1, - le=16 + default=None, description="Mixing pot position", ge=1, le=16 ) - + dosing_ethanol_dispense_volume: int | None = Field( - default=None, - description="Volume of ethanol dispensed in µL", - ge=0 + default=None, description="Volume of ethanol dispensed in µL", ge=0 ) - + dosing_target_transfer_volume: int | None = Field( - default=None, - description="Target transfer volume in µL", - ge=0 + default=None, description="Target transfer volume in µL", ge=0 ) - + dosing_actual_transfer_mass: float | None = Field( - default=None, - description="Actual mass transferred in g", - ge=0 + default=None, description="Actual mass transferred in g", ge=0 ) - + dosing_dac_duration: int | None = Field( - default=None, - description="DAC duration in seconds", - ge=0 + default=None, description="DAC duration in seconds", ge=0 ) - + dosing_dac_speed: int | None = Field( - default=None, - description="DAC rotation speed in rpm", - ge=0 + default=None, description="DAC rotation speed in rpm", ge=0 ) - + dosing_actual_heat_duration: int | None = Field( default=None, description="Actual heating duration during dosing in seconds", - ge=0 + ge=0, ) - + dosing_end_reason: str | None = Field( - default=None, - description="Reason for ending dosing session" + default=None, description="Reason for ending dosing session" ) - diff --git a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/powder_doses.py b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/powder_doses.py index f2f376f32..4d7e75d4f 100644 --- a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/powder_doses.py +++ b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/powder_doses.py @@ -9,68 +9,53 @@ """ from pydantic import BaseModel, Field +from emmet.core.types.typing import JsonListType +from emmet.core.utils import type_override +@type_override( + { + "Doses": str, + } +) class PowderItem(BaseModel, extra="forbid"): """Schema for a powder item within a dose.""" - - PowderName: str | None = Field( - default=None, - description="Name of the powder" - ) + + PowderName: str | None = Field(default=None, description="Name of the powder") TargetMass: float | None = Field( - default=None, - description="Target mass in mg", - ge=0 + default=None, description="Target mass in mg", ge=0 ) - Doses: list[dict] | None = Field( - default=None, - description="List of individual doses" + Doses: JsonListType | None = Field( + default=None, description="List of individual doses" ) class PowderDose(BaseModel, extra="forbid"): """ Individual powder dose event. - + Each experiment can have multiple powder doses (1:N relationship). This is the flattened representation from the nested Powders[].Doses[] structure. """ - - experiment_id: str = Field( - description="Reference to parent experiment" - ) - - powder_name: str = Field( - description="Name of the powder material" - ) - - target_mass: float = Field( - description="Target mass in mg", - ge=0 - ) - - actual_mass: float = Field( - description="Actual mass dispensed in mg", - ge=0 - ) - - accuracy_percent: float = Field( - description="Dosing accuracy (actual/target * 100)" - ) - + + experiment_id: str = Field(description="Reference to parent experiment") + + powder_name: str = Field(description="Name of the powder material") + + target_mass: float = Field(description="Target mass in mg", ge=0) + + actual_mass: float = Field(description="Actual mass dispensed in mg", ge=0) + + accuracy_percent: float = Field(description="Dosing accuracy (actual/target * 100)") + dose_sequence: int = Field( - description="Sequence number within the experiment", - ge=0 + description="Sequence number within the experiment", ge=0 ) - + head_position: int | None = Field( - default=None, - description="Dispenser head position" + default=None, description="Dispenser head position" ) - + dose_timestamp: str | None = Field( - default=None, - description="Timestamp of the dose event" + default=None, description="Timestamp of the dose event" ) - diff --git a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/temperature_logs.py b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/temperature_logs.py index 980c7af9d..e2f49b286 100644 --- a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/temperature_logs.py +++ b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/temperature_logs.py @@ -14,38 +14,29 @@ class TemperatureLogEntry(BaseModel, extra="forbid"): """ Single temperature reading. - + Each experiment can have thousands of temperature log entries (1:N relationship). """ - - experiment_id: str = Field( - description="Reference to parent experiment" - ) - - sequence_number: int = Field( - description="Sequence number in the time series", - ge=0 - ) - + + experiment_id: str = Field(description="Reference to parent experiment") + + sequence_number: int = Field(description="Sequence number in the time series", ge=0) + time_minutes: float = Field( description="Time elapsed since heating start in minutes" ) - - temperature_celsius: float = Field( - description="Temperature reading in °C" - ) + + temperature_celsius: float = Field(description="Temperature reading in °C") # For backward compatibility with nested structure class TemperatureLog(BaseModel, extra="forbid"): """Temperature log with array structure (MongoDB format).""" - + time_minutes: list[float] | None = Field( default=None, - description="Time elapsed in minutes since the start of the heating process" + description="Time elapsed in minutes since the start of the heating process", ) temperature_celsius: list[float] | None = Field( - default=None, - description="Temperature of the sample in Celsius" + default=None, description="Temperature of the sample in Celsius" ) - diff --git a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/workflow_tasks.py b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/workflow_tasks.py index 7b6c6a1d2..2b00928ab 100644 --- a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/workflow_tasks.py +++ b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/workflow_tasks.py @@ -13,43 +13,33 @@ class WorkflowTask(BaseModel, extra="forbid"): """ Single task in the experiment workflow. - + Each experiment has multiple tasks (1:N relationship). Tasks include: PowderDosing, Heating, RecoverPowder, Diffraction, Ending, etc. """ - - experiment_id: str = Field( - description="Reference to parent experiment" - ) - - task_id: str = Field( - description="Unique task identifier" - ) - + + experiment_id: str = Field(description="Reference to parent experiment") + + task_id: str = Field(description="Unique task identifier") + task_type: str = Field( description="Type of task (PowderDosing, Heating, RecoverPowder, Diffraction, Ending)" ) - + status: Literal["completed", "error", "pending", "running"] = Field( description="Task execution status" ) - - created_at: datetime = Field( - description="When the task was created" - ) - + + created_at: datetime = Field(description="When the task was created") + started_at: datetime | None = Field( - default=None, - description="When the task started execution" + default=None, description="When the task started execution" ) - + completed_at: datetime | None = Field( - default=None, - description="When the task completed" + default=None, description="When the task completed" ) - + message: str | None = Field( - default=None, - description="Task result message or error" + default=None, description="Task result message or error" ) - diff --git a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/xrd_data_points.py b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/xrd_data_points.py index 4db376e72..48a8526da 100644 --- a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/xrd_data_points.py +++ b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/xrd_data_points.py @@ -14,25 +14,15 @@ class XRDDataPoint(BaseModel, extra="forbid"): """ Single XRD data point (2θ, counts). - + Each experiment can have ~8000 data points (1:N relationship). This is the flattened representation from twotheta[] and counts[] arrays. """ - - experiment_id: str = Field( - description="Reference to parent experiment" - ) - - point_index: int = Field( - description="Index in the diffraction pattern", - ge=0 - ) - - twotheta: float = Field( - description="2θ angle in degrees" - ) - - counts: float = Field( - description="Intensity counts at this angle" - ) + experiment_id: str = Field(description="Reference to parent experiment") + + point_index: int = Field(description="Index in the diffraction pattern", ge=0) + + twotheta: float = Field(description="2θ angle in degrees") + + counts: float = Field(description="Intensity counts at this angle") diff --git a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/xrd_phases.py b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/xrd_phases.py index dccd7ef92..ffeb8f748 100644 --- a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/xrd_phases.py +++ b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/xrd_phases.py @@ -11,54 +11,36 @@ class XRDPhase(BaseModel, extra="forbid"): """ Identified crystal phase from XRD refinement. - + Each experiment can have multiple phases (1:N relationship). """ - - experiment_id: str = Field( - description="Reference to parent experiment" - ) - - experiment_name: str = Field( - description="Experiment name (for convenience)" - ) - - phase_name: str = Field( - description="Name of the identified phase" - ) - - spacegroup: str = Field( - description="Space group of the phase (e.g., Fm-3m)" - ) - + + experiment_id: str = Field(description="Reference to parent experiment") + + experiment_name: str = Field(description="Experiment name (for convenience)") + + phase_name: str = Field(description="Name of the identified phase") + + spacegroup: str = Field(description="Space group of the phase (e.g., Fm-3m)") + weight_fraction: float = Field( - description="Weight fraction of this phase (0-1)", - ge=0, - le=1 + description="Weight fraction of this phase (0-1)", ge=0, le=1 ) - + weight_fraction_error: float | None = Field( - default=None, - description="Error in weight fraction" + default=None, description="Error in weight fraction" ) - + lattice_a_nm: float | None = Field( - default=None, - description="Lattice parameter a in nm" + default=None, description="Lattice parameter a in nm" ) - + lattice_b_nm: float | None = Field( - default=None, - description="Lattice parameter b in nm" + default=None, description="Lattice parameter b in nm" ) - + lattice_c_nm: float | None = Field( - default=None, - description="Lattice parameter c in nm" - ) - - r_phase: float | None = Field( - default=None, - description="R-factor for this phase" + default=None, description="Lattice parameter c in nm" ) + r_phase: float | None = Field(default=None, description="R-factor for this phase") diff --git a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/xrd_refinements.py b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/xrd_refinements.py index 9087b1136..7f030c485 100644 --- a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/xrd_refinements.py +++ b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/xrd_refinements.py @@ -11,79 +11,50 @@ class XRDRefinement(BaseModel, extra="forbid"): """ XRD Rietveld refinement result from DARA analysis. - + One result per experiment (1:1 relationship). """ - - experiment_id: str = Field( - description="Reference to parent experiment" - ) - - experiment_name: str = Field( - description="Experiment name (for convenience)" - ) - - success: bool = Field( - description="Whether the refinement was successful" - ) - + + experiment_id: str = Field(description="Reference to parent experiment") + + experiment_name: str = Field(description="Experiment name (for convenience)") + + success: bool = Field(description="Whether the refinement was successful") + error: str | None = Field( - default=None, - description="Error message if refinement failed" + default=None, description="Error message if refinement failed" ) - + error_type: str | None = Field( - default=None, - description="Classification of error type" + default=None, description="Classification of error type" ) - + rwp: float | None = Field( - default=None, - description="Weighted profile R-factor (goodness of fit)" - ) - - rp: float | None = Field( - default=None, - description="Profile R-factor" - ) - - rexp: float | None = Field( - default=None, - description="Expected R-factor" - ) - - num_phases: int = Field( - default=0, - description="Number of phases identified" + default=None, description="Weighted profile R-factor (goodness of fit)" ) - + + rp: float | None = Field(default=None, description="Profile R-factor") + + rexp: float | None = Field(default=None, description="Expected R-factor") + + num_phases: int = Field(default=0, description="Number of phases identified") + chemical_system: str | None = Field( - default=None, - description="Chemical system (e.g., Na-Mg-P-O)" - ) - - target_formula: str | None = Field( - default=None, - description="Target formula" + default=None, description="Chemical system (e.g., Na-Mg-P-O)" ) - + + target_formula: str | None = Field(default=None, description="Target formula") + analysis_timestamp: str | None = Field( - default=None, - description="When the analysis was performed" - ) - - mode: str | None = Field( - default=None, - description="Analysis mode" + default=None, description="When the analysis was performed" ) - + + mode: str | None = Field(default=None, description="Analysis mode") + wmin: int | None = Field( - default=None, - description="Minimum 2θ angle for refinement" + default=None, description="Minimum 2θ angle for refinement" ) - + wmax: int | None = Field( - default=None, - description="Maximum 2θ angle for refinement" + default=None, description="Maximum 2θ angle for refinement" ) - diff --git a/mpcontribs-lux/tests/test_autogen.py b/mpcontribs-lux/tests/test_autogen.py index 17b877ddb..71cd04a67 100644 --- a/mpcontribs-lux/tests/test_autogen.py +++ b/mpcontribs-lux/tests/test_autogen.py @@ -60,7 +60,7 @@ def test_schema_generation(test_dir): schema = schemer.schema() # Avoiding `eval` here; `eval` also can't handle import statements - with open(test_py_file := Path("./test_no_kwargs.py").resolve(), "wt") as py_temp: + with open(test_py_file := Path("./test_no_kwargs.py").resolve(), "w") as py_temp: py_temp.write(schema) test_no_kwargs = dynamically_load_module_from_path(test_py_file, "test_no_kwargs") @@ -74,7 +74,7 @@ def test_schema_generation(test_dir): new_desc = {"b0": "Bulk modulus", "cif": "CIF"} with open( - test_py_file := Path("./test_w_name_and_anno.py").resolve(), "wt" + test_py_file := Path("./test_w_name_and_anno.py").resolve(), "w" ) as py_temp: py_temp.write(schemer.schema(model_name="TestClass", descriptions=new_desc)) test_w_kwargs = dynamically_load_module_from_path(test_py_file, "test_w_kwargs") diff --git a/mpcontribs-portal/mpcontribs/portal/__init__.py b/mpcontribs-portal/mpcontribs/portal/__init__.py index e69de29bb..59ebc2dd1 100644 --- a/mpcontribs-portal/mpcontribs/portal/__init__.py +++ b/mpcontribs-portal/mpcontribs/portal/__init__.py @@ -0,0 +1,9 @@ +"""Set version for mpcontribs-portal.""" + +from importlib.metadata import version + +try: + __version__ = version("mpcontribs-portal") +except Exception: + # package is not installed + pass diff --git a/mpcontribs-portal/pyproject.toml b/mpcontribs-portal/pyproject.toml new file mode 100644 index 000000000..4daab63ee --- /dev/null +++ b/mpcontribs-portal/pyproject.toml @@ -0,0 +1,91 @@ +[build-system] +requires = ["setuptools>=80.0.0","setuptools-scm>=8"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +root = ".." +relative_to = "__file__" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +where = ["."] +exclude = ["scripts","supervisord"] +include = ["mpcontribs.portal", "mpcontribs.users"] + +[project] +name = "mpcontribs-portal" +dynamic = ["version"] +requires-python = ">=3.11" +description="MPContribs Portal" +license = {text = "MIT"} +authors = [ + {name = "Patrick Huck", email = "phuck@lbl.gov"}, + {name = "The Materials Project", email="feedback@materialsproject.org"}, +] +dependencies = [ + "boltons", + "boto3", + "ddtrace", + "Django>=3.2,<4.0", + "django-extensions", + "django-settings-file", + "django-webpack4-loader", + "fastnumbers", + "gunicorn[gevent]", + "ipykernel", + "ipython-genutils", + "jinja2", + "json2html", + "monty", + "mpcontribs-client", + "nbconvert", + "nbformat", + "redis", + "scipy", + "setproctitle", + "whitenoise", +] + +[project.urls] +Homepage = "https://github.com/materialsproject/MPContribs" +Documentation = "https://docs.materialsproject.org/services/mpcontribs" + +[project.optional-dependencies] +dev = [ + "flake8", + "pytest", + "pytest-flake8", + "pytest-pycodestyle", + "pytest-xdist", +] +all = [ + "mpcontribs-portal[dev]" +] + +[tool.pytest] +markers = [ + "base: basic resource testing", + "extra: all extra views", +] + +[tool.pycodestyle] +count = true +ignore = ["E121","E123","E126","E133","E226","E241","E242","E704","W503","W504","W505","E741","W605"] +max-line-length = 120 +statistics = true +exclude = ["supervisord","notebooks"] + +[tool.flake8] +exclude = [".git","__pycache__","tests","flasgger","flask-mongorest"] +extend-ignore = ["E741",] +max-line-length = 120 + +[tool.pydocstyle] +ignore = ["D105","D2","D4"] + +[tool.mypy] +ignore_missing_imports = true +namespace_packages = true +python_version = 3.11 diff --git a/mpcontribs-portal/setup.py b/mpcontribs-portal/setup.py deleted file mode 100644 index 7ee9d1e22..000000000 --- a/mpcontribs-portal/setup.py +++ /dev/null @@ -1,40 +0,0 @@ -# -*- coding: utf-8 -*- -import datetime -from setuptools import setup - -setup( - name="mpcontribs-portal", - version=datetime.datetime.today().strftime("%Y.%m.%d"), - description="MPContribs Portal", - author="Patrick Huck", - author_email="phuck@lbl.gov", - url="https://docs.mpcontribs.org", - packages=["mpcontribs.portal", "mpcontribs.users"], - install_requires=[ - "boltons", - "boto3", - "ddtrace", - "Django>=3.2,<4.0", - "django-extensions", - "django-settings-file", - "django-webpack4-loader", - "fastnumbers", - "gunicorn[gevent]", - "ipykernel", - "ipython-genutils", - "jinja2", - "json2html", - "monty", - "mpcontribs-client", - "nbconvert", - "nbformat", - "redis", - "scipy", - "setproctitle", - "whitenoise", - ], - extras_require={"dev": ["pytest", "flake8"]}, - license="MIT", - zip_safe=False, - include_package_data=True, -)