diff --git a/.circleci/config.yml b/.circleci/config.yml index a1474b2e9..997062024 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -27,6 +27,9 @@ executors: python313: docker: - image: cimg/python:3.13 + python314: + docker: + - image: cimg/python:3.14 jobs: test_generic: @@ -56,20 +59,21 @@ jobs: source venv_test/bin/activate pip freeze - run: cat /tmp/tests_run + - run: command: | source venv_test/bin/activate cd grid2op/tests/ export _GRID2OP_FORCE_TEST=1 + export COVERAGE_FILE=/tmp/coverage_shards/.coverage.shard_test_generic_$CIRCLE_NODE_INDEX coverage run -m unittest -v $(cat /tmp/tests_run) - # ls -lah | grep .coverage - # - run: - # command: | - # ls -lah /Grid2Op/grid2op/tests/.coverage.* - - # - store_artifacts: - # path: "/Grid2Op/grid2op/tests/test_Action.py" - # destination: coverage_artifacts/ + ls -lah | grep .coverage + + - persist_to_workspace: + root: /tmp/coverage_shards + paths: + - .coverage.* + test_agent: executor: grid2op-executor resource_class: small @@ -82,9 +86,9 @@ jobs: - run: command: | source venv_test/bin/activate - pip install -U pip setuptools wheel + pip install -U pip setuptools wheel coverage - run: - command: | + command: | source venv_test/bin/activate pip install -e .[test] pip freeze @@ -92,8 +96,24 @@ jobs: command: | source venv_test/bin/activate export _GRID2OP_FORCE_TEST=1 + export COVERAGE_FILE=/tmp/coverage_shards/.coverage.test_agent_$CIRCLE_NODE_INDEX cd grid2op/tests/ - python -m unittest -v test_Agent test_AgentsFast test_recopowerlineperarea + coverage run -m unittest -v \ + test_Agent \ + test_AgentsFast \ + test_recopowerlineperarea \ + test_Opponent \ + test_baseline_alert + ls -lah | grep .coverage + + - persist_to_workspace: + root: /tmp/coverage_shards + paths: + - .coverage.* + + - store_artifacts: + path: /tmp/coverage_shards + destination: coverage_artifacts/ test_converter: executor: grid2op-executor @@ -107,7 +127,7 @@ jobs: - run: command: | source venv_test/bin/activate - pip install -U pip setuptools wheel + pip install -U pip setuptools wheel coverage - run: command: | source venv_test/bin/activate @@ -117,8 +137,18 @@ jobs: command: | source venv_test/bin/activate export _GRID2OP_FORCE_TEST=1 + export COVERAGE_FILE=/tmp/coverage_shards/.coverage.test_converter_$CIRCLE_NODE_INDEX cd grid2op/tests/ - python -m unittest -v test_AgentConverter test_Converter test_BackendConverter + coverage run -m unittest -v test_AgentConverter test_Converter test_BackendConverter + + - persist_to_workspace: + root: /tmp/coverage_shards + paths: + - .coverage.* + + - store_artifacts: + path: /tmp/coverage_shards + destination: coverage_artifacts/ test_runner_epdata_score: executor: grid2op-executor @@ -128,11 +158,11 @@ jobs: - run: apt-get update -y - run: apt-get install -y coinor-cbc - run: python -m pip install virtualenv - - run: python -m virtualenv venv_test + - run: python -m virtualenv venv_test - run: command: | source venv_test/bin/activate - pip install -U pip setuptools wheel + pip install -U pip setuptools wheel coverage - run: command: | source venv_test/bin/activate @@ -142,8 +172,32 @@ jobs: command: | source venv_test/bin/activate export _GRID2OP_FORCE_TEST=1 + export COVERAGE_FILE=/tmp/coverage_shards/.coverage.test_runner_epdata_score_$CIRCLE_NODE_INDEX cd grid2op/tests/ - python -m unittest -v test_EpisodeData test_runner_kwargs_backend test_Runner test_RunnerFast test_score_idf_2023_assistant test_score_idf_2023_nres test_score_idf_2023 test_score_wcci_2022 test_AlarmScore test_RewardAlertCostScore test_RewardNewRenewableSourcesUsageScore test_utils test_CompactEpisodeData test_reset_options_runner + coverage run -m unittest -v \ + test_EpisodeData \ + test_runner_kwargs_backend \ + test_Runner \ + test_RunnerFast \ + test_score_idf_2023_assistant \ + test_score_idf_2023_nres \ + test_score_idf_2023 \ + test_score_wcci_2022 \ + test_AlarmScore \ + test_RewardAlertCostScore \ + test_RewardNewRenewableSourcesUsageScore \ + test_utils \ + test_CompactEpisodeData \ + test_reset_options_runner + + - persist_to_workspace: + root: /tmp/coverage_shards + paths: + - .coverage.* + + - store_artifacts: + path: /tmp/coverage_shards + destination: coverage_artifacts/ test_env_general: executor: grid2op-executor @@ -157,7 +211,7 @@ jobs: - run: command: | source venv_test/bin/activate - pip install -U pip setuptools wheel + pip install -U pip setuptools wheel coverage - run: command: | source venv_test/bin/activate @@ -167,8 +221,32 @@ jobs: command: | source venv_test/bin/activate export _GRID2OP_FORCE_TEST=1 + export COVERAGE_FILE=/tmp/coverage_shards/.coverage.test_env_general_$CIRCLE_NODE_INDEX cd grid2op/tests/ - python -m unittest -v test_attached_envs test_attached_envs_compat test_l2rpn_idf_2023 test_MultiMix test_timeOutEnvironment test_MaskedEnvironment test_MakeEnv test_multi_steps_env test_simenv_blackout test_get_default_env_kwargs + coverage run -m unittest -v \ + test_attached_envs \ + test_attached_envs_compat \ + test_l2rpn_idf_2023 \ + test_MultiMix \ + test_timeOutEnvironment \ + test_MaskedEnvironment \ + test_MakeEnv \ + test_multi_steps_env \ + test_simenv_blackout \ + test_get_default_env_kwargs \ + test_Environment \ + test_EnvironmentCpy \ + test_generate_classes \ + test_basic_env_ls + + - persist_to_workspace: + root: /tmp/coverage_shards + paths: + - .coverage.* + + - store_artifacts: + path: /tmp/coverage_shards + destination: coverage_artifacts/ test_alert_alarm: executor: grid2op-executor @@ -182,7 +260,7 @@ jobs: - run: command: | source venv_test/bin/activate - pip install -U pip setuptools wheel + pip install -U pip setuptools wheel coverage - run: command: | source venv_test/bin/activate @@ -192,22 +270,36 @@ jobs: command: | source venv_test/bin/activate export _GRID2OP_FORCE_TEST=1 + export COVERAGE_FILE=/tmp/coverage_shards/.coverage.test_alert_alarm_$CIRCLE_NODE_INDEX cd grid2op/tests/ - python -m unittest -v test_AlarmFeature test_alert_gym_compat test_alert_obs_act test_alert_trust_score test_AlertReward + coverage run -m unittest -v \ + test_AlarmFeature \ + test_alert_gym_compat \ + test_alert_obs_act \ + test_alert_trust_score \ + test_AlertReward + + - persist_to_workspace: + root: /tmp/coverage_shards + paths: + - .coverage.* + + - store_artifacts: + path: /tmp/coverage_shards + destination: coverage_artifacts/ - test_issue: + test_time_series: executor: grid2op-executor resource_class: small steps: - checkout - run: apt-get update -y - - run: apt-get install -y coinor-cbc - run: python -m pip install virtualenv - run: python -m virtualenv venv_test - run: command: | source venv_test/bin/activate - pip install -U pip setuptools wheel + pip install -U pip setuptools wheel coverage - run: command: | source venv_test/bin/activate @@ -217,34 +309,149 @@ jobs: command: | source venv_test/bin/activate export _GRID2OP_FORCE_TEST=1 + export COVERAGE_FILE=/tmp/coverage_shards/.coverage.test_alert_alarm_$CIRCLE_NODE_INDEX cd grid2op/tests/ - python -m unittest -v test_issue_* + coverage run -m unittest -v \ + test_ts_handlers \ + test_ChronicsHandler \ + test_env_from_episode + + - persist_to_workspace: + root: /tmp/coverage_shards + paths: + - .coverage.* - gen_coverage: + - store_artifacts: + path: /tmp/coverage_shards + destination: coverage_artifacts/ + + test_backend: executor: grid2op-executor resource_class: small steps: - checkout + - run: apt-get update -y - run: python -m pip install virtualenv - run: python -m virtualenv venv_test - run: command: | source venv_test/bin/activate pip install -U pip setuptools wheel coverage + - run: + command: | + source venv_test/bin/activate + pip install -e .[test] + pip freeze - run: command: | source venv_test/bin/activate + export _GRID2OP_FORCE_TEST=1 + export COVERAGE_FILE=/tmp/coverage_shards/.coverage.test_alert_alarm_$CIRCLE_NODE_INDEX cd grid2op/tests/ - coverage combine ../../coverage_artifacts/ --keep - coverage report -m -i - coverage html -i - coverage xml -i + coverage run -m unittest -v \ + test_basicBackendInterface \ + test_action_set_orig_state \ + test_PandaPowerBackendDefaultFunc \ + test_PandaPowerBackend \ + test_BackendAction + + - persist_to_workspace: + root: /tmp/coverage_shards + paths: + - .coverage.* + - store_artifacts: - path: grid2op/tests/htmlcov - destination: htmlcov + path: /tmp/coverage_shards + destination: coverage_artifacts/ + + test_issue: + executor: grid2op-executor + resource_class: small + steps: + - checkout + - run: apt-get update -y + - run: apt-get install -y coinor-cbc + - run: python -m pip install virtualenv + - run: python -m virtualenv venv_test + - run: + command: | + source venv_test/bin/activate + pip install -U pip setuptools wheel coverage + - run: + command: | + source venv_test/bin/activate + pip install -e .[test] + pip freeze + - run: + command: | + source venv_test/bin/activate + export _GRID2OP_FORCE_TEST=1 + export COVERAGE_FILE=/tmp/coverage_shards/.coverage.test_issue_$CIRCLE_NODE_INDEX + cd grid2op/tests/ + coverage run -m unittest -v test_issue_* + + - persist_to_workspace: + root: /tmp/coverage_shards + paths: + - .coverage.* + + - store_artifacts: + path: /tmp/coverage_shards + destination: coverage_artifacts/ + + upload_coverage: + executor: grid2op-executor + steps: + - checkout + - run: + name: Fetch full git history for SonarScanner SCM blame + command: git fetch --unshallow || true + - attach_workspace: + at: /tmp/coverage_shards + - run: python -m pip install virtualenv + - run: python -m virtualenv venv_coverage + - run: + command: | + source venv_coverage/bin/activate + pip install coverage + + - run: + name: Debug env vars + command: | + echo "CODACY_PROJECT_TOKEN is set: $([ -n "$CODACY_PROJECT_TOKEN" ] && echo YES || echo NO)" + + - run: + name: Combine and convert coverage + command: | + source venv_coverage/bin/activate + cd /tmp/coverage_shards + coverage combine --keep + coverage xml --ignore-errors -o /tmp/coverage.xml + - store_artifacts: - path: grid2op/tests/coverage.xml - destination: coverage.xml + path: /tmp/coverage.xml + destination: coverage_artifacts/ + + - run: + name: Upload to Codacy + command: | + bash <(curl -Ls https://coverage.codacy.com/get.sh) report \ + -r /tmp/coverage.xml \ + --project-token "$CODACY_PROJECT_TOKEN" + + - run: + name: Upload to SonarQube Cloud + command: | + curl -Ls https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-8.1.0.6389-linux-x64.zip \ + -o sonar-scanner.zip + unzip -q sonar-scanner.zip + ./sonar-scanner-8.1.0.6389-linux-x64/bin/sonar-scanner \ + -Dsonar.token="$SONAR_TOKEN" \ + -Dsonar.host.url=https://sonarcloud.io \ + -Dsonar.organization=grid2op \ + -Dsonar.projectKey=Grid2op_grid2op \ + -Dsonar.scm.disabled=true \ + -Dsonar.python.coverage.reportPaths=/tmp/coverage.xml install36: executor: python36 @@ -308,7 +515,7 @@ jobs: export _GRID2OP_FORCE_TEST=1 grid2op.testinstall - install38: + install38: executor: python38 resource_class: small steps: @@ -354,7 +561,7 @@ jobs: cd /tmp grid2op.testinstall - legacy_lightsim_old_pp: + legacy_lightsim_old_pp: executor: python38 # needs to be 38: whl of lightsim were not released for 3.10 at the time resource_class: small steps: @@ -381,7 +588,7 @@ jobs: export _GRID2OP_FORCE_TEST=1 python -m unittest grid2op/tests/test_basic_env_ls.py - legacy_lightsim: + legacy_lightsim: executor: python38 # needs to be 38: whl of lightsim were not released for 3.10 at the time resource_class: small steps: @@ -408,7 +615,7 @@ jobs: export _GRID2OP_FORCE_TEST=1 python -m unittest grid2op/tests/test_basic_env_ls.py - test_chronix2grid: + test_chronix2grid: executor: python310 resource_class: small steps: @@ -524,6 +731,7 @@ jobs: export _GRID2OP_FORCE_TEST=1 cd /tmp grid2op.testinstall + install312: executor: python312 resource_class: small @@ -557,6 +765,7 @@ jobs: export _GRID2OP_FORCE_TEST=1 cd /tmp grid2op.testinstall + install313: executor: python313 resource_class: small @@ -591,6 +800,40 @@ jobs: cd /tmp grid2op.testinstall + install314: + executor: python314 + resource_class: small + steps: + - checkout + - run: python -m pip install virtualenv + - run: python -m virtualenv venv_test + - run: + command: | + source venv_test/bin/activate + python -m pip install -U pip setuptools wheel + - run: + command: | # oldest numpy version >= 2 supported by python 3.14 + source venv_test/bin/activate + python -m pip install -U "numpy>=2.3,<2.4" . + pip freeze + - run: + command: | + source venv_test/bin/activate + export _GRID2OP_FORCE_TEST=1 + cd /tmp + grid2op.testinstall + - run: + command: | # most recent numpy version + source venv_test/bin/activate + python -m pip install -U numpy . + pip freeze + - run: + command: | + source venv_test/bin/activate + export _GRID2OP_FORCE_TEST=1 + cd /tmp + grid2op.testinstall + workflows: version: 2.1 test: @@ -602,10 +845,22 @@ workflows: - test_env_general - test_alert_alarm - test_issue + - test_time_series - legacy_lightsim_old_pp - legacy_lightsim - test_chronix2grid - + - upload_coverage: + context: + - coverage_codacy + - coverage_sonarqube + requires: + - test_generic + - test_agent + - test_converter + - test_runner_epdata_score + - test_env_general + - test_alert_alarm + - test_issue install: jobs: - install38 @@ -614,19 +869,4 @@ workflows: - install311 - install312 - install313 - - # gather_test_report: - # requires: - # - test - # jobs: - # - gen_coverage - - # send_tests_report: - # requires: - # - gather_test_report - # orbs: - # coverage-reporter: codacy/coverage-reporter@14.1.0 - # jobs: - # coverage-reporter/send_report: - # coverage-reports: coverage.xml - # project-token: $CODACY_PROJECT_TOKEN + # - install314 diff --git a/.codacy.yml b/.codacy.yml index 38dd3e42c..cb0c33df9 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -4,3 +4,4 @@ exclude_paths: - '*.md' - "examples/**" - "getting_started/**" + - utils/** diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..765194649 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,52 @@ +[report] +exclude_lines = + pragma: no cover + def __repr__ + if self.debug: + if settings.DEBUG + raise AssertionError + raise NotImplementedError + if 0: + if __name__ == .__main__.: + import +omit = + # omit anything in a .local directory anywhere + */.local/* + grid2op/data/* + grid2op/data_test/* + grid2op/data_grid2op/* + # omit this single file + grid2op/tests/*test*.py + grid2op/tests/*BaseBackendTest.py + grid2op/tests/*BaseRedispTest.py + *__init__.py + *config.py + dtypes.py + +[run] +exclude_lines = + pragma: no cover + def __repr__ + if self.debug: + if settings.DEBUG + raise AssertionError + raise NotImplementedError + if 0: + if __name__ == .__main__.: + import +omit = + # omit anything in a .local directory anywhere + */.local/* + # omit everything in /usr + /usr/* + grid2op/data/* + grid2op/data_test/* + grid2op/data_grid2op/* + # omit this single file + grid2op/tests/*test*.py + grid2op/tests/*BaseBackendTest.py + grid2op/tests/*BaseRedispTest.py + *__init__.py + +[coverage:report] +ignore_errors = True diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c9286822d..ec21400b6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,9 +7,13 @@ on: tags: - 'v*.*.*' +permissions: {} # deny all by default + jobs: manylinux_build: name: Build linux ${{ matrix.python.name }} wheel + permissions: + contents: read runs-on: ubuntu-latest container: quay.io/pypa/manylinux_2_28_x86_64 env: @@ -18,11 +22,11 @@ jobs: strategy: matrix: python: - - { - name: cp38, - abi: cp38, - version: '3.8', - } + # - { + # name: cp38, + # abi: cp38, + # version: '3.8', + # } - { name: cp39, abi: cp39, @@ -122,6 +126,8 @@ jobs: macos_windows_build: name: Build ${{ matrix.config.name }} ${{ matrix.python.name }} wheel + permissions: + contents: read runs-on: ${{ matrix.config.os }} strategy: matrix: @@ -135,10 +141,10 @@ jobs: os: windows-2022, } python: - - { - name: cp38, - version: '3.8', - } + # - { + # name: cp38, + # version: '3.8', + # } - { name: cp39, version: '3.9', @@ -215,6 +221,8 @@ jobs: source_build: name: Build tar.gz grid2op file on ${{ matrix.config.name}} + permissions: + contents: read runs-on: ${{ matrix.config.os }} strategy: matrix: @@ -271,6 +279,8 @@ jobs: auto_class_in_file: name: Test ${{ matrix.config.name }} OS can handle automatic class generation for python ${{matrix.python.version}} + permissions: + contents: read runs-on: ${{ matrix.config.os }} strategy: matrix: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..bfeed86ec --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,98 @@ +name: Publish to PyPI + +on: + push: + tags: + - "v*" # triggers on v1.12.3, v2.0.0, etc. + +# Least privilege by default: every job gets read-only contents and nothing +# else. The publish job re-declares the extra scopes it actually needs. +permissions: + contents: read + +jobs: + build: + name: Build distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # 6.2.0 + with: + python-version: "3.12" + - name: Install build + run: pip install build + - name: Build sdist and wheel + run: python -m build + - name: Upload build artifacts + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # 7.0.1 + with: + name: dist + path: dist/ + + sbom: + name: Generate SBOM + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # 6.2.0 + with: + python-version: "3.12" + - name: Download build artifacts + uses: actions/download-artifact@ad191675b41f6a5b46da9a048cb6893812da158b # 8.0.1 + with: + name: dist + path: dist/ + - name: Generate SBOM (Python dependencies) + run: | + # Install the freshly built wheel into an ISOLATED venv so the SBOM + # captures only the package's own resolved dependencies, and not the + # SBOM tool itself or anything else present on the runner. + python -m venv .sbom-venv + .sbom-venv/bin/pip install --upgrade pip + .sbom-venv/bin/pip install dist/*.whl + + # Install cyclonedx-bom in the runner's default environment and point + # it at the isolated venv's interpreter. Because the tool lives in a + # different environment than the one being scanned, it never appears + # as a component in the generated bill of materials. + pip install cyclonedx-bom + + TAG="${GITHUB_REF_NAME}" # e.g. v1.12.3 + PKG="${GITHUB_REPOSITORY##*/}" # e.g. grid2op + cyclonedx-py environment .sbom-venv/bin/python \ + --output-format json \ + --outfile "${PKG}-${TAG}-sbom.cdx.json" + - name: Upload SBOM artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # 7.0.1 + with: + name: sbom + path: "*-sbom.cdx.json" + + publish: + name: Publish to PyPI + needs: [build, sbom] + runs-on: ubuntu-latest + environment: pypi # must match the environment name from the Trusted Publisher config + permissions: + contents: write # needed to attach assets to the GitHub Release + id-token: write # needed for OIDC trusted publishing + Sigstore + steps: + - name: Download build artifacts + uses: actions/download-artifact@ad191675b41f6a5b46da9a048cb6893812da158b # 8.0.1 + with: + name: dist + path: dist/ + - name: Download SBOM + uses: actions/download-artifact@ad191675b41f6a5b46da9a048cb6893812da158b # 8.0.1 + with: + name: sbom + path: sbom/ + - name: Publish to PyPI (with Sigstore attestations) + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b + with: + attestations: true # this is what triggers Sigstore signing + - name: Attach SBOM to GitHub Release + uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda + with: + files: sbom/*-sbom.cdx.json diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 000000000..aecd19beb --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,78 @@ +# This workflow uses actions that are not certified by GitHub. They are provided +# by a third-party and are governed by separate terms of service, privacy +# policy, and support documentation. + +name: Scorecard supply-chain security +on: + # For Branch-Protection check. Only the default branch is supported. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection + branch_protection_rule: + # To guarantee Maintained check is occasionally updated. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained + schedule: + - cron: '25 0 * * 5' + push: + branches: [ "master" ] + +# Declare default permissions as read only. +permissions: {} # deny all by default + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + # `publish_results: true` only works when run from the default branch. conditional can be removed if disabled. + if: github.event.repository.default_branch == github.ref_name || github.event_name == 'pull_request' + permissions: + # Needed to upload the results to code-scanning dashboard. + security-events: write + # Needed to publish results and get a badge (see publish_results below). + id-token: write + # Uncomment the permissions below if installing in a private repository. + contents: read + actions: read + + steps: + - name: "Checkout code" + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: "Run analysis" + uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 + with: + results_file: results.sarif + results_format: sarif + # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: + # - you want to enable the Branch-Protection check on a *public* repository, or + # - you are installing Scorecard on a *private* repository + # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional. + # repo_token: ${{ secrets.SCORECARD_TOKEN }} + + # Public repositories: + # - Publish results to OpenSSF REST API for easy access by consumers + # - Allows the repository to include the Scorecard badge. + # - See https://github.com/ossf/scorecard-action#publishing-results. + # For private repositories: + # - `publish_results` will always be set to `false`, regardless + # of the value entered here. + publish_results: true + + # (Optional) Uncomment file_mode if you have a .gitattributes with files marked export-ignore + # file_mode: git + + # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF + # format to the repository Actions tab. + - name: "Upload artifact" + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + # Upload the results to GitHub's code scanning dashboard (optional). + # Commenting out will disable upload of results to your repo's Code Scanning dashboard + - name: "Upload to code-scanning" + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: results.sarif diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 000000000..e89f67572 --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,16 @@ +name: Security checks + +on: [push, pull_request] + +jobs: + detect-secrets: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - run: pip install detect-secrets + - run: detect-secrets scan --baseline .secrets.baseline diff --git a/.gitignore b/.gitignore index 5e9b4a29a..c43935104 100644 --- a/.gitignore +++ b/.gitignore @@ -123,8 +123,13 @@ celerybeat-schedule # pipenv Pipfile.lock -# dotenv +# dotenv, secrets etc. .env +.env.* +*.key +*.pem +*.p12 +*.pfx # virtualenv venv/ @@ -437,6 +442,12 @@ saved_setup.cfg saved_setup.py requirements_pp2.txt requirements_pp3.txt +Grid2Op-*.tar.gz +*-py3-none-any.whl +grid2op/data/**/__init__.py +grid2op/tests/aux_test_baseAction_extended.py +release_digests.json + # profiling files **.prof diff --git a/.mailmap b/.mailmap new file mode 100644 index 000000000..336a8a8d1 --- /dev/null +++ b/.mailmap @@ -0,0 +1,5 @@ +# Canonical name +Benjamin DONNOT +BDonnot +Eva BOGUSLAWSKI +Xavier WEISS diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..8bc598273 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,22 @@ +repos: + - repo: https://github.com/Yelp/detect-secrets + rev: 68e8b45440415753fff70a312ece8da92ba85b4a # v1.5.0 # latest stable on 2026-05-20 + hooks: + - id: detect-secrets + args: ['--baseline', '.secrets.baseline'] + exclude: package.lock.json + stages: [pre-commit] + + - repo: https://github.com/jumanjihouse/pre-commit-hooks + rev: 3.0.0 + hooks: + - id: check-mailmap + stages: [pre-commit] + + - repo: local + hooks: + - id: dco-signoff + name: DCO Signed-off-by check + language: python + entry: python scripts/check_dco_msg.py + stages: [commit-msg] diff --git a/.secrets.baseline b/.secrets.baseline new file mode 100644 index 000000000..7d42e88ea --- /dev/null +++ b/.secrets.baseline @@ -0,0 +1,160 @@ +{ + "version": "1.5.0", + "plugins_used": [ + { + "name": "ArtifactoryDetector" + }, + { + "name": "AWSKeyDetector" + }, + { + "name": "AzureStorageKeyDetector" + }, + { + "name": "Base64HighEntropyString", + "limit": 4.5 + }, + { + "name": "BasicAuthDetector" + }, + { + "name": "CloudantDetector" + }, + { + "name": "DiscordBotTokenDetector" + }, + { + "name": "GitHubTokenDetector" + }, + { + "name": "GitLabTokenDetector" + }, + { + "name": "HexHighEntropyString", + "limit": 3.0 + }, + { + "name": "IbmCloudIamDetector" + }, + { + "name": "IbmCosHmacDetector" + }, + { + "name": "IPPublicDetector" + }, + { + "name": "JwtTokenDetector" + }, + { + "name": "KeywordDetector", + "keyword_exclude": "" + }, + { + "name": "MailchimpDetector" + }, + { + "name": "NpmDetector" + }, + { + "name": "OpenAIDetector" + }, + { + "name": "PrivateKeyDetector" + }, + { + "name": "PypiTokenDetector" + }, + { + "name": "SendGridDetector" + }, + { + "name": "SlackDetector" + }, + { + "name": "SoftlayerDetector" + }, + { + "name": "SquareOAuthDetector" + }, + { + "name": "StripeDetector" + }, + { + "name": "TelegramBotTokenDetector" + }, + { + "name": "TwilioKeyDetector" + } + ], + "filters_used": [ + { + "path": "detect_secrets.filters.allowlist.is_line_allowlisted" + }, + { + "path": "detect_secrets.filters.common.is_baseline_file", + "filename": ".secrets.baseline" + }, + { + "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", + "min_level": 2 + }, + { + "path": "detect_secrets.filters.heuristic.is_indirect_reference" + }, + { + "path": "detect_secrets.filters.heuristic.is_likely_id_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_lock_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_potential_uuid" + }, + { + "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" + }, + { + "path": "detect_secrets.filters.heuristic.is_sequential_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_swagger_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_templated_secret" + } + ], + "results": { + ".pre-commit-config.yaml": [ + { + "type": "Hex High Entropy String", + "filename": ".pre-commit-config.yaml", + "hashed_secret": "86242b7a7b67c1fd83514757a6b319602d648e94", + "is_verified": false, + "line_number": 3, + "is_secret": false + } + ], + "grid2op/tests/test_MakeEnv.py": [ + { + "type": "Hex High Entropy String", + "filename": "grid2op/tests/test_MakeEnv.py", + "hashed_secret": "9c2afd62cc79a55cd587f91cf09311ad7096e43f", + "is_verified": false, + "line_number": 760, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "grid2op/tests/test_MakeEnv.py", + "hashed_secret": "a7795b1601032fae09df4084d6e332b64f8757ba", + "is_verified": false, + "line_number": 765, + "is_secret": false + } + ] + }, + "generated_at": "2026-05-21T12:31:08Z" +} diff --git a/.sonarcloud.properties b/.sonarcloud.properties new file mode 100644 index 000000000..afaf3c7a6 --- /dev/null +++ b/.sonarcloud.properties @@ -0,0 +1,17 @@ +sonar.organization=grid2op +sonar.projectKey=Grid2op_grid2op +# Path to sources +sonar.sources=grid2op +#sonar.exclusions= +#sonar.inclusions= + +# Path to tests +#sonar.tests= +#sonar.test.exclusions= +#sonar.test.inclusions= + +# Source encoding +#sonar.sourceEncoding=UTF-8 + +# Exclusions for copy-paste detection +#sonar.cpd.exclusions= \ No newline at end of file diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 20f03d64a..73b1c4860 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -100,6 +100,19 @@ Native multi agents support: - add detachment - add change_bus / set_bus +[1.12.5] - 2026-05-xx +------------------------- +- [FIXED] license issues in the documentation +- [FIXED] remove the "assert" outside of the tests +- [FIXED] some security quality issue spotted by codeQL (I/O function call with non totally checked user input) +- [FIXED] license issues in the documentation +- [FIXED] remove the "assert" outside of the tests +- [ADDED] automatic release on pypi +- [IMPROVED] clarifying the CONTRIBUTING.md document +- [IMPROVED] added pre commit hooks to detect potential secret push on github +- [IMPROVED] add a SECRETS_MANAGEMENTS.md file to explicitely detail how secret should be + handled by commiters +- [IMPROVED] tests splitting to reduce the duration on circle ci [1.12.4] - 2026-04-28 ---------------------- diff --git a/COMMITTERS.csv b/COMMITTERS.csv new file mode 100644 index 000000000..e66c4d7b2 --- /dev/null +++ b/COMMITTERS.csv @@ -0,0 +1,16 @@ +Name,Company,Email,GitHub,status +Benjamin Donnot,RTE,benjamin.donnot@rte-france.com,BDonnot,Active +Jean Grizet,RTE,jean.grizet@gmail.com,Tezirg,Emeritus +Vincent Renault,Artelys,vincent.renault@artelys.com,vinault,Emeritus +Laure Crochepierre,RTE,laure.crochepierre@rte-france.com,laure-crochepierre,Emeritus +Antoine Marot,RTE,antoine-marot@live.fr,marota,Emeritus +Loïc Omnes,RTE,loic.omnes@free.fr,Ashargin,Emeritus +Clément Goubet,RTE,clemgoub@gmail.com,GoubetClem,Emeritus +Xavier Weiss,KTH,xavierw@kth.se,DEUCE1957,Active +Mario Jothy,Artelys,mario.jothy@artelys.com,mjothy,Emeritus +Maxime Mohandi,Artelys,maxime.mohandi@artelys.com,mmohandi,Emeritus +Jochen Cremer,RTE,32834884+JochenC@users.noreply.github.com,JochenC,Emeritus +Jérôme Dejaegher,RTE,jerome.dejaegher@rte-france.com,jeromedejaegher,Emeritus +Farid Najar,RTE,farid.najar98@gmail.com,Farid-Najar,Emeritus +Frank Marten,Fraunhofer IEE,frank.marten@iee.fraunhofer.de,fmarten,Emeritus +Eva Boguslawski,,eva.boguslawski@gmail.com,EBoguslawski,Active \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d5928bdcf..02c34cb2f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,67 +1,384 @@ -# Contribution +# Contributing to Grid2Op -We welcome contributions from everyone. +Grid2Op is an [LF Energy](https://lfenergy.org/) project, and we welcome +contributions from everyone — power systems engineers, machine learning +researchers, software developers, students, and curious newcomers. -If you want to contribute, a good starting point is to get in touch with us through: -- github (discussion, issues or pull-request) -- the project [discord](https://discord.gg/cYsYrPT) -- mail (see current corresponding author from the grid2op package on pypi here https://pypi.org/project/Grid2Op/) +This document explains how to get involved, what the contribution workflow +looks like, and what we expect from contributions to the codebase. -Contribution can take different forms: +If anything here is unclear or out of date, please open an issue or a pull +request — improving this document is itself a valuable contribution. -- reporting bugs -- improving the documentation -- improving the code examples (notebooks or example in the doc) -- fixing some reported issues (for example at https://github.com/rte-france/Grid2Op/issues ) -- adding a new functionality to grid2op (or increase its speed) -- extend grid2op +--- -# On which topic to contribute ? +## Table of contents -If you want to contribute but are not sure were to start you can, for example: +- [Code of Conduct](#code-of-conduct) +- [Ways to contribute](#ways-to-contribute) +- [Where to start](#where-to-start) +- [Reporting bugs and security issues](#reporting-bugs-and-security-issues) +- [Code contribution workflow](#code-contribution-workflow) +- [Developer Certificate of Origin (DCO)](#developer-certificate-of-origin-dco) +- [Pre-commit hooks](#pre-commit-hooks) +- [Code style](#code-style) +- [Tests](#tests) +- [Documentation](#documentation) +- [Commit messages and pull requests](#commit-messages-and-pull-requests) +- [Becoming a committer or maintainer](#becoming-a-committer-or-maintainer) +- [License of contributions](#license-of-contributions) +- [Getting help](#getting-help) -- tackle an opened issue tagged as `good first issue` -- try to solve an opened issue marked with `helps wanted` -- there are also some contribution ideas written in the `Work "in progress"` section of the `CHANGELOG.rst` - at the top level of the github repo. +--- -In any case, if you are not sure about what is written here, feel free to ask in the grid2op [github discussion](https://github.com/orgs/Grid2op/discussions), -in the [grid2op discord](https://discord.gg/cYsYrPT) or by contacting by mail the person in charge of the [pypi package](https://pypi.org/project/Grid2Op/). +## Code of Conduct -# What to do concretely ? +All participants in the Grid2Op community are expected to follow the project's +[Code of Conduct](./CODE_OF_CONDUCT.md). By participating, you agree to uphold +a friendly, welcoming, and harassment-free environment. -For smaller changes (including, but not limited to the reporting of a bug or a contribution to the explanotory notebooks or the documentations) -a simple "pull request" with your modifications by detailing what you had in mind and the goal of your changes. +Reports of unacceptable behavior can be sent to the project maintainers via +the contacts listed in the Code of Conduct. -In case of a major change (or if you have a doubt on what is "a small change"), please open an issue first -to discuss what you would like to change and then follow as closely as possible the guidelines below. This is to ensure -first that no other contribution is this direction is being currently made but also to make sure that it -does not conflicts with some core ideas of the project. +--- -# Guidelines for contributing to the codebase +## Ways to contribute -For larger contributions, you can follow the given : +Contributions don't have to be code. All of the following are valuable: -1. fork the repository located at -2. synch your fork with the "latest developement branch of grid2op". For example, if the latest grid2op release - on pypi is `1.10.3` you need to synch your repo with the branch named `dev_1.10.4` or `dev_1.11.0` (if - the branch `dev_1.10.4` does not exist). It will be the highest number in the branches `dev_*` on - grid2op official github repository. -3. implement your functionality / code your modifications, add documentation or any kind of contribution -4. make sure to add tests and documentation if applicable -5. once it is developed, synch your repo with the last development branch again (see point 2 above) and - make sure to solve any possible conflicts -6. write a pull request and make sure to target the right branch (the "last development branch") +- Reporting bugs and reproducing issues filed by others. +- Improving the documentation, including docstrings and the + [official docs](https://grid2op.readthedocs.io/). +- Improving or adding example notebooks in `getting_started/` and `examples/`. +- Fixing issues — especially those tagged + [`good first issue`](https://github.com/Grid2op/grid2op/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) + or [`help wanted`](https://github.com/Grid2op/grid2op/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22). +- Adding new functionality, optimizations, or new backends or type of observation. +- Reviewing pull requests opened by other contributors. +- Helping users on the project [Discord](https://discord.gg/cYsYrPT) and + [GitHub Discussions](https://github.com/orgs/Grid2op/discussions). +- Writing blog posts, tutorials, or talks about Grid2Op. -# When will it be merged ? +--- -A contribution will be merged once approved by the maintainers (this is why it is recommended to first -get in touch with the team) +## Where to start -Code in the contribution should pass all the current tests, have some dedicated tests for the new feature (if applicable) -and documentation (if applicable). +If you want to contribute code but aren't sure where to begin: -If you contribute to some grid2op notebooks, please make sure to "clear all outputs" -before making the pull request. +- Browse issues labeled + [`good first issue`](https://github.com/Grid2op/grid2op/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) + and [`help wanted`](https://github.com/Grid2op/grid2op/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22). +- Check the **Work "in progress"** section at the top of + [`CHANGELOG.rst`](./CHANGELOG.rst) for ideas the maintainers have flagged. +- Ask in [Discussions](https://github.com/orgs/Grid2op/discussions) or on + [Discord](https://discord.gg/cYsYrPT) — we're happy to help you find + something that matches your interests and skill level. -New contributions should respect the "**Code of Conduct**" of grid2op. +For larger contributions, **please open an issue first** to discuss the design +before writing code. This avoids duplicate work and ensures the change fits +the project's direction. + +--- + +## Reporting bugs and security issues + +### Regular bugs + +Open a [GitHub issue](https://github.com/Grid2op/grid2op/issues) and include: + +- Grid2Op version (`pip show grid2op`). +- Python version and operating system. +- Backend in use (PandaPower, lightsim2grid, pypowsybl2grid, custom). +- A minimal, reproducible code example. +- The full traceback or unexpected behavior, with what you expected instead. + +### Security vulnerabilities + +**Do not open a public issue for security vulnerabilities.** Follow the +disclosure process described in [`SECURITY.md`](./SECURITY.md). The +maintainers will coordinate a fix and a coordinated disclosure. + +--- + +## Code contribution workflow + +1. **Fork** the repository at + [`https://github.com/Grid2op/grid2op`](https://github.com/Grid2op/grid2op). + +2. **Identify the active development branch.** Grid2Op uses a `dev_X.Y.Z` + convention. The active development branch is the one with the highest + `dev_*` number on the official repository — it targets the next release. + For example, if the latest release on PyPI is `1.12.2`, the active + development branch will typically be `dev_1.12.3` or `dev_1.13.0`. + +3. **Create a topic branch** off that development branch in your fork + (e.g. `fix/observation-typing` or `feature/new-reward`). Do not work + directly on `master` or on `dev_*` branches in your fork. + +4. **Set up a development install:** + + ```bash + git clone https://github.com//grid2op.git + cd grid2op + git remote add upstream https://github.com/Grid2op/grid2op.git + pip install -e .[optional,docs] + ``` + +5. **Implement your changes**, including: + - Tests for new functionality or bug fixes (see [Tests](#tests)). + - Updated documentation if your change affects user-facing behavior + (see [Documentation](#documentation)). + - An entry in `CHANGELOG.rst` under the appropriate section. + +6. **Sign your commits** with the Developer Certificate of Origin + (see [DCO](#developer-certificate-of-origin-dco)). + +7. **Sync with the latest development branch** before opening the PR and + resolve any conflicts: + + ```bash + git fetch upstream + git rebase upstream/dev_X.Y.Z + ``` + +8. **Open a pull request** targeting the active `dev_X.Y.Z` branch (not + `master`). Describe what you changed, why, and link any relevant issues. + +--- + +## Developer Certificate of Origin (DCO) + +Grid2Op uses the [Developer Certificate of Origin](https://developercertificate.org/) +to certify that contributors have the right to submit the code they contribute. +This is the same lightweight mechanism used by the Linux kernel and most +other Linux Foundation projects. + +**Every commit must be signed off.** Add a `Signed-off-by` line to your +commit message by using the `-s` flag with `git commit`: + +```bash +git commit -s -m "Fix typo in observation docstring" +``` + +This produces a trailer in the commit message like: + +``` +Signed-off-by: Jane Doe +``` + +The name and email must match the values configured in +`git config user.name` and `git config user.email`. By signing off, you +certify the statements in the [DCO text](https://developercertificate.org/). + +A DCO check runs automatically on every pull request. PRs with unsigned +commits cannot be merged. If you forget to sign off, you can amend the +last commit with `git commit --amend -s` or rewrite history with +`git rebase --signoff ` for a series of commits. + +--- + +## Pre-commit hooks + +Grid2Op uses [pre-commit](https://pre-commit.com/) to run security and style checks +automatically before each commit. The hooks cover two stages: + +- **pre-commit stage** — [`detect-secrets`](https://github.com/Yelp/detect-secrets) + prevents accidentally committing API keys, tokens, or passwords, as required by the + project's [Secrets Management Policy](./SECRETS_MANAGEMENT.md); + [`check-mailmap`](https://github.com/jumanjihouse/pre-commit-hooks) checks author + identity against `.mailmap`. +- **commit-msg stage** — a Python script (`scripts/check_dco_msg.py`) verifies that + every commit message contains a `Signed-off-by` trailer (see + [DCO](#developer-certificate-of-origin-dco)). Python is the only runtime + dependency for this check — no extra tools are needed beyond what you already have. + +Setting up the hooks is a one-time step and takes about two minutes. + +### Installation + +Install the required tools into your development virtual environment (the same one +you use for Grid2Op itself): + +```bash +pip install pre-commit detect-secrets +``` + +### First-time setup + +Run these two commands once after cloning (or after the hooks were first added to the +repo). Both are required: `pre-commit install` covers the file-level hooks, while the +`--hook-type commit-msg` flag activates the DCO check, which runs at the commit-msg +stage. + +```bash +pre-commit install # pre-commit and mailmap hooks +pre-commit install --hook-type commit-msg # DCO Signed-off-by check +``` + +### Normal workflow + +Once installed, the hooks run automatically on every `git commit`. You will see +output like: + +``` +detect-secrets...............................................Passed +``` + +If a hook fails, the commit is blocked and the problem is described. Fix it, then +`git add` and retry. + +To run all hooks manually without committing (useful before opening a PR): + +```bash +pre-commit run --all-files +``` + +### Updating the baseline after a false positive + +If a new file triggers a `detect-secrets` false positive (e.g. a test fixture +containing a dummy token string), update the baseline: + +```bash +detect-secrets scan --baseline .secrets.baseline +detect-secrets audit .secrets.baseline # mark the new finding as a false positive +git add .secrets.baseline +git commit -s -m "chore: update secrets baseline" +``` + +### Skipping hooks in exceptional cases + +You can bypass hooks with `git commit --no-verify`, but **note that +`detect-secrets` also runs in CI** — a skipped local check will still be caught +on the pull request. Only skip if you have a specific, time-sensitive reason, +and follow up with a proper fix. + +--- + +## Code style + +We aim for code that is readable, well-typed where practical, and +consistent with the rest of the codebase. + +- **Formatting:** follow the style of existing code in the file you are + modifying. PEP 8 conventions apply by default. +- **Type hints:** encouraged for new code, especially in public APIs. +- **Docstrings:** use the Sphinx / reST format already used throughout the + codebase. Public classes, functions, and methods must be documented. +- **Imports:** group standard library, third-party, then Grid2Op imports. +- **No commented-out code** in submitted PRs. Use git history if you need + to recover something later. + +The C++ code in dependent backends (e.g. `lightsim2grid`) follows the +conventions documented in those repositories. + +--- + +## Tests + +Grid2Op uses Python's standard `unittest` framework. The full suite lives +under `grid2op/tests/`. + +To run the tests locally: + +```bash +pip install -e .[optional] +cd grid2op/tests +python3 -m unittest discover +``` + +For new contributions: + +- **Bug fixes** must include a test that fails before the fix and passes + after. +- **New features** must include tests covering the new behavior, including + edge cases. +- Tests should be deterministic. Seed any random number generators. +- Avoid tests that require network access or external services. + +The full continuous integration suite runs on Linux with the latest NumPy +on Python 3.12. Smaller subsets run on Windows and macOS, and across +Python 3.8 through 3.13. Please make sure your changes do not regress +support for the supported Python versions documented in the README. + +--- + +## Documentation + +User-facing documentation lives in `docs/` and is built with Sphinx. + +To build the docs locally: + +```bash +pip install -e .[docs] +make html # on Linux/macOS +# or: +sphinx-build -b html docs documentation # any platform +``` + +The rendered output is placed in `documentation/html/index.html`. + +When contributing: + +- New public APIs must be added to the relevant `.rst` file in `docs/`. +- Code examples in docs and notebooks should be runnable as written. +- For Jupyter notebooks, **clear all outputs** before committing + (`jupyter nbconvert --clear-output --inplace your_notebook.ipynb`). + +--- + +## Commit messages and pull requests + +- Use clear, present-tense commit subjects (e.g. `Fix race in Runner.run`, + not `Fixed a race`). +- Keep the subject under ~72 characters; expand in the body if needed. +- Reference issues with `Fixes #123` or `Refs #123` in the commit body or + PR description. +- Keep PRs focused. A PR that fixes a typo, adds a feature, and refactors + three modules is harder to review than three separate PRs. +- Mark PRs as **Draft** if they are not ready for review. + +--- + +## Becoming a committer or maintainer + +Grid2Op is governed by a Technical Steering Committee (TSC) as documented +in [`GOVERNANCE.md`](./GOVERNANCE.md). Current committers and emeritus +members are listed in [`COMMITTERS.csv`](./COMMITTERS.csv). + +Contributors who consistently provide high-quality contributions — +whether code, reviews, documentation, or community support — may be +invited to become committers by a vote of the TSC. There is no fixed +number of contributions required; we look for sustained engagement, +good technical judgment, and alignment with the project's values. + +If you are interested in deeper involvement, the best path is to start +contributing regularly and reviewing other people's pull requests. + +--- + +## License of contributions + +Grid2Op is licensed under the +[Mozilla Public License 2.0 (MPL-2.0)](./LICENSE.md). + +By submitting a contribution, you agree that your contribution will be +licensed under MPL-2.0 (inbound = outbound). Combined with the DCO +sign-off, this is the only licensing step required for contributions. + +If your contribution includes code from another source, make sure that +source is compatible with MPL-2.0 and that the third-party license is +properly attributed in `LicensesInformation.md`. + +--- + +## Getting help + +- **Questions and discussion:** + [GitHub Discussions](https://github.com/orgs/Grid2op/discussions) +- **Real-time chat:** [Grid2Op Discord](https://discord.gg/cYsYrPT) +- **Bug reports and feature requests:** + [GitHub Issues](https://github.com/Grid2op/grid2op/issues) +- **Direct contact:** see the corresponding author on the + [PyPI page](https://pypi.org/project/Grid2Op/). + +Thanks for taking the time to contribute to Grid2Op! \ No newline at end of file diff --git a/GOVERNANCE.md b/GOVERNANCE.md new file mode 100644 index 000000000..f9af7af8a --- /dev/null +++ b/GOVERNANCE.md @@ -0,0 +1,323 @@ +# Grid2Op — Governance Document + +*[LF Energy Foundation](https://lfenergy.org/)* + +> **NOTE:** This document provides the governance structure for the Grid2Op +> project hosted by LF Energy Foundation. All participants are bound by the +> project's [Code of Conduct](https://github.com/Grid2op/grid2op/blob/master/CODE_OF_CONDUCT.md). + +--- + +## 1. Overview + +This project aims to be governed in a transparent, accessible way for the +benefit of the community. All participation in this project is open and not +bound to corporate affiliation. Participants are bound by the project's +[Code of Conduct](https://github.com/Grid2op/grid2op/blob/master/CODE_OF_CONDUCT.md). + +--- + +## 2. Ecosystem Structure + +The Grid2Op ecosystem is organized into two categories of packages: + +### 2.1 Core Packages + +Core packages are well-maintained, well-documented, and extensively tested. +They form the foundation of the Grid2Op ecosystem that the community can rely +on and reuse. The current Core packages are: + +- grid2op +- lightsim2grid + +Core packages follow the full governance process described in this document. +All Core repositories fulfill the OpenSSF Best Practices badge requirements. +Unless stated otherwise, each "Core" package is independently versioned and released. + +### 2.2 Example / Companion Packages + +Example packages are experimental or companion contributions that demonstrate +usage of Core packages. They may include: + +- Computation engines (e.g. pypowsybl2grid) +- Baselines solving L2RPN competitions +- Example UX or HMI implementations +- Companion packages (e.g. chronix2grid for data generation) +- External contributions from research papers + +Example packages have lighter governance requirements, acknowledging their +experimental nature. Contributors and users should be aware that these packages +may not be maintained to the same standard as Core packages. + +### 2.3 Archive Policy + +An Example package that is no longer maintained may be archived by a TSC +majority vote, or upon request by the project Lead. When a repository is +archived: + +- The repository description is updated to explicitly note that the project is + not maintained. +- The archival date is recorded. +- The last known compatible versions of Core packages are documented. + +### 2.4 Adoption into Core + +An Example package that has matured — demonstrating sustained maintenance, +broad community use, and alignment with the project's quality standards — may +be adopted into Core. This process is intentionally lightweight to encourage +contribution while ensuring the quality bar for Core packages is upheld. + +**Criteria for adoption** + +A package may be considered for adoption when it meets all of the following: + +- It is actively maintained, with timely responses to issues and pull requests. +- It has comprehensive documentation and test coverage comparable to existing + Core packages. +- It fulfills, or has a credible short-term plan to fulfill, the OpenSSF Best + Practices badge requirements. +- It is demonstrably used by members of the community beyond its original + authors. +- Its maintainers are willing to commit to the Committer responsibilities + defined in Section 4.2. + +**Adoption process** + +- The package maintainer (or any TSC member on their behalf) opens a GitHub + issue in the project's governance repository proposing the adoption and + summarising how the criteria above are met. +- The TSC reviews the proposal, inviting the maintainer to present at a TSC + meeting if needed. +- Adoption is approved by a majority vote of the TSC. +- Upon approval, the repository is moved under the Core governance process: it + is added to the list of Core packages in this document, its committers are + added to `COMMITTERS.csv`, and it inherits all Core requirements (OpenSSF + badge, SBOM on release, SCA scans, etc.). + +> **Note:** There is no automatic adoption route. A package remaining in the +> Example category indefinitely is perfectly acceptable; adoption should only +> happen when the community and the maintainers are both ready. + +--- + +## 3. OpenSSF Best Practices Badge + +Grid2Op maintains a single OpenSSF Best Practices badge covering the entire +Grid2Op organization. + +Each Core repository is expected to fulfill the badge +requirements. + +Example / Companion repositories are encouraged to inherit processes and +policies from the Core project, but are not strictly required to do so in +early stages of development. + +--- + +## 4. Project Roles + +### 4.1 Contributor + +The contributor role is the starting role for anyone participating in the +project and wishing to contribute code. + +**Process for becoming a Contributor** + +- Review the Contribution Guidelines to ensure your contribution is in line + with the project's coding and styling guidelines. +- Submit your code as a Pull Request with the appropriate + [Developer Certificate of Origin (DCO)](https://developercertificate.org/) + sign-off. +- Have your submission approved by the committer(s) and merged into the + codebase. + +### 4.2 Committer + +The committer role enables the contributor to commit code directly to the +core repositories, and comes with the responsibility of being a responsible leader in +the community. + +**Process for becoming a Committer** + +- Show your experience with the codebase through contributions and engagement + on Grid2Op community channels. +- Request to become a committer by creating a new Pull Request that adds your + name and details to the Committers file, and request existing committers to + approve. +- After a majority of committers approve, merge the PR and tag whoever manages + GitHub permissions to update the committers team. + +**Committer Responsibilities** + +- Monitor email aliases (if any). +- Monitor Grid2Op communication channels (Slack, Discord — delayed response is + acceptable). +- Triage GitHub issues and perform pull request reviews for other committers + and the community. +- Ensure ongoing PRs are moving forward at the right pace or are closed. +- In general, be willing to spend at least 20% of one's time working on the + project (~1 business days per week). + +**Loss of Committer Status** + +If a committer is no longer interested or cannot perform the duties listed +above, they should volunteer to be moved to emeritus status. In extreme cases +this can also occur by a vote of the committers per the voting process below. + +### 4.3 Technical Steering Committee (TSC) + +The TSC acts as the guiding committee for the Grid2Op LF Energy Foundation +community, providing governance and oversight, and ensuring that the project +adheres to all of LF's guidelines and stipulations. Serving on the TSC is a +distinct role from the Committer role, but community members may serve in both +roles simultaneously. + +**Process for becoming a TSC Member** + +- Show your experience and engagement in the broader AI and energy community, + and with the Grid2Op community specifically via contributions and engagement + on Grid2Op community channels. +- Express your interest by opening a GitHub issue in the project's governance + repository requesting TSC membership. +- New TSC members will be confirmed by a majority vote of the existing TSC + members. + +**TSC Member Responsibilities** + +- Attend periodic meetings, held likely monthly but at least quarterly. +- Actively participate in discussions and activities surrounding project + governance, oversight, milestones, and monitoring — including synchronous + participation during meetings and asynchronous participation in between + meetings. +- TSC members are expected to contribute on a best-effort basis; there is no + fixed time commitment, but consistent engagement is expected. + +**Loss of TSC Status** + +If a TSC member is no longer interested or cannot perform the TSC member +duties listed above, they should volunteer to be moved to emeritus status. In +extreme cases this can also occur by a vote of the remaining TSC members per +the voting process below. + +### 4.4 Lead + +The TSC members will elect a Lead (and optionally a Co-Lead) who will be the +primary point of contact for the project and representative to the TAC. The +Lead is responsible for the overall project health and direction, coordination +of activities, and working with other projects and committees for the continued +growth of the project. + +The Lead role has no fixed term. Re-election or replacement occurs if the +current Lead steps down, is no longer able to fulfill the role, or upon a +majority vote of the TSC. + +### 4.5 Collaborator Review Policy + +**Review Process** + +Before any contributor is granted escalated permissions, the following steps +must be completed: + +- **Identity verification.** The contributor's identity must be traceable to a + known, trusted entity. Acceptable evidence includes: a verifiable + organizational email address; a consistent history of DCO-signed commits + under a stable identity; or known association with an organization already + active in the Grid2Op community. +- **Contribution history review.** The existing committers and/or TSC must + review the candidate's prior contributions for code quality, responsiveness + to review, and alignment with project values. +- **Explicit approval vote.** A majority vote of the existing committers (for + committer access) or TSC members (for TSC-level or infrastructure access) is + required before permissions are granted. +- **Least-privilege assignment.** Permissions are scoped to the minimum + necessary. Repository write access, merge rights, and secret access are + granted separately and only as each becomes justified. +- **Recorded decision.** The granting of escalated permissions must be + documented — typically via the merged pull request that adds the person to + `COMMITTERS.csv` or an equivalent record — so the decision is auditable. + +**Secrets and Infrastructure Access** + +Access to project secrets (PyPI tokens, CI signing keys, cloud infrastructure) +is restricted to a named subset of committers. Granting or revoking this +access requires TSC approval and must be logged in the project's internal +access register. + +**Periodic Review** + +The TSC reviews the list of active committers and their permission levels at +least once per year. Committers who have been inactive for 12 months or more +will be contacted; absent a response, they will be moved to emeritus status +and their permissions revoked. + +--- + +## 5. Release Process + +Project releases are made on a rolling/ad hoc basis, as agreed upon by the +committers based on the readiness of features and fixes. There is no fixed +release cadence. + +The project follows Semantic Versioning (`major.minor.patch`): + +- **MAJOR** — incompatible API changes. +- **MINOR** — new backward-compatible functionality. +- **PATCH** — backward-compatible bug fixes. + +Release criteria, changelogs, tagging procedures, and per-repository release +checklists (including supply-chain requirements such as Software Bills of +Materials) are documented in the `RELEASE` file of the respective repository. + +Each Core repository must pass a Software Composition Analysis (SCA) scan +with no unaddressed critical or high-severity findings prior to any release. + +--- + +## 6. Conflict Resolution and Voting + +In general, we prefer that technical issues, committer membership, and TSC +membership are amicably worked out between the persons involved. If a dispute +cannot be decided independently, the committers and/or TSC members can be +called in to decide an issue. If the committers and/or TSC members themselves +cannot decide an issue, the issue will be resolved by voting. + +The voting process is a simple majority in which each committer and/or TSC +member receives one vote. + +--- + +## 7. Code of Conduct + +This project follows the +[LF Energy Code of Conduct](https://lfprojects.org/policies/code-of-conduct/). +All participants are expected to uphold this code. + +Please report unacceptable +behavior to the project committers or TSC. + +--- + +## 8. Communication + +This project, like all open source, is a global community. In addition to the +Code of Conduct, this project will: + +- Keep all communication on open channels (see below). +- Be respectful of time and language differences between community members + (e.g. when scheduling meetings, for email/issue responsiveness). +- Ensure tools are able to be used by community members regardless of their + region. + +### 8.1 Communication Channels + +- **Discord:** https://discord.gg/cYsYrPT +- **Mailing list:** https://lists.lfenergy.org/g/grid2op-discussion +- **Slack:** LF Energy Slack workspace, `#grid2op` channel +- **GitHub Discussions:** https://github.com/rte-france/Grid2Op/discussions + +If you have concerns about communication challenges for this project, please +contact the committers. + +--- + +*Last updated: May 2026* \ No newline at end of file diff --git a/LicensesInformation.md b/LicensesInformation.md index c0a520fa2..38b963289 100644 --- a/LicensesInformation.md +++ b/LicensesInformation.md @@ -1,4 +1,4 @@ -Copyright (c) 2019-2020, RTE (https://www.rte-france.com) + # Project dependencies diff --git a/README.md b/README.md index 581a8203d..2a7737de5 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,11 @@ source venv_grid2op/bin/activate pip3 install grid2op ``` +### Release integrity + +See [release-verification.md](release-verification.md) for +instructions on verifying the authenticity and integrity of release assets. + ### Install from source ```commandline diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 000000000..50854d96b --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,219 @@ +# Release Process + +This document describes the release methodology, criteria, and checklist for +this repository. It applies to all Core packages in the Grid2Op ecosystem +(grid2op, lightsim2grid, pypowsybl2grid). + +--- + +## Versioning + +This project follows [Semantic Versioning](https://semver.org/) (`major.minor.patch`): + +- **MAJOR** — incompatible API changes. +- **MINOR** — new backward-compatible functionality. +- **PATCH** — backward-compatible bug fixes. + +--- + +## Release cadence + +Releases are made on a rolling/ad hoc basis, as agreed upon by the committers +based on the readiness of features and fixes. There is no fixed cadence. + +--- + +## Pre-release checklist + +Before tagging a release, the release manager must verify the following: + +### Code quality +- [ ] All CI checks pass on the target release branch (`dev_X.Y.Z`). +- [ ] No open issues tagged `release-blocker`. +- [ ] All deprecation warnings introduced in this cycle are documented. +- [ ] SCA scan passes with no unaddressed critical or high-severity findings (see [Software Composition Analysis](#software-composition-analysis-osps-vm-0502) below). + +### Documentation +- [ ] `CHANGELOG.rst` has a complete entry for this version (features, fixes, + breaking changes, deprecations). +- [ ] Public API documentation is up to date. +- [ ] Version number is updated in `pyproject.toml` (and `setup.py` / + `CMakeLists.txt` if applicable). + +### Tagging and branching +- [ ] A release PR merges `dev_X.Y.Z` into `master`. +- [ ] The release commit is tagged `vX.Y.Z` and the tag is signed. + +--- + +## Building and publishing + +### Pure-Python packages (grid2op) + +```bash +python -m build # produces sdist + wheel in dist/ +twine check dist/* # verify metadata before upload +twine upload dist/* # upload to PyPI +``` + +### Compiled packages (lightsim2grid, pypowsybl2grid) + +Binary wheels are built via the CI/CD pipeline (GitHub Actions + `cibuildwheel`) +for all supported platforms and Python versions before upload. Do not upload +manually built wheels. + +--- + +## Software Bill of Materials (OSPS-QA-02.02) + +> **Requirement:** When the project makes a release, all compiled released +> software assets **MUST** be delivered with a Software Bill of Materials +> (SBOM). [OSPS-QA-02.02] +> +> It is recommended to auto-generate SBOMs at build time using a tool that +> has been vetted for accuracy. This enables users to ingest this data in a +> standardized approach alongside other projects in their environment. + +### What this means in practice + +An SBOM is a machine-readable inventory of every component bundled in a +release artifact — direct and transitive dependencies, their versions, and +their licenses. + +**grid2op** (pure Python): generating an SBOM is good practice and satisfies +the spirit of the requirement, even though the wheel contains no compiled code. + +**lightsim2grid / pypowsybl2grid** (compiled wheels): the requirement applies +directly. Each release must ship an SBOM alongside the binary wheels, because +the compiled artifacts bundle vendored C++ libraries (Eigen, SuiteSparse/KLU, +CKTSO, etc.) that are otherwise invisible to downstream users. + +### Recommended format + +Use **CycloneDX JSON** (`application/vnd.cyclonedx+json`) or **SPDX**. Both +are widely supported by dependency-scanning tooling. + +### How to generate automatically (GitHub Actions) + +Add the following step to the release workflow, **after** the build step and +**before** the upload step: + +```yaml +- name: Generate SBOM + run: | + pip install cyclonedx-bom + # Captures the full Python environment (all installed deps) + cyclonedx-py environment --output-format json --outfile sbom.cdx.json + +- name: Attach SBOM to GitHub Release + uses: softprops/action-gh-release@v2 + with: + files: sbom.cdx.json +``` + +For **lightsim2grid**, the Python-level SBOM will not capture the vendored C++ +libraries. Supplement it by either: + +- Running `syft` on the built wheel file: + ```yaml + - uses: anchore/sbom-action@v0 + with: + path: dist/ + format: cyclonedx-json + output-file: sbom.cdx.json + ``` +- Or maintaining a hand-written `sbom-cpp-deps.cdx.json` that lists Eigen, + SuiteSparse, KLU, and CKTSO with their pinned versions, and merging it into + the final SBOM. + +### Where to publish the SBOM + +Attach `sbom.cdx.json` as a release asset on the GitHub Release page, +alongside the wheels and the source archive. Name it clearly, e.g. +`lightsim2grid-0.13.1-sbom.cdx.json`. + +--- + +## Software Composition Analysis (OSPS-VM-05.02) + +> **Requirement:** While active, the project documentation MUST include a +> policy to address SCA violations prior to any release. [OSPS-VM-05.02] + +### Policy + +No Core package may be released if any dependency has an unaddressed +critical or high-severity vulnerability (CVSS ≥ 7.0) known at the time of +release. Medium and low findings must be triaged and either remediated or +explicitly accepted with a written rationale before the release is tagged. + +An exception may be granted by a TSC majority vote when no fix is available +upstream and the vulnerability is determined to be unexploitable in the +project's use context. Exceptions must be recorded in a `security-exceptions` +section of the repository's `SECURITY.md`, with the CVE identifier, the +rationale, and an agreed review date. + +### Scope + +**grid2op** (pure Python): the SCA scan covers all direct and transitive +Python dependencies as resolved in the release environment. + +**lightsim2grid / pypowsybl2grid** (compiled wheels): the scan covers Python +dependencies **and** the vendored C++ libraries (Eigen, SuiteSparse/KLU, +CKTSO, etc.). Because `pip-audit` does not reach compiled vendored code, the +C++ dependency list must be reviewed manually against the +[OSV database](https://osv.dev/) or equivalent prior to each release. + +### Recommended tool + +Use `pip-audit` for Python dependencies: + +```bash +pip install pip-audit +pip-audit --strict # exits non-zero on any finding +``` + +`--strict` treats findings as errors. Use `--ignore-vuln ` only for +vulnerabilities covered by an accepted exception in `SECURITY.md`. + +### How to enforce automatically (GitHub Actions) + +Add a required status check to the release workflow, **before** the build +step: + +```yaml +- name: SCA — Python dependencies + run: | + pip install pip-audit + pip-audit --strict +``` + +For **lightsim2grid**, add a manual review step as a checklist item (see +below) since tooling cannot automatically scan vendored C++ libraries. + +Mark this job as a required status check in the branch protection rules for +`master` so that the release PR cannot be merged if it fails. + +### Pre-release checklist additions + +The following items are appended to the pre-release checklist above: + +- [ ] `pip-audit --strict` passes on the release environment with no + unaddressed findings. +- [ ] *(lightsim2grid / pypowsybl2grid only)* Vendored C++ libraries (Eigen, + SuiteSparse/KLU, CKTSO) reviewed against OSV or NVD; findings recorded + or accepted in `SECURITY.md`. +- [ ] Any accepted exceptions in `SECURITY.md` have been reviewed and are + still valid for this release. + +--- + +## Post-release checklist + +- [ ] GitHub Release page created with the correct tag, release notes (copied + from `CHANGELOG.rst`), and SBOM asset attached. +- [ ] PyPI page reflects the new version. +- [ ] Announcement posted to the Grid2Op mailing list and Discord. +- [ ] `dev_X.Y.(Z+1)` (or `dev_X.(Y+1).0`) branch opened for the next cycle. +- [ ] check the doc is updated in https://app.readthedocs.org and trigger + the build of the dev branch +- [ ] add the new dev branch to codacy (app.codacy.com) diff --git a/REPOSITORIES.md b/REPOSITORIES.md new file mode 100644 index 000000000..6f98472da --- /dev/null +++ b/REPOSITORIES.md @@ -0,0 +1,17 @@ +# Related Repositories + +See the full list of official subprojects at the +[Grid2Op GitHub organization](https://github.com/Grid2op). + +## Official backends +- [lightsim2grid](https://github.com/Grid2op/lightsim2grid) — optional fast C++ backend +- [pypowsybl2grid](https://github.com/Grid2op/pypowsybl2grid) — optional PowSyBl backend with high fidelity + +## Data and agents +- [l2rpn-baselines](https://github.com/Grid2op/l2rpn-baselines) — code examples allowing to use some trained agent +- [chronix2grid](https://github.com/Grid2op/chronix2grid) — package used to generate time series data for grid2op environment + + +## UX and UI +- [grid2game](https://github.com/Grid2op/grid2game) — interactive grid management tool +- [grid2viz](https://github.com/Grid2op/grid2viz) — a posteriori studies of policies / agents on grid2op environments diff --git a/SECRETS_MANAGEMENT.md b/SECRETS_MANAGEMENT.md new file mode 100644 index 000000000..6417d4091 --- /dev/null +++ b/SECRETS_MANAGEMENT.md @@ -0,0 +1,155 @@ +# Secrets and Credentials Management Policy + +**Project:** Grid2Op (LF Energy Foundation) +**Requirement:** OSPS-BR-07.02 +**Status:** Active + +--- + +## 1. Overview + +This document defines how the Grid2Op project manages secrets and credentials — including API keys, tokens, passwords, certificates, and any other sensitive information. All contributors, committers, and Technical Steering Committee (TSC) members are expected to follow this policy. + +The core principle is simple: **no secret should ever be committed to version control.** + +--- + +## 2. Scope + +This policy applies to: + +- All repositories under the Grid2Op GitHub organization +- CI/CD pipelines (GitHub Actions workflows) +- Maintainer and contributor tooling +- Any third-party service integrations (PyPI publishing, cloud testing environments, etc.) + +It covers secrets such as: + +- API keys (PyPI tokens, cloud provider keys, etc.) +- Authentication tokens (GitHub tokens, OAuth tokens) +- Passwords and passphrases +- Private keys and certificates +- Webhook secrets + +--- + +## 3. Storage + +### 3.1 What is prohibited + +The following are strictly forbidden across all repositories: + +- Hard-coding secrets in source code (`.py`, `.cpp`, `.cu`, `.h`, scripts, etc.) +- Storing secrets in configuration files checked into version control (`.env`, `config.yaml`, `settings.json`, etc.) +- Including secrets in documentation, comments, or test fixtures +- Logging secrets to standard output or error streams + +### 3.2 Approved storage locations + +| Secret type | Approved storage | +|---|---| +| CI/CD secrets (tokens, keys used in workflows) | GitHub Actions Encrypted Secrets (repository or organization level) | +| Local development credentials | Developer's local `.env` file (must be listed in `.gitignore`) | +| Shared maintainer credentials | GitHub Actions organization-level secrets, accessible only to designated workflows | + +All repositories **must** include a `.gitignore` that excludes `.env`, `.env.*`, `*.key`, `*.pem`, and similar file patterns. + +--- + +## 4. Access Control + +- **Principle of least privilege:** each secret is accessible only to the workflow or person that needs it, and nothing more. +- GitHub Actions secrets are scoped at the repository level by default. Organization-level secrets are granted only when multiple repositories legitimately need the same credential (e.g., a shared PyPI publishing token). +- Only TSC members and designated committers may add or modify secrets in the GitHub organization settings. +- Access to secrets is reviewed at each annual project review and whenever a committer or TSC member steps down (see Section 5). + +--- + +## 5. Rotation and Revocation + +### 5.1 Scheduled rotation + +- **PyPI and package publishing tokens:** rotated at least once per year, or upon a new release cycle if that is shorter. +- **Long-lived API keys:** rotated at least once per year. + +### 5.2 Event-triggered rotation + +Secrets **must** be rotated immediately when: + +- A committer or TSC member with access to secrets leaves the project or moves to emeritus status. +- A secret is accidentally exposed (committed to a repo, logged, shared in a public channel, etc.). +- A third-party service reports a breach or compromise. +- A repository is forked publicly under circumstances that may have exposed workflow secrets. + +### 5.3 Incident response + +If a secret is suspected or confirmed to have been exposed: + +1. **Revoke** the secret immediately via the relevant platform (GitHub, PyPI, etc.). +2. **Notify** the TSC and the LF Energy security contact within 24 hours. +3. **Audit** recent usage of the secret for unauthorized activity. +4. **Replace** with a new secret following the storage and access guidelines above. +5. **Document** the incident in a private TSC record. + +--- + +## 6. CI/CD Pipelines + +- Secrets are injected into GitHub Actions workflows exclusively via `${{ secrets.SECRET_NAME }}` syntax. +- Workflows must never print secret values, even for debugging. Use `::add-mask::` if a dynamic value derived from a secret needs to appear in logs. +- Pull requests from forks do **not** have access to repository secrets by default; this GitHub Actions default must not be overridden. +- Workflow files (`.github/workflows/*.yml`) are reviewed by at least one committer before merge, with attention to any `env:` blocks that could inadvertently expose secrets. + +--- + +## 7. Tiered Repository Considerations + +Consistent with the Grid2Op "Core" and "Examples" project structure: + +- **Core repositories** (grid2op, lightsim2grid, pypowsybl2grid): this full policy applies without exception. +- **Example/companion repositories**: the prohibition on hard-coded secrets and the incident response procedure apply unconditionally. Rotation schedules may be relaxed if a repository has no active CI publishing credentials, but this must be explicitly documented in that repository's README. +- **Archived repositories**: any active secrets must be revoked before archiving, and the archive notice should confirm that no live credentials are present. + +--- + +## 8. Developer Guidance + +Contributors are encouraged to: + +- Use [`detect-secrets`](https://github.com/Yelp/detect-secrets) as a local pre-commit hook to catch accidental commits (see [`CONTRIBUTING.md`](./CONTRIBUTING.md) for setup instructions). +- Store local development credentials in a `.env` file at the project root (never committed). +- Use environment variable loading libraries (e.g., `python-dotenv`) rather than hard-coding values. + +Example `.env` usage in Python: + +```python +import os +from dotenv import load_dotenv + +load_dotenv() # loads from .env file if present +api_key = os.environ["MY_API_KEY"] # raises KeyError if not set — intentional +``` + +--- + +## 9. Compliance and Enforcement + +- Compliance with this policy is a condition of maintaining committer or TSC status. +- Automated scanning (e.g., GitHub secret scanning, which is enabled on all public repositories by default) is used to detect accidental exposure. +- Any pull request found to contain secrets will be closed immediately. The contributor will be notified and asked to rotate the exposed credential before resubmitting. + +--- + +## 10. Policy Maintenance + +This document is owned by the TSC. It must be reviewed: + +- At each annual LF Energy project review +- Whenever a significant change to project infrastructure occurs +- Following any security incident + +Proposed changes follow the standard pull request process, with approval from a majority of active TSC members. + +--- + +*For questions or to report a suspected exposure, contact the TSC via the project's [mailing list](https://lists.lfenergy.org/g/grid2op-discussion) or the [Discord server](https://discord.gg/cYsYrPT).* \ No newline at end of file diff --git a/SECURITY.md b/SECURITY.md index 18c4e2412..7d516a687 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -28,3 +28,8 @@ relase is 1.11.1 (major release 1, minor release 11, patch release 1) then versi For critical, important and moderate patches will be applied for the last patch release of grid2op. Security patches of all level of severity will be implemented in the current "release train", available in the next release. + +## Release integrity + +See [docs/release-verification.md](./docs/release-verification.rst) for +instructions on verifying the authenticity and integrity of release assets. diff --git a/SUPPORTS.md b/SUPPORTS.md new file mode 100644 index 000000000..49f124e64 --- /dev/null +++ b/SUPPORTS.md @@ -0,0 +1,96 @@ +# Support + +Grid2Op is an [LF Energy](https://lfenergy.org/) project. +This document describes the support lifecycle for Grid2Op releases, including +the scope and duration of support, the types of support provided, and how to +obtain help. + +--- + +## Release support lifecycle + +Grid2Op uses a three-part version number `MAJOR.MINOR.PATCH` (e.g. `1.10.4`). +Security fixes are backported according to severity as follows: + +| Severity | Releases patched | +|---|---| +| **Critical** | All patch releases of the last minor release published within the previous calendar year, **and** all patch releases of the current minor release train | +| **Important** | All patch releases of the last minor release published within the previous calendar year, **and** all patch releases of the current minor release train | +| **Moderate** | Latest patch release of the current minor release train only | +| **Low** | Included in the next scheduled release of the current release train | +| **Any severity — end of life** | Minor releases older than the last minor release published within the previous calendar year receive **no security updates of any kind** and should be considered end-of-life. Users are strongly encouraged to upgrade. | + +**Example.** Suppose the current release is `1.11.1` and `1.10` was the last +minor release published within the previous calendar year: + +- A *critical* vulnerability found in `1.10.4` would be patched in `1.10.0`, + `1.10.1`, `1.10.2`, `1.10.3`, `1.10.4`, `1.11.0`, and `1.11.1`. +- A *moderate* vulnerability would only be patched in `1.11.1` (current latest + patch). +- Any severity is also included in the next release of the active development + branch. + +Severity is assessed using the +[Apache severity rating](https://security.apache.org/blog/severityrating/). +When in doubt, report privately — see [`SECURITY.md`](./SECURITY.md). + +> The list of currently supported versions is kept up to date in +> [`SECURITY.md`](./SECURITY.md). + +--- + +## Types of support provided + +For **supported releases**, the project aims to provide: + +- **Bug fixes** — reproducible defects that affect correct behavior. +- **Security updates** — vulnerabilities are handled under the coordinated + disclosure process described in [`SECURITY.md`](./SECURITY.md). +- **Documentation corrections** — inaccurate or missing documentation for + existing behavior. + +The following are **out of scope** for stable releases and will be deferred to +a future release: + +- New features or behavioral changes. +- Performance improvements that require API or interface changes. +- Dependency upgrades that introduce breaking changes. + +--- + +## How to obtain support + +| Channel | Purpose | +|---|---| +| [GitHub Issues](https://github.com/Grid2op/grid2op/issues) | Bug reports and confirmed defects | +| [GitHub Discussions](https://github.com/orgs/Grid2op/discussions) | Questions, usage help, and general discussion | +| [Discord](https://discord.gg/cYsYrPT) | Real-time community chat | +| [Mailing list](https://lists.lfenergy.org/g/grid2op-discussion) | Announcements and broader community discussion | + +For **security vulnerabilities**, do **not** open a public issue. Follow the +private disclosure process in [`SECURITY.md`](./SECURITY.md). + +--- + +## Supported Python and dependency versions + +Each release's `README` and release notes document the supported Python +versions (currently Python 3.9 – 3.13) and the compatible versions of key +dependencies (NumPy, PandaPower, lightsim2grid, pypowsybl2grid). Support for +older Python versions is dropped when they reach their upstream end-of-life. + +--- + +## Commercial support + +Grid2Op is a community-maintained open-source project. No commercial support +contracts are offered by the project itself. Organizations requiring guaranteed +service levels should consider engaging with contributors or LF Energy member +companies directly. + +--- + +## Questions about this document + +If anything here is unclear or out of date, please open an issue or a pull +request — improving this document is itself a valuable contribution. diff --git a/docs/_static/colors.css b/docs/_static/colors.css new file mode 100644 index 000000000..0ce050339 --- /dev/null +++ b/docs/_static/colors.css @@ -0,0 +1,130 @@ +/* + SPDX-License-Identifier: MPL-2.0 + This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. +*/ + +/*Allow to add some custom colors and text decoration. + For example, :black:`text in black` or :blink:`text blinking` in rST. */ + +.aqua { + color: aqua; +} + +.black { + color: black; +} + +.blue { + color: blue; +} + +.cyan { + color: cyan; +} + +.fuchsia { + color: fuchsia; +} + +.gray { + color: gray; +} + +.green { + color: green; +} + +.lime { + color: lime; +} + +.magenta { + color: magenta; +} + +.maroon { + color: maroon; +} + +.navy { + color: navy; +} + +.olive { + color: olive; +} + +.orange { + color: orange; +} + +.pink { + color: pink; +} + +.purple { + color: purple; +} + +.red { + color: red; +} + +.silver { + color: silver; +} + +.teal { + color: teal; +} + +.white { + color: white; +} + +.yellow { + color: yellow; +} + + +/** font type **/ + +.blink { + text-decoration: blink; +} + +.it { + font-style: italic; +} + +.large { + font-size: large; +} + +.line { + text-decoration: line-through; +} + +.ob { + font-style: oblique; +} + +.over { + text-decoration: overline; +} + +.small { + font-size: small; +} + +.smallpar { + font-size: small; +} + +.strike { + text-decoration: line-through; +} + +.under { + text-decoration: underline; +} \ No newline at end of file diff --git a/docs/_static/hacks.css b/docs/_static/hacks.css deleted file mode 100644 index a0fa73de4..000000000 --- a/docs/_static/hacks.css +++ /dev/null @@ -1,326 +0,0 @@ -/* - * CSS hacks and small modification for my Sphinx website - * :copyright: Copyright 2013-2016 Lilian Besson - * :license: GPLv3, see LICENSE for details. - */ - - -/* Colors and text decoration. - For example, :black:`text in black` or :blink:`text blinking` in rST. */ - - .black { - color: black; -} - -.gray { - color: gray; -} - -.grey { - color: gray; -} - -.silver { - color: silver; -} - -.white { - color: white; -} - -.maroon { - color: maroon; -} - -.red { - color: red; -} - -.magenta { - color: magenta; -} - -.fuchsia { - color: fuchsia; -} - -.pink { - color: pink; -} - -.orange { - color: orange; -} - -.yellow { - color: yellow; -} - -.lime { - color: lime; -} - -.green { - color: green; -} - -.olive { - color: olive; -} - -.teal { - color: teal; -} - -.cyan { - color: cyan; -} - -.aqua { - color: aqua; -} - -.blue { - color: blue; -} - -.navy { - color: navy; -} - -.purple { - color: purple; -} - -.under { - text-decoration: underline; -} - -.over { - text-decoration: overline; -} - -.blink { - text-decoration: blink; -} - -.line { - text-decoration: line-through; -} - -.strike { - text-decoration: line-through; -} - -.it { - font-style: italic; -} - -.ob { - font-style: oblique; -} - -.small { - font-size: small; -} - -.large { - font-size: large; -} - -.smallpar { - font-size: small; -} - - -/* Style pour les badges en bas de la page. */ - -div.supportBadges { - margin: 1em; - text-align: right; -} - -div.supportBadges ul { - padding: 0; - display: inline; -} - -div.supportBadges li { - display: inline; -} - -div.supportBadges a { - margin-right: 1px; - opacity: 0.6; -} - -div.supportBadges a:hover { - opacity: 1; -} - - -/* Details elements in the sidebar */ - -a.reference { - border-bottom: none; - text-decoration: none; -} - -ul.details { - font-size: 80%; -} - -ul.details li p { - font-size: 85%; -} - -ul.externallinks { - font-size: 85%; -} - - -/* Pour le drapeau de langue */ - -img.languageswitch { - width: 50px; - height: 32px; - margin-left: 5px; - vertical-align: bottom; -} - -div.sphinxsidebar { - overflow: hidden !important; - font-size: 120%; - word-wrap: break-word; - width: 300px; - max-width: 300px; -} - -div.sphinxsidebar h3 { - font-size: 125%; -} - -div.sphinxsidebar h4 { - font-size: 110%; -} - -div.sphinxsidebar a { - font-size: 85%; -} - - -/* Image style for scrollUp jQuery plugin */ - -#scrollUpLeft { - bottom: 50px; - left: 260px; - height: 38px; - width: 38px; - background: url('//perso.crans.org/besson/_images/.top.svg'); - background: url('../_images/.top.svg'); -} - -@media screen and (max-width: 875px) { - #scrollUpLeft { - right: 50px; - left: auto; - } -} - - -/* responsive for font-size. */ - -@media (max-width: 875px) { - body { - font-size: 105%; - /* Increase font size for responsive theme */ - } -} - -@media (max-width: 1480px) and (min-width: 876px) { - body { - font-size: 110%; - /* Increase font size for not-so-big screens */ - } -} - -@media (min-width: 1481px) { - body { - font-size: 115%; - /* Increase even more font size for big screens */ - } -} - - -/* Social Icons in the sidebar (available: twitter, facebook, linkedin, google+, bitbucket, github) */ - -.social-icons { - display: inline-block; - margin: 0; - text-align: center; -} - -.social-icons a { - background: none no-repeat scroll center top #444444; - border: 1px solid #F6F6F6; - border-radius: 50% 50% 50% 50%; - display: inline-block; - height: 35px; - width: 35px; - margin: 0; - text-indent: -9000px; - transition: all 0.2s ease 0s; - text-align: center; - border-bottom: none; -} - -.social-icons li { - display: inline-block; - list-style-type: none; - border-bottom: none; -} -.social-icons li a { - border-bottom: none; -} - -.social-icons a:hover { - background-color: #666666; - transition: all 0.2s ease 0s; - text-decoration: none; -} - -.social-icons a.facebook { - background-image: url('../_images/.facebook.png'); - background-image: url('//perso.crans.org/besson/_images/.facebook.png'); - display: block; - margin-left: auto; - margin-right: auto; - background-size: 35px 35px; -} - -.social-icons a.bitbucket { - background-image: url('../_images/.bitbucket.png'); - background-image: url('//perso.crans.org/besson/_images/.bitbucket.png'); - display: block; - margin-left: auto; - margin-right: auto; - background-size: 35px 35px; -} - -.social-icons li a.github { - background-image: url('../_images/.github.png'); - background-image: url('//perso.crans.org/besson/_images/.github.png'); - display: block; - margin-left: auto; - margin-right: auto; - background-size: 35px 35px; -} - -.social-icons li a.wikipedia { - background-image: url('../_images/.wikipedia.png'); - background-image: url('//perso.crans.org/besson/_images/.wikipedia.png'); - display: block; - margin-left: auto; - margin-right: auto; - background-size: 35px 35px; -} \ No newline at end of file diff --git a/docs/_static/style.css b/docs/_static/style.css index 7269f2cde..7f0813316 100644 --- a/docs/_static/style.css +++ b/docs/_static/style.css @@ -1,3 +1,8 @@ +/* + SPDX-License-Identifier: MPL-2.0 + This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. +*/ + /* this stuff uses a couple of themes as a base with some custom stuff added In particular thanks to: diff --git a/docs/available_envs.rst b/docs/available_envs.rst index f2c8da22c..8e30f9820 100644 --- a/docs/available_envs.rst +++ b/docs/available_envs.rst @@ -1,3 +1,5 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. .. |l2rpn_case14_sandbox_layout| image:: ./img/l2rpn_case14_sandbox_layout.png .. |R2_full_grid| image:: ./img/R2_full_grid.png diff --git a/docs/conf.py b/docs/conf.py index 1bcfcaef8..76211ee49 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -22,7 +22,7 @@ author = 'Benjamin Donnot' # The full version, including alpha/beta/rc tags -release = '1.12.4' +release = '1.12.5.dev0' version = '1.12' @@ -76,7 +76,7 @@ # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] -html_css_files = ['hacks.css'] +html_css_files = ['colors.css'] # for pdf pdf_documents = [('index', u'rst2pdf', u'Grid2op documentation', u'B. DONNOT'),] diff --git a/docs/data_pipeline.rst b/docs/data_pipeline.rst index 0c316667e..f78fc7cee 100644 --- a/docs/data_pipeline.rst +++ b/docs/data_pipeline.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. _environment-module-data-pipeline: Optimize the data pipeline diff --git a/docs/dive_into_time_series.rst b/docs/dive_into_time_series.rst index 5a5264996..04965c3be 100644 --- a/docs/dive_into_time_series.rst +++ b/docs/dive_into_time_series.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. _chornix2grid_github: https://github.com/bdonnot/chronix2grid .. _doc_timeseries: diff --git a/docs/final.rst b/docs/final.rst index f4aef68aa..46b39ad6c 100644 --- a/docs/final.rst +++ b/docs/final.rst @@ -1,3 +1,5 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. If you still can't find what you're looking for, try in one of the following pages: diff --git a/docs/grid2op.rst b/docs/grid2op.rst index 93f1f0e43..658a55144 100644 --- a/docs/grid2op.rst +++ b/docs/grid2op.rst @@ -1,4 +1,8 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. module:: grid2op + .. _grid2op-module: Grid2Op module diff --git a/docs/grid2op_dev.rst b/docs/grid2op_dev.rst index 3d44046de..c777bf17d 100644 --- a/docs/grid2op_dev.rst +++ b/docs/grid2op_dev.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. toctree:: :maxdepth: 1 diff --git a/docs/grid2op_dev/action.rst b/docs/grid2op_dev/action.rst index 3cf669624..d81f0a89d 100644 --- a/docs/grid2op_dev/action.rst +++ b/docs/grid2op_dev/action.rst @@ -1,3 +1,7 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + + How to add a new type of action =================================== diff --git a/docs/grid2op_dev/final.rst b/docs/grid2op_dev/final.rst index f095ba7ca..4178cbef9 100644 --- a/docs/grid2op_dev/final.rst +++ b/docs/grid2op_dev/final.rst @@ -1,2 +1,4 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. .. include:: ../final.rst diff --git a/docs/grid2op_extend.rst b/docs/grid2op_extend.rst index 1bce8296b..9cda0bb3d 100644 --- a/docs/grid2op_extend.rst +++ b/docs/grid2op_extend.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. toctree:: :maxdepth: 1 diff --git a/docs/grid2op_extend/create_an_environment.rst b/docs/grid2op_extend/create_an_environment.rst index f7f97d7c2..0b788a131 100644 --- a/docs/grid2op_extend/create_an_environment.rst +++ b/docs/grid2op_extend/create_an_environment.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. |l2rpn_case14_sandbox_layout| image:: ../img/l2rpn_case14_sandbox_layout.png .. |R2_full_grid| image:: ../img/R2_full_grid.png .. |l2rpn_neurips_2020_track1_layout| image:: ../img/l2rpn_neurips_2020_track1_layout.png diff --git a/docs/grid2op_extend/createbackend.rst b/docs/grid2op_extend/createbackend.rst index 124925f88..49ab08c61 100644 --- a/docs/grid2op_extend/createbackend.rst +++ b/docs/grid2op_extend/createbackend.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. _name_load: ./space.html#grid2op.Space.GridObjects.name_load .. _name_gen: ./space.html#grid2op.Space.GridObjects.name_gen .. _name_line: ./space.html#grid2op.Space.GridObjects.name_line diff --git a/docs/grid2op_extend/env_content.rst b/docs/grid2op_extend/env_content.rst index c4351185a..3d08c6817 100644 --- a/docs/grid2op_extend/env_content.rst +++ b/docs/grid2op_extend/env_content.rst @@ -1,3 +1,5 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. .. |l2rpn_case14_sandbox_layout| image:: ../img/l2rpn_case14_sandbox_layout.png .. |R2_full_grid| image:: ../img/R2_full_grid.png diff --git a/docs/grid2op_extend/final.rst b/docs/grid2op_extend/final.rst index f095ba7ca..4178cbef9 100644 --- a/docs/grid2op_extend/final.rst +++ b/docs/grid2op_extend/final.rst @@ -1,2 +1,4 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. .. include:: ../final.rst diff --git a/docs/grid2op_extend/observation.rst b/docs/grid2op_extend/observation.rst index 29e738660..2b4e98f42 100644 --- a/docs/grid2op_extend/observation.rst +++ b/docs/grid2op_extend/observation.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + How to add a new attribute to the observation ============================================== diff --git a/docs/grid_graph.rst b/docs/grid_graph.rst index 3d78338a5..13c54fc6c 100644 --- a/docs/grid_graph.rst +++ b/docs/grid_graph.rst @@ -1,3 +1,5 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. .. |grid_graph_1| image:: ./img/grid_graph_1.png :width: 45% diff --git a/docs/gym.rst b/docs/gym.rst index 931bb0935..74a471741 100644 --- a/docs/gym.rst +++ b/docs/gym.rst @@ -1,3 +1,5 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. .. _BaseGymSpaceConverter.add_key: ./gym.html#grid2op.gym_compat.gym_space_converter._BaseGymSpaceConverter.add_key .. _BaseGymSpaceConverter.keep_only_attr: ./gym.html#grid2op.gym_compat.gym_space_converter._BaseGymSpaceConverter.keep_only_attr diff --git a/docs/index.rst b/docs/index.rst index 6cf59ea3b..65f2560ae 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. Grid2Op documentation master file, created by sphinx-quickstart on Wed Jul 24 15:07:20 2019. You can adapt this file completely to your liking, but it should at least @@ -70,6 +73,7 @@ or overall view of what is happing when you "run" a scenario using in grid2op. quickstart grid2op + release_verification Usage examples --------------------- diff --git a/docs/makeenv.rst b/docs/makeenv.rst index f30933401..fd5fedab9 100644 --- a/docs/makeenv.rst +++ b/docs/makeenv.rst @@ -1,4 +1,8 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. currentmodule:: grid2op.MakeEnv + .. _make-env-module: Make: Using pre defined Environments diff --git a/docs/mdp.rst b/docs/mdp.rst index c889287ed..f1d2715ae 100644 --- a/docs/mdp.rst +++ b/docs/mdp.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. for the color .. include:: special.rst diff --git a/docs/model_based.rst b/docs/model_based.rst index c0d3eda9b..4b2d332f6 100644 --- a/docs/model_based.rst +++ b/docs/model_based.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. _model_based_rl: Model Based / Planning methods diff --git a/docs/model_free.rst b/docs/model_free.rst index 10424d7c7..d1f6f6a3b 100644 --- a/docs/model_free.rst +++ b/docs/model_free.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. _model_free_rl: Model Free Reinforcement Learning diff --git a/docs/modeled_elements.rst b/docs/modeled_elements.rst index 9dc4509d3..0df3351de 100644 --- a/docs/modeled_elements.rst +++ b/docs/modeled_elements.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. |5subs_grid_layout| image:: ./img/5subs_grid_layout.jpg .. |5subs_grid_5_sub1_graph| image:: ./img/5subs_grid_5_sub1_graph.jpg .. |5subs_grid_all_1| image:: ./img/5subs_grid_all_1.jpg diff --git a/docs/optimization.rst b/docs/optimization.rst index ba9407a8e..2a06a5908 100644 --- a/docs/optimization.rst +++ b/docs/optimization.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. _optimization_page: Using "optimization" technique diff --git a/docs/plot.rst b/docs/plot.rst index ab7f6f93e..3cee29632 100644 --- a/docs/plot.rst +++ b/docs/plot.rst @@ -1,4 +1,8 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. currentmodule:: grid2op.PlotGrid + .. |replaygif| image:: ./img/random_agent.gif .. |14bus_1| image:: ./img/14bus_1.png .. |14bus_2| image:: ./img/14bus_2.png diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 3694626c5..d60018bd9 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + Getting started =================================== diff --git a/docs/release_verification.rst b/docs/release_verification.rst new file mode 100644 index 000000000..0db88d1ea --- /dev/null +++ b/docs/release_verification.rst @@ -0,0 +1,254 @@ +.. _release-verification: + +Verifying release integrity +============================ + +This document explains how to verify that a Grid2Op release asset (``.whl`` or +``.tar.gz``) is authentic and has not been tampered with. + +Two methods are available depending on the version you are verifying: + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Grid2Op version + - Recommended method + * - ≥ 1.12.5 + - :ref:`Sigstore attestations via PyPI ` + * - < 1.12.5 + - :ref:`Digest check via check-release ` + +Both methods are independent of each other. For versions ≥ 1.12.5 you may +use either or both. + +---- + +.. _sigstore-method: + +Method 1 — Sigstore attestations (Grid2Op ≥ 1.12.5) +----------------------------------------------------- + +Starting from version 1.12.5, Grid2Op releases are signed using +`Sigstore `_ via PyPI's attestation mechanism. +This cryptographically proves that a release asset was built by the official +Grid2Op GitHub Actions pipeline and has not been modified since. + +.. warning:: + + This method does not work for versions prior to 1.12.5. + Use :ref:`Method 2 ` for those. + +Prerequisites +~~~~~~~~~~~~~ + +.. code-block:: bash + + pip install pypi-attestations + +Verify a wheel or sdist +~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + python -m pypi_attestations verify pypi \ + --repository https://github.com/Grid2op/grid2op \ + Grid2Op--py3-none-any.whl + +Replace ```` with the release you are verifying, for example ``1.12.5``. +The same command works for ``.tar.gz`` files. + +Expected output +~~~~~~~~~~~~~~~ + +.. code-block:: text + + OK: Grid2Op-1.12.5-py3-none-any.whl + +If verification fails, the tool will print an error. Do not use the file. + +What is being verified +~~~~~~~~~~~~~~~~~~~~~~ + +- The asset was produced by the ``release.yml`` workflow in the + ``Grid2op/grid2op`` GitHub repository. +- The asset has not been modified since it was uploaded to PyPI. +- The signing identity matches the Grid2Op repository — not just any + GitHub Actions run. + +You can also inspect the attestation directly on the PyPI release page at +``https://pypi.org/project/Grid2Op//`` under the **Provenance** +section. + +Expected Sigstore identity +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When verifying a Grid2Op release ≥ 1.12.5, the Sigstore certificate must +match **all** of the following values exactly. Any deviation should be treated +as a verification failure. + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Field + - Expected value + * - **Issuer** + - ``https://token.actions.githubusercontent.com`` + * - **Subject (workflow identity)** + - ``https://github.com/Grid2op/grid2op/.github/workflows/release.yml@refs/tags/v*`` + * - **Source repository** + - ``https://github.com/Grid2op/grid2op`` + +The ``*`` in the subject matches any version tag (e.g. ``v1.12.5``, +``v2.0.0``). The issuer confirms the signature came from GitHub Actions +specifically, not any other Sigstore-compatible system. + +If the ``pypi_attestations verify`` command succeeds but the certificate +fields above do not match when inspected manually, do not trust the asset. +To inspect the certificate fields directly: + +.. code-block:: bash + + python -m pypi_attestations inspect \ + Grid2Op--py3-none-any.whl + +---- + +.. _digest-method: + +Method 2 — Digest check (legacy releases < 1.12.5) +---------------------------------------------------- + +For releases prior to 1.12.5, integrity can be verified by comparing the +asset's cryptographic digest against the value recorded by PyPI at upload +time. + +Grid2Op ships a built-in command for this: + +.. code-block:: bash + + python -m grid2op check-release + +Prerequisites +~~~~~~~~~~~~~ + +A working Grid2Op installation (any version). No extra dependencies are +required — the command uses only the Python standard library. + +Basic usage +~~~~~~~~~~~ + +.. code-block:: bash + + # Default: verifies BLAKE2b-256 digest against live PyPI + python -m grid2op check-release Grid2Op-1.9.8-py3-none-any.whl + +Choosing which digest algorithm to check +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + # BLAKE2b-256 (default, strongest) + python -m grid2op check-release Grid2Op-1.9.8-py3-none-any.whl --blake2b + + # SHA-256 + python -m grid2op check-release Grid2Op-1.9.8-py3-none-any.whl --sha256 + + # MD5 (weak — use only if the above are unavailable) + python -m grid2op check-release Grid2Op-1.9.8-py3-none-any.whl --md5 + + # All three at once + python -m grid2op check-release Grid2Op-1.9.8-py3-none-any.whl --all + +Expected output +~~~~~~~~~~~~~~~ + +.. code-block:: text + + File : Grid2Op-1.9.8-py3-none-any.whl + Version : 1.9.8 + Source : PyPI (live) + Checking: blake2b_256 + + BLAKE2B-256 PASS + + ✓ All checks passed. + +If any digest does not match, the tool prints the expected and actual values +and exits with a non-zero status. Do not use the file. + +Offline verification (air-gapped environments) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you cannot reach PyPI, first generate a local digest database on a +connected machine using the helper script in the repository: + +.. code-block:: bash + + # On a machine with internet access — run once, or per release + python scripts/fetch_release_digests.py --output digests.json + + # Transfer digests.json to the air-gapped machine, then: + python -m grid2op check-release Grid2Op-1.9.8-py3-none-any.whl \ + --offline digests.json + +The ``fetch_release_digests.py`` script records all historical releases by +default, so a single ``digests.json`` file covers all versions. + +What is being verified +~~~~~~~~~~~~~~~~~~~~~~ + +The digest recorded by PyPI at the time of upload is compared against a +freshly computed digest of the local file. A mismatch means the file has +been modified after upload. Because the digests are fetched directly from +PyPI (or from a snapshot you generated yourself), this method does not rely +on any file bundled inside the Grid2Op package. + +Expected identity for legacy releases +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For releases prior to 1.12.5, there is no cryptographic workflow identity. +The expected identity is the PyPI account that owns the ``Grid2Op`` package: + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Field + - Expected value + * - **PyPI package name** + - ``Grid2Op`` (https://pypi.org/project/Grid2Op/) + * - **PyPI owning organisation** + - ``Grid2op`` (https://pypi.org/org/Grid2op/) + * - **PyPI collaborator** + - ``lfeoperations`` — LF Energy Operations (https://pypi.org/user/lfeoperations/) + * - **Source repository** + - ``https://github.com/Grid2op/grid2op`` + +The digests returned by ``check-release`` are fetched directly from the +PyPI JSON API for the above package. Verifying that the package name and +owning organisation match before trusting the digests is recommended. + +---- + +Summary +------- + +.. code-block:: text + + Is your version ≥ 1.12.5? + + YES → Method 1 (Sigstore) + pip install pypi-attestations + python -m pypi_attestations verify pypi \ + --repository https://github.com/Grid2op/grid2op \ + Grid2Op--py3-none-any.whl + + NO → Method 2 (digest) + python -m grid2op check-release \ + Grid2Op--py3-none-any.whl + +If you have concerns about a release, please open an issue at +https://github.com/Grid2op/grid2op/issues or follow the process described +in :ref:`security` for security-sensitive matters. diff --git a/docs/special.rst b/docs/special.rst index 68ffebc55..d63b1c786 100644 --- a/docs/special.rst +++ b/docs/special.rst @@ -1,28 +1,33 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. Color profiles for Sphinx. -.. Has to be used with hacks.css -.. (https://bitbucket.org/lbesson/web-sphinx/src/master/.static/hacks.css) +.. You can use it only if you have said, in config.py that the "colors.css" file should be included +.. with something like : +.. +.. html_static_path = ['_static'] +.. html_css_files = ['colors.css'] +.. role:: aqua .. role:: black -.. role:: gray -.. role:: grey -.. role:: silver -.. role:: white -.. role:: maroon -.. role:: red -.. role:: magenta +.. role:: blue +.. role:: cyan .. role:: fuchsia -.. role:: pink -.. role:: orange -.. role:: yellow -.. role:: lime +.. role:: gray .. role:: green -.. role:: olive -.. role:: teal -.. role:: cyan -.. role:: aqua -.. role:: blue +.. role:: lime +.. role:: magenta +.. role:: maroon .. role:: navy +.. role:: olive +.. role:: orange +.. role:: pink .. role:: purple +.. role:: red +.. role:: silver +.. role:: teal +.. role:: white +.. role:: yellow .. role:: under .. role:: over @@ -33,10 +38,9 @@ .. role:: it .. role:: ob -.. role:: small .. role:: large +.. role:: small .. role:: center .. role:: left .. role:: right -.. (c) Lilian Besson, 2011-2016, https://bitbucket.org/lbesson/web-sphinx/ diff --git a/docs/topology.rst b/docs/topology.rst index 2c81c7226..924bb7fbc 100644 --- a/docs/topology.rst +++ b/docs/topology.rst @@ -1,4 +1,5 @@ - +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. .. _topology-modeling-module: diff --git a/docs/troubleshoot.rst b/docs/troubleshoot.rst index d07539fda..2f13f1cb2 100644 --- a/docs/troubleshoot.rst +++ b/docs/troubleshoot.rst @@ -1,3 +1,5 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. .. _troubleshoot_page: diff --git a/docs/user.rst b/docs/user.rst index d1b715423..16f832bf2 100644 --- a/docs/user.rst +++ b/docs/user.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. toctree:: :maxdepth: 1 diff --git a/docs/user/action.rst b/docs/user/action.rst index 1f9f0bb28..52bad55c7 100644 --- a/docs/user/action.rst +++ b/docs/user/action.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. currentmodule:: grid2op.Action .. _n_gen: ./space.html#grid2op.Space.GridObjects.n_gen diff --git a/docs/user/agent.rst b/docs/user/agent.rst index 557603878..4ab111f5f 100644 --- a/docs/user/agent.rst +++ b/docs/user/agent.rst @@ -1,4 +1,8 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. currentmodule:: grid2op.Agent + .. _agent-module: Agent diff --git a/docs/user/backend.rst b/docs/user/backend.rst index 1d52adccd..9730d6057 100644 --- a/docs/user/backend.rst +++ b/docs/user/backend.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. currentmodule:: grid2op.Backend .. _backend-module: diff --git a/docs/user/chronics.rst b/docs/user/chronics.rst index 1557ab07f..91338a9fe 100644 --- a/docs/user/chronics.rst +++ b/docs/user/chronics.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. currentmodule:: grid2op.Chronics .. _time-series-module: diff --git a/docs/user/converter.rst b/docs/user/converter.rst index 3f8e5c8c8..45dd9cb61 100644 --- a/docs/user/converter.rst +++ b/docs/user/converter.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. currentmodule:: grid2op.Converter Converters diff --git a/docs/user/environment.rst b/docs/user/environment.rst index 5c1c9613a..55f14f24f 100644 --- a/docs/user/environment.rst +++ b/docs/user/environment.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. currentmodule:: grid2op.Environment .. _environment-module: diff --git a/docs/user/episode.rst b/docs/user/episode.rst index 9d8be3d8f..72bd59dcb 100644 --- a/docs/user/episode.rst +++ b/docs/user/episode.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + Episode =================================== diff --git a/docs/user/exception.rst b/docs/user/exception.rst index ac842250f..694a48f68 100644 --- a/docs/user/exception.rst +++ b/docs/user/exception.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + Exception =================================== This page is organized as follow: diff --git a/docs/user/final.rst b/docs/user/final.rst index 79beb6191..f768718e1 100644 --- a/docs/user/final.rst +++ b/docs/user/final.rst @@ -1,2 +1,4 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. .. include:: ../final.rst \ No newline at end of file diff --git a/docs/user/observation.rst b/docs/user/observation.rst index 05bb35a75..410e3caf7 100644 --- a/docs/user/observation.rst +++ b/docs/user/observation.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. currentmodule:: grid2op.Observation .. include:: special.rst diff --git a/docs/user/opponent.rst b/docs/user/opponent.rst index 0998a3d36..efc94ea56 100644 --- a/docs/user/opponent.rst +++ b/docs/user/opponent.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. currentmodule:: grid2op.Opponent Opponent Modeling diff --git a/docs/user/parameters.rst b/docs/user/parameters.rst index 727a422e5..8e219d551 100644 --- a/docs/user/parameters.rst +++ b/docs/user/parameters.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. _parameters-module: Parameters diff --git a/docs/user/reward.rst b/docs/user/reward.rst index 684eaccaf..c439da55a 100644 --- a/docs/user/reward.rst +++ b/docs/user/reward.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. currentmodule:: grid2op.Reward .. _reward-module: diff --git a/docs/user/rules.rst b/docs/user/rules.rst index a6967fccf..c2bb9f258 100644 --- a/docs/user/rules.rst +++ b/docs/user/rules.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. currentmodule:: grid2op.Rules .. _rule-module: diff --git a/docs/user/runner.rst b/docs/user/runner.rst index 8f96ffaf8..d8b7337b8 100644 --- a/docs/user/runner.rst +++ b/docs/user/runner.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. _runner-module: Runner diff --git a/docs/user/simulator.rst b/docs/user/simulator.rst index 0ce752046..902a1580a 100644 --- a/docs/user/simulator.rst +++ b/docs/user/simulator.rst @@ -1,4 +1,8 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. currentmodule:: grid2op.simulator + .. _simulator_page: Simulator diff --git a/docs/user/space.rst b/docs/user/space.rst index ac4f4b747..e810672ff 100644 --- a/docs/user/space.rst +++ b/docs/user/space.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. currentmodule:: grid2op.Space Space diff --git a/docs/user/special.rst b/docs/user/special.rst index 8f6680780..f0939d05e 100644 --- a/docs/user/special.rst +++ b/docs/user/special.rst @@ -1,4 +1,5 @@ -.. for the color +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. .. include:: ../special.rst diff --git a/docs/user/timeserie_handlers.rst b/docs/user/timeserie_handlers.rst index bd76abddf..4d6e849cc 100644 --- a/docs/user/timeserie_handlers.rst +++ b/docs/user/timeserie_handlers.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. currentmodule:: grid2op.Chronics.handlers .. _tshandler-module: diff --git a/docs/user/utils.rst b/docs/user/utils.rst index d30de21a1..40c6a5b64 100644 --- a/docs/user/utils.rst +++ b/docs/user/utils.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. currentmodule:: grid2op.utils Utility classes diff --git a/docs/user/voltagecontroler.rst b/docs/user/voltagecontroler.rst index 1c85a3552..3bec11940 100644 --- a/docs/user/voltagecontroler.rst +++ b/docs/user/voltagecontroler.rst @@ -1,3 +1,6 @@ +.. SPDX-License-Identifier: MPL-2.0 +.. This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + .. currentmodule:: grid2op.VoltageControler .. _voltage-controler-module: diff --git a/getting_started/test_renderer_14.py b/getting_started/test_renderer_14.py deleted file mode 100644 index c67db2fa0..000000000 --- a/getting_started/test_renderer_14.py +++ /dev/null @@ -1,127 +0,0 @@ -import grid2op -from grid2op.Agent import DoNothingAgent -from grid2op.Agent import GreedyAgent, RandomAgent -import numpy as np -import pdb -import warnings - - -with warnings.catch_warnings(): - warnings.filterwarnings("ignore") - env = grid2op.make("case14_realistic") - - -class MyExpertAgent(GreedyAgent): - def __init__(self, action_space): - GreedyAgent.__init__(self, action_space) - self.saved_score = [] - - def act(self, observation, reward, done=False): - """ - By definition, all "greedy" agents are acting the same way. The only thing that can differentiate multiple - agents is the actions that are tested. - - These actions are defined in the method :func:`._get_tested_action`. This :func:`.act` method implements the - greedy logic: take the actions that maximizes the instantaneous reward on the simulated action. - - Parameters - ---------- - observation: :class:`grid2op.BaseObservation.BaseObservation` - The current observation of the :class:`grid2op.Environment` - - reward: ``float`` - The current reward. This is the reward obtained by the previous action - - done: ``bool`` - Whether the episode has ended or not. Used to maintain gym compatibility - - Returns - ------- - res: :class:`grid2op.BaseAction.BaseAction` - The action chosen by the bot / controller / agent. - - """ - # print("________________\nbeginning simulate") - self.tested_action = self._get_tested_action(observation) - if len(self.tested_action) > 1: - all_rewards = np.full(shape=len(self.tested_action), fill_value=np.NaN, dtype=np.float) - for i, action in enumerate(self.tested_action): - simul_obs, simul_reward, simul_has_error, simul_info = observation.simulate(action) - all_rewards[i] = simul_reward - # if simul_reward > 19: - # pdb.set_trace() - - reward_idx = np.argmax(all_rewards) # rewards.index(max(rewards)) - expected_reward = np.max(all_rewards) - best_action = self.tested_action[reward_idx] -# print("BaseAction taken:\n{}".format(best_action)) - else: - all_rewards = [None] - expected_reward = None - best_action = self.tested_action[0] - - self.saved_score.append(((best_action, expected_reward), - [el for el in zip(self.tested_action, all_rewards)])) - # print("end simulate\n_____________") - return best_action - - def _get_tested_action(self, observation): - res = [self.action_space({})] # add the do nothing - for i, el in enumerate(observation.line_status): - # try to reconnect powerlines - if not el: - tmp = np.zeros(self.action_space.n_line, dtype=np.int) - tmp[i] = 1 - action = self.action_space({"set_line_status": tmp}) - action = action.update({"set_bus": {"lines_or_id": [(i, 1)], "lines_ex_id": [(i, 1)]}}) - res.append(action) - - # disconnect the powerlines - ## 12 to 13, 10 to 9 # 5 to 12, 5 to 10, - for i in [19, 17]: # , 10 ,12 <- with that it takes action that leads to divergence, check that! - tmp = np.full(self.action_space.n_line, fill_value=False, dtype=np.bool) - tmp[i] = True - action = self.action_space({"change_line_status": tmp}) - if not observation.line_status[i]: - # so the action consisted in reconnecting the powerline - # i need to say on which bus - action = action.update({"set_bus": {"lines_or_id": [(i, 1)], "lines_ex_id": [(i, 1)]}}) - res.append(action) - - # play with the topology - ## i put powerlines going from 1 to 4 with powerline going from 3 to 4 at substation 4 - action = self.action_space({"change_bus": - {"substations_id": [(4, np.array([False, True, True, False, False]))]}}) - res.append(action) - - ## i put powerline from 5 to 12 with powerline from 5 to 10 at substation 5 - action = self.action_space({"change_bus": - {"substations_id": [(5, np.array([False, True, False, True, False, False]))]}}) - res.append(action) - - ## i put powerline from 1 to 4 with powerline from 1 to 3 with at substation 1 - action = self.action_space({"change_bus": - {"substations_id": [(1, np.array([False, False, True, True, False, False]))]}}) - res.append(action) - return res - - -my_agent = MyExpertAgent(env.action_space) -# my_agent = RandomAgent(env.action_space) -print("Total unitary action possible: {}".format(my_agent.action_space.n)) - -all_obs = [] -obs = env.reset() -all_obs.append(obs) -reward = env.reward_range[0] -done = False -nb_step = 0 -while True: - env.render() - action = my_agent.act(obs, reward, done) - obs, reward, done, _ = env.step(action) - print("Rendering timestep {}".format(nb_step)) - if done: - break - all_obs.append(obs) - nb_step += 1 diff --git a/grid2op/Action/actionSpace.py b/grid2op/Action/actionSpace.py index 86e56152e..01cc18d93 100644 --- a/grid2op/Action/actionSpace.py +++ b/grid2op/Action/actionSpace.py @@ -8,7 +8,7 @@ import warnings import copy -from typing import Dict, Literal, Optional +from typing import Dict, Literal, Optional, Type import grid2op from grid2op.typing_variables import DICT_ACT_TYPING @@ -43,7 +43,7 @@ def __init__( self, gridobj, legal_action, - actionClass=BaseAction, # need to be a base grid2op type (and not a type generated on the fly) + actionClass: Type[BaseAction]=BaseAction, # need to be a base grid2op type (and not a type generated on the fly) _local_dir_cls=None, ): """ diff --git a/grid2op/Action/baseAction.py b/grid2op/Action/baseAction.py index b2d2d4e41..d4288c631 100644 --- a/grid2op/Action/baseAction.py +++ b/grid2op/Action/baseAction.py @@ -16,7 +16,7 @@ try: from typing import Self -except ImportError: +except ImportError: # pragma: no cover from typing_extensions import Self # type: ignore from packaging import version @@ -24,6 +24,7 @@ from grid2op.typing_variables import DICT_ACT_TYPING, BACKEND_TYPE from grid2op.dtypes import dt_int, dt_bool, dt_float from grid2op.Exceptions import (Grid2OpException, + InvalidAction, AmbiguousAction, InvalidNumberOfLines, IllegalAction, @@ -827,9 +828,10 @@ def _build_dict_attr_if_needed(cls): # shunts if cls.shunts_data_available: - cls.DICT_ATTR_["_shunt_p"] = np.full(shape=cls.n_shunt, fill_value=np.nan, dtype=dt_float) - cls.DICT_ATTR_["_shunt_q"] = np.full(shape=cls.n_shunt, fill_value=np.nan, dtype=dt_float) - cls.DICT_ATTR_["_shunt_bus"] = np.full(shape=cls.n_shunt, fill_value=0, dtype=dt_int) + n_shunt : int = cls.n_shunt # type: ignore n_shunt is set here + cls.DICT_ATTR_["_shunt_p"] = np.full(shape=n_shunt, fill_value=np.nan, dtype=dt_float) + cls.DICT_ATTR_["_shunt_q"] = np.full(shape=n_shunt, fill_value=np.nan, dtype=dt_float) + cls.DICT_ATTR_["_shunt_bus"] = np.full(shape=n_shunt, fill_value=0, dtype=dt_int) if cls.detachment_is_allowed: cls.DICT_ATTR_["_detach_load"] = np.full(cls.n_load, dtype=dt_bool, fill_value=False) @@ -841,8 +843,7 @@ def _build_attr(cls, attr_nm: str): # False(line is disconnected) / True(line is connected) cls._build_dict_attr_if_needed() if attr_nm not in cls.DICT_ATTR_: - # TODO raise ActionException - raise Grid2OpException( + raise InvalidAction( 'Impossible to find the attribute "{}" ' 'into the BaseAction of type "{}"'.format(attr_nm, cls) ) @@ -857,9 +858,9 @@ def process_shunt_static_data(cls): cls.attr_list_set = copy.deepcopy(cls.attr_list_set) # remove the shunts from the list to vector for el in ["_shunt_p", "_shunt_q", "_shunt_bus"]: - if el in cls.attr_list_vect: - try: - cls.attr_list_vect.remove(el) + if el in cls.attr_list_vect: # type: ignore + try: # pragma: no cover + cls.attr_list_vect.remove(el) # type: ignore except ValueError: pass cls._update_value_set() @@ -888,12 +889,6 @@ def process_detachment(cls): def copy(self) -> "BaseAction": # sometimes this method is used... return self.__deepcopy__() - - def shape(self): - return type(self).shapes() - - def dtype(self): - return type(self).dtypes() def _aux_copy(self, other: Self) -> None: attr_simple = [ @@ -944,7 +939,7 @@ def _aux_copy(self, other: Self) -> None: if arr is not None: arr_oth = getattr(other, attr_nm) if arr_oth is not None: - arr_oth[:] = getattr(self, attr_nm) + arr_oth[:] = getattr(self, attr_nm) # pragma: no cover else: setattr(other, attr_nm, copy.deepcopy(arr)) else: @@ -958,7 +953,7 @@ def __copy__(self) -> "BaseAction": self._aux_copy(other=res) # handle dict_inj - for k, el in self._dict_inj.items(): + for k, el in self._dict_inj.items(): # pragma: no cover res._dict_inj[k] = copy.copy(el) # just copy @@ -1032,17 +1027,21 @@ def as_serializable_dict(self) -> dict: cls = type(self) # bool elements if self._modif_alert: + if self._private_raise_alert is None: # pragma: no cover + raise InvalidAction("_modif_alert is True but _private_raise_alert is None.") res["raise_alert"] = [ int(id_) for id_, val in enumerate(self._private_raise_alert) if val ] - if not res["raise_alert"]: + if not res["raise_alert"]: # pragma: no cover del res["raise_alert"] if self._modif_alarm: + if self._private_raise_alarm is None: # pragma: no cover + raise InvalidAction("_modif_alarm is True but _private_raise_alarm is None.") res["raise_alarm"] = [ int(id_) for id_, val in enumerate(self._private_raise_alarm) if val ] - if not res["raise_alarm"]: + if not res["raise_alarm"]: # pragma: no cover del res["raise_alarm"] if self._modif_change_bus: @@ -1053,14 +1052,16 @@ def as_serializable_dict(self) -> dict: self._aux_serialize_add_key_change("line_ex_change_bus", "lines_ex_id", res["change_bus"], cls.name_line) if hasattr(cls, "n_storage") and cls.n_storage: self._aux_serialize_add_key_change("storage_change_bus", "storages_id", res["change_bus"], cls.name_storage) - if not res["change_bus"]: + if not res["change_bus"]: # pragma: no cover del res["change_bus"] if self._modif_change_status: + if self._private_switch_line_status is None: # pragma: no cover + raise InvalidAction("_modif_change_status is True but _private_switch_line_status is None.") res["change_line_status"] = [ str(cls.name_line[id_]) for id_, val in enumerate(self._private_switch_line_status) if val ] - if not res["change_line_status"]: + if not res["change_line_status"]: # pragma: no cover del res["change_line_status"] # int elements @@ -1072,44 +1073,52 @@ def as_serializable_dict(self) -> dict: self._aux_serialize_add_key_set("line_ex_set_bus", "lines_ex_id", res["set_bus"], cls.name_line) if hasattr(cls, "n_storage") and cls.n_storage: self._aux_serialize_add_key_set("storage_set_bus", "storages_id", res["set_bus"], cls.name_storage) - if not res["set_bus"]: + if not res["set_bus"]: # pragma: no cover del res["set_bus"] if self._modif_set_status: + if self._private_set_line_status is None: # pragma: no cover + raise InvalidAction("_modif_set_status is True but _private_set_line_status is None.") res["set_line_status"] = [ (str(cls.name_line[id_]), int(val)) for id_, val in enumerate(self._private_set_line_status) if val != 0 ] - if not res["set_line_status"]: + if not res["set_line_status"]: # pragma: no cover del res["set_line_status"] # float elements if self._modif_redispatch: + if self._private_redispatch is None: # pragma: no cover + raise InvalidAction("_modif_redispatch is True but _private_redispatch is None.") res["redispatch"] = [ (str(cls.name_gen[id_]), float(val)) for id_, val in enumerate(self._private_redispatch) if np.abs(val) >= 1e-7 ] - if not res["redispatch"]: + if not res["redispatch"]: # pragma: no cover del res["redispatch"] if self._modif_storage: + if self._private_storage_power is None: # pragma: no cover + raise InvalidAction("_modif_storage is True but _private_storage_power is None.") res["set_storage"] = [ (str(cls.name_storage[id_]), float(val)) for id_, val in enumerate(self._private_storage_power) if np.abs(val) >= 1e-7 ] - if not res["set_storage"]: + if not res["set_storage"]: # pragma: no cover del res["set_storage"] if self._modif_curtailment: + if self._private_curtail is None: # pragma: no cover + raise InvalidAction("_modif_curtailment is True but _private_curtail is None.") res["curtail"] = [ (str(cls.name_gen[id_]), float(val)) for id_, val in enumerate(self._private_curtail) if np.abs(val + 1.) >= 1e-7 ] - if not res["curtail"]: + if not res["curtail"]: # pragma: no cover del res["curtail"] # more advanced options @@ -1119,26 +1128,26 @@ def as_serializable_dict(self) -> dict: [cls.name_gen, cls.name_gen, cls.name_load, cls.name_load]): if ky in self._dict_inj: res["injection"][ky] = {str(vect_nm[i]): float(val) for i, val in enumerate(self._dict_inj[ky])} - if not res["injection"]: + if not res["injection"]: # pragma: no cover del res["injection"] if cls.shunts_data_available: res["shunt"] = {} if self._private_shunt_p is not None and np.isfinite(self._private_shunt_p).any(): res["shunt"]["shunt_p"] = [ - (str(cls.name_shunt[sh_id]), float(val)) for sh_id, val in enumerate(self._private_shunt_p) if np.isfinite(val) + (str(cls.name_shunt[sh_id]), float(val)) for sh_id, val in enumerate(self._private_shunt_p) if np.isfinite(val) # type: ignore ] if self._private_shunt_q is not None and np.isfinite(self._private_shunt_q).any(): res["shunt"]["shunt_q"] = [ - (str(cls.name_shunt[sh_id]), float(val)) for sh_id, val in enumerate(self._private_shunt_q) if np.isfinite(val) + (str(cls.name_shunt[sh_id]), float(val)) for sh_id, val in enumerate(self._private_shunt_q) if np.isfinite(val) # type: ignore ] if self._private_shunt_bus is not None and (self._private_shunt_bus != 0).any(): res["shunt"]["shunt_bus"] = [ - (str(cls.name_shunt[sh_id]), int(val)) + (str(cls.name_shunt[sh_id]), int(val)) # type: ignore for sh_id, val in enumerate(self._private_shunt_bus) if val != 0 ] - if not res["shunt"]: + if not res["shunt"]: # pragma: no cover del res["shunt"] if cls.detachment_is_allowed: @@ -1152,7 +1161,7 @@ def as_serializable_dict(self) -> dict: continue if vect_.any(): res[attr_key] = [str(xxx_name[el]) for el in vect_.nonzero()[0]] - if not res[attr_key]: + if not res[attr_key]: # pragma: no cover del res[attr_key] return res @@ -1163,7 +1172,7 @@ def _add_shunt_data(cls): cls.shunt_added = True cls.attr_list_vect = copy.deepcopy(cls.attr_list_vect) - cls.attr_list_vect += ["_shunt_p", "_shunt_q", "_shunt_bus"] + cls.attr_list_vect += ["_shunt_p", "_shunt_q", "_shunt_bus"] # type: ignore cls.authorized_keys = copy.deepcopy(cls.authorized_keys) cls.authorized_keys.add("shunt") @@ -1204,7 +1213,7 @@ def alert_raised(self) -> np.ndarray: """ if self._modif_alert: - return (self._private_raise_alert).nonzero[0] + return (self._private_raise_alert).nonzero()[0] else: return np.zeros(0, dtype=int) @@ -1385,7 +1394,7 @@ def _set_array_from_attr_name(self, allowed_keys, key: str, array_) -> None: raise Grid2OpException( 'Impossible to find the attribute "{}" ' 'into the BaseAction of type "{}"'.format(key, type(self)) - ) + ) # pragma: no cover def _post_process_from_vect_injflags(self): modif_inj = False @@ -2254,28 +2263,39 @@ def reset(self): """ # False(line is disconnected) / True(line is connected) - self._private_set_line_status[:] = 0 - self._private_switch_line_status[:] = False + if self._private_set_line_status is not None: + self._private_set_line_status[:] = 0 + + if self._private_switch_line_status is not None: + self._private_switch_line_status[:] = False # injection change self._dict_inj = {} + self._modif_inj = False # topology changed - self._private_set_topo_vect[:] = 0 - self._private_change_bus_vect[:] = False + if self._private_set_topo_vect is not None: + self._private_set_topo_vect[:] = 0 + if self._private_change_bus_vect is not None: + self._private_change_bus_vect[:] = False # add the hazards and maintenance usefull for saving. - self._private_hazards[:] = False - self._private_maintenance[:] = False + if self._private_hazards is not None: + self._private_hazards[:] = False + if self._private_maintenance is not None: + self._private_maintenance[:] = False # redispatching vector - self._private_redispatch[:] = 0.0 + if self._private_redispatch is not None: + self._private_redispatch[:] = 0.0 # storage - self._private_storage_power[:] = 0.0 + if self._private_storage_power is not None: + self._private_storage_power[:] = 0.0 # storage - self._private_curtail[:] = -1.0 + if self._private_curtail is not None: + self._private_curtail[:] = -1.0 self._vectorized = None self._lines_impacted = None @@ -2283,16 +2303,31 @@ def reset(self): # shunts if type(self).shunts_data_available: - self._private_shunt_p[:] = np.nan - self._private_shunt_q[:] = np.nan - self._private_shunt_bus[:] = 0 + if self._private_shunt_p is not None: + self._private_shunt_p[:] = np.nan + if self._private_shunt_q is not None: + self._private_shunt_q[:] = np.nan + if self._private_shunt_bus is not None: + self._private_shunt_bus[:] = 0 # alarm - self._private_raise_alarm[:] = False + if self._private_raise_alarm is not None: + self._private_raise_alarm[:] = False # alert - self._private_raise_alert[:] = False + if self._private_raise_alert is not None: + self._private_raise_alert[:] = False + + # detachment + if type(self).detachment_is_allowed: + for el in ["load", "gen", "storage"]: + if getattr(self, el) is not None: + getattr(self, el)[:] = False + # backend dependant callback + self.backend_dependant_callback = None + + # and now the flags self._reset_modified_flags() def _assign_iadd_or_warn(self, attr_name, new_value): @@ -2342,7 +2377,7 @@ def _aux_iadd_curtail(self, other): curtailment = other._curtail ok_ind = np.isfinite(curtailment) & (np.abs(curtailment + 1.0) >= 1e-7) if ok_ind.any(): - if "_curtail" not in self.attr_list_set: + if "_curtail" not in self.attr_list_set: # pragma: no cover warnings.warn( type(self).ERR_ACTION_CUT.format("_curtail") ) @@ -2358,7 +2393,7 @@ def _aux_iadd_storage(self, other: "BaseAction"): set_storage = other._private_storage_power ok_ind = np.isfinite(set_storage) & (np.abs(set_storage) >= 1e-7).any() if ok_ind.any(): - if "_storage_power" not in self.attr_list_set: + if "_storage_power" not in self.attr_list_set: # pragma: no cover warnings.warn( type(self).ERR_ACTION_CUT.format("_storage_power") ) @@ -2384,7 +2419,7 @@ def _aux_iadd_modif_flags(self, other: Self): self._modif_detach_storage = self._modif_detach_storage or other._modif_detach_storage def _aux_iadd_shunt(self, other: "BaseAction"): - if not type(other).shunts_data_available: + if not type(other).shunts_data_available: # pragma: no cover warnings.warn("Trying to add an action that does not support " "shunt with an action that does.") return @@ -2656,7 +2691,7 @@ def __call__(self) -> Tuple[dict, np.ndarray, np.ndarray, np.ndarray, np.ndarray storage_power = self._private_storage_power # remark: curtailment is handled by an algorithm in the environment, so don't need to be returned here shunts = {} - if type(self).shunts_data_available: + if type(self).shunts_data_available: shunts["_private_shunt_p"] = self._private_shunt_p shunts["_private_shunt_q"] = self._private_shunt_q shunts["_private_shunt_bus"] = self._private_shunt_bus @@ -2675,7 +2710,7 @@ def __call__(self) -> Tuple[dict, np.ndarray, np.ndarray, np.ndarray, np.ndarray def _aux_digest_shunt_issue_warning_if_needed(self, ddict_): cls = type(self) for k in ddict_: - if k not in cls.key_shunt_reco: + if k not in cls.key_shunt_reco: # pragma: no cover warn = "The key {} is not recognized by BaseAction when trying to modify the shunt.".format( k ) @@ -2773,7 +2808,7 @@ def _digest_injection(self, dict_): self._dict_inj[k] = np.array(tmp_d[k]).astype(dt_float) self._cached_is_not_ambiguous = False # invalidate cache for ambiguous action # TODO check the size based on the input data ! - else: + else: # pragma: no cover warn = ( "The key {} is not recognized by BaseAction when trying to modify the injections." "".format(k) @@ -3058,14 +3093,14 @@ def _check_keys_exist(action_cls:GridObjects, act_dict): act_dict (str:Any): Dictionary representation of an action """ for kk in act_dict.keys(): - if kk not in action_cls.authorized_keys: + if kk not in action_cls.authorized_keys: # pragma: no cover if kk == "shunt" and not action_cls.shunts_data_available: # no warnings are raised in this case because if a warning # were raised it could crash some environment # with shunt in "init_state.json" with a backend that does not # handle shunt continue - if kk == "set_storage" and action_cls.n_storage == 0: + if kk == "set_storage" and action_cls.n_storage == 0: # pragma: no cover # no warnings are raised in this case because if a warning # were raised it could crash some environment # with storage in "init_state.json" but if the backend did not @@ -4525,20 +4560,24 @@ def as_dict(self) -> Dict[Literal["load_p", "load_q", "prod_p", "prod_v", # saving the injections for k in ["load_p", "prod_p", "load_q", "prod_v"]: if k in self._dict_inj: - res[k] = 1.0 * self._dict_inj[k] + res[k] = self._dict_inj[k].copy() self._aux_as_dict_line_status(res) self._aux_as_dict_topo(res) self._aux_as_dict_hazards_maintenance(res) if self._private_redispatch is not None and (np.abs(self._private_redispatch) >= 1e-7).any(): - res["redispatch"] = 1.0 * self._private_redispatch + res["redispatch"] = self._private_redispatch.copy() if self._modif_storage: - res["storage_power"] = 1.0 * self._private_storage_power + if self._private_storage_power is None: + raise InvalidAction("self._private_storage_power is None yet self._modif_storage flag is True") + res["storage_power"] = self._private_storage_power.copy() if self._modif_curtailment: - res["curtailment"] = 1.0 * self._private_curtail + if self._private_curtail is None: + raise InvalidAction("self._private_curtail is None yet self._modif_curtailment flag is True") + res["curtailment"] = self._private_curtail.copy() if type(self).shunts_data_available: self._aux_as_dict_shunt(res) diff --git a/grid2op/Action/serializableActionSpace.py b/grid2op/Action/serializableActionSpace.py index 5fb02da91..f230ccad6 100644 --- a/grid2op/Action/serializableActionSpace.py +++ b/grid2op/Action/serializableActionSpace.py @@ -9,7 +9,7 @@ import warnings import numpy as np import itertools -from typing import Dict, List, Literal +from typing import Dict, List, Literal, Type try: from typing import Self except ImportError: @@ -79,7 +79,7 @@ def __init__(self, gridobj, actionClass=BaseAction, _init_grid=True, _local_dir_ _init_grid=_init_grid, _local_dir_cls=_local_dir_cls ) - self.actionClass = self.subtype + self.actionClass: Type[BaseAction] = self.subtype self._template_act = self.actionClass() @staticmethod @@ -189,10 +189,11 @@ def supports_type(self, "detach_gen", # new in 1.11.0 "detach_storage", # new in 1.11.0 ] - assert action_type in name_action_types, ( - f"The action type provided should be in {name_action_types}. " - f"You provided {action_type} which is not supported." - ) + if action_type not in name_action_types: + raise AmbiguousAction( + f"The action type provided should be in {name_action_types}. " + f"You provided {action_type} which is not supported." + ) cls = type(self) if action_type == "storage_power" or action_type == "set_storage": return (cls.n_storage > 0) and ( diff --git a/grid2op/Chronics/fromNPY.py b/grid2op/Chronics/fromNPY.py index 7ebb2ac6c..efdbc8400 100644 --- a/grid2op/Chronics/fromNPY.py +++ b/grid2op/Chronics/fromNPY.py @@ -6,7 +6,7 @@ # SPDX-License-Identifier: MPL-2.0 # This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. -from typing import Optional, Union, Dict, Literal +from typing import Optional, Union, Dict, Literal, TYPE_CHECKING import numpy as np import hashlib from datetime import datetime, timedelta @@ -16,6 +16,10 @@ from grid2op.Chronics.gridValue import GridValue from grid2op.Exceptions import ChronicsError +if TYPE_CHECKING: + from grid2op.Backend import Backend + from grid2op.Action import PlayableAction + class FromNPY(GridValue): """ @@ -169,7 +173,9 @@ def __init__( if maintenance is not None: self.has_maintenance = True self.n_line = maintenance.shape[1] - assert load_p.shape[0] == maintenance.shape[0] + if load_p.shape[0] != maintenance.shape[0]: + raise ChronicsError("The maintenance file does not have the same number " + "of time step at the load file") self.maintenance = maintenance # TODO copy self.maintenance_time = ( @@ -198,8 +204,11 @@ def __init__( self._forecasts = None if load_p_forecast is not None: - assert load_q_forecast is not None - assert prod_p_forecast is not None + if load_q_forecast is None: + raise ChronicsError("If you provide load_p_forecast you should provide load_q_forecast.") + if prod_p_forecast is None: + raise ChronicsError("If you provide load_p_forecast you should provide prod_p_forecast.") + self._forecasts = FromNPY( load_p=load_p_forecast, load_q=load_q_forecast, @@ -232,12 +241,17 @@ def initialize( order_backend_subs, names_chronics_to_backend=None, ): - assert len(order_backend_prods) == self.n_gen, f"len(order_backend_prods)={len(order_backend_prods)} vs self.n_gen={self.n_gen}" - assert len(order_backend_loads) == self.n_load, f"len(order_backend_loads)={len(order_backend_loads)} vs self.n_load={self.n_load}" + if len(order_backend_prods) != self.n_gen: + raise ChronicsError(f"len(order_backend_prods)={len(order_backend_prods)} vs self.n_gen={self.n_gen}") + + if len(order_backend_loads) != self.n_load: + raise ChronicsError(f"len(order_backend_loads)={len(order_backend_loads)} vs self.n_load={self.n_load}") + if self.n_line is None: self.n_line = len(order_backend_lines) else: - assert len(order_backend_lines) == self.n_line, f"len(order_backend_lines)={len(order_backend_lines)} vs self.n_line={self.n_line}" + if len(order_backend_lines) != self.n_line: + raise ChronicsError(f"len(order_backend_lines)={len(order_backend_lines)} vs self.n_line={self.n_line}") if self._forecasts is not None: self._forecasts.initialize( @@ -255,7 +269,7 @@ def initialize( self.current_index = self._i_start - 1 self._max_iter = self._i_end - self._i_start - def _get_long_hash(self, hash_: hashlib.blake2b = None): + def _get_long_hash(self, hash_: Optional[hashlib.blake2b] = None): # get the "long hash" from blake2b if hash_ is None: hash_ = ( @@ -361,33 +375,50 @@ def load_next(self): ) def check_validity( - self, backend: Optional["grid2op.Backend.backend.Backend"] + self, backend: Optional["Backend"] ) -> None: - # TODO raise the proper errors from ChronicsError here rather than AssertError - assert self._load_p.shape[0] == self._load_q.shape[0] - assert self._load_p.shape[0] == self._prod_p.shape[0] + if self._load_p.shape[0] != self._load_q.shape[0]: + raise ChronicsError("Loads P and Q must have the same number of timesteps / rows") + if self._load_p.shape[0] != self._prod_p.shape[0]: + raise ChronicsError("Loads P and gen p must have the same number of timesteps / rows") if self._prod_v is not None: - assert self._load_p.shape[0] == self._prod_v.shape[0] + if self._load_p.shape[0] != self._prod_v.shape[0]: + raise ChronicsError("Loads P and gen v (as it is provided) must have the same number of timesteps / rows") if self.hazards is not None: - assert self.hazards.shape[1] == self.n_line + if self.hazards.shape[1] != self.n_line: + raise ChronicsError("Loads P and hazards (as it is provided) must have the same number of timesteps / rows") + if self.maintenance is not None: - assert self.maintenance.shape[1] == self.n_line + if self.maintenance.shape[1] != self.n_line: + raise ChronicsError("Maintenance file (as it is provided) should have the " + "same number of columns as the number of powerlines on the grid") if self.maintenance_duration is not None: - assert self.n_line == self.maintenance_duration.shape[1] + if self.n_line != self.maintenance_duration.shape[1]: + raise ChronicsError("maintenance_duration file (as it is provided) should have the same " + "number of columns as the number of powerlines on the grid") if self.maintenance_time is not None: - assert self.n_line == self.maintenance_time.shape[1] + if self.n_line != self.maintenance_time.shape[1]: + raise ChronicsError("maintenance_time file (as it is provided) should have the same " + "number of columns as the number of powerlines on the grid") # TODO forecast if self._forecasts is not None: - assert self._forecasts.n_line == self.n_line - assert self._forecasts.n_gen == self.n_gen - assert self._forecasts.n_load == self.n_load - assert self._load_p.shape[0] == self._forecasts._load_p.shape[0] - assert self._load_q.shape[0] == self._forecasts._load_q.shape[0] - assert self._prod_p.shape[0] == self._forecasts._prod_p.shape[0] + if self._forecasts.n_line != self.n_line: + raise ChronicsError("self._forecasts.n_line != self.n_line") + if self._forecasts.n_gen != self.n_gen: + raise ChronicsError("self._forecasts.n_gen != self.n_gen") + if self._forecasts.n_load != self.n_load: + raise ChronicsError("self._forecasts.n_load != self.n_load") + if self._load_p.shape[0] != self._forecasts._load_p.shape[0]: + raise ChronicsError("self._load_p.shape[0] != self._forecasts._load_p.shape[0]") + if self._load_q.shape[0] != self._forecasts._load_q.shape[0]: + raise ChronicsError("self._load_q.shape[0] != self._forecasts._load_q.shape[0]") + if self._prod_p.shape[0] != self._forecasts._prod_p.shape[0]: + raise ChronicsError("self._prod_p.shape[0] != self._forecasts._prod_p.shape[0]") if self._prod_v is not None and self._forecasts._prod_v is not None: - assert self._prod_v.shape[0] == self._forecasts._prod_v.shape[0] + if self._prod_v.shape[0] != self._forecasts._prod_v.shape[0]: + raise ChronicsError("self._prod_v.shape[0] != self._forecasts._prod_v.shape[0]") self._forecasts.check_validity(backend=backend) def next_chronics(self): @@ -698,7 +729,9 @@ def change_i_end(self, new_i_end: Union[int, None]): else: self.__new_iend = None - def get_init_action(self, names_chronics_to_backend: Optional[Dict[Literal["loads", "prods", "lines"], Dict[str, str]]]=None) -> Union["grid2op.Action.playableAction.PlayableAction", None]: + def get_init_action(self, + names_chronics_to_backend: Optional[Dict[Literal["loads", "prods", "lines"], + Dict[str, str]]]=None) -> Union["PlayableAction", None]: # names_chronics_to_backend is ignored, names should be consistent between the environment # and the initial state diff --git a/grid2op/Converter/BackendConverter.py b/grid2op/Converter/BackendConverter.py index bf6c968cd..5d852a032 100644 --- a/grid2op/Converter/BackendConverter.py +++ b/grid2op/Converter/BackendConverter.py @@ -539,14 +539,16 @@ def assert_grid_correct(self, _local_dir_cls=None) -> None: if self.sub_source_target is None: # automatic mode for substations, names must match - assert np.all( + if ( self.target_backend.name_sub[self._sub_tg2sr] - == self.source_backend.name_sub - ) - assert np.all( + != self.source_backend.name_sub + ).any(): + raise BackendError("The names of the substations do not match") + if ( self.source_backend.name_sub[self._sub_sr2tg] - == self.target_backend.name_sub - ) + != self.target_backend.name_sub + ).any(): + raise BackendError("The names of the substations do not match") # check that all corresponding vectors are valid (and properly initialized, like every component above 0 etc.) self._check_both_consistent(self._line_tg2sr, self._line_sr2tg) @@ -561,15 +563,22 @@ def assert_grid_correct(self, _local_dir_cls=None) -> None: # n_storage == 0 and there are storage units on the source backend # this means that the target_backend supports storage but not # the source one - assert np.all(self._topo_sr2tg[self._topo_tg2sr] >= 0) - assert np.all(sorted(self._topo_sr2tg[self._topo_tg2sr]) == np.arange(self.dim_topo)) + if (self._topo_sr2tg[self._topo_tg2sr] < 0).any(): + raise BackendError("every topo element should be mapped some (self._topo_sr2tg[self._topo_tg2sr] < 0)") + if (sorted(self._topo_sr2tg[self._topo_tg2sr]) != np.arange(self.dim_topo)).any(): + raise BackendError("Some element of topology are not mapped") + topo_sr2tg_without_storage = self._topo_sr2tg[self._topo_sr2tg >= 0] - assert (self._topo_sr2tg == -1).sum() == tg_cls.n_storage - assert np.all(self._topo_tg2sr[topo_sr2tg_without_storage] >= 0) + if (self._topo_sr2tg == -1).sum() != tg_cls.n_storage: + raise BackendError("(self._topo_sr2tg == -1).sum() != tg_cls.n_storage") + + if (self._topo_tg2sr[topo_sr2tg_without_storage] < 0).any(): + raise BackendError("(self._topo_tg2sr[topo_sr2tg_without_storage] >= 0).any()") target_without_storage = np.array([i for i in range(tg_cls.dim_topo) if i not in tg_cls.storage_pos_topo_vect]) - assert np.all(sorted(self._topo_tg2sr[topo_sr2tg_without_storage]) == target_without_storage) + if (sorted(self._topo_tg2sr[topo_sr2tg_without_storage]) != target_without_storage).any(): + raise BackendError("(sorted(self._topo_tg2sr[topo_sr2tg_without_storage]) != target_without_storage).any()") self._topo_sr2tg = topo_sr2tg_without_storage if type(self).shunts_data_available: @@ -591,23 +600,22 @@ def assert_grid_correct(self, _local_dir_cls=None) -> None: self.names_target_to_source = dict_ def _check_vect_valid(self, vect): - assert np.all( - vect >= 0 - ), ERROR_INVALID_VECTOR - assert sorted(np.unique(vect)) == sorted( - vect - ), ERROR_INVALID_VECTOR + if np.any(vect < 0): + raise BackendError(ERROR_INVALID_VECTOR) + if sorted(np.unique(vect)) != sorted(vect): + raise BackendError(ERROR_INVALID_VECTOR) if vect.shape[0] > 0: - assert ( - np.max(vect) == vect.shape[0] - 1 - ), ERROR_INVALID_VECTOR + if np.max(vect) != vect.shape[0] - 1: + raise BackendError(ERROR_INVALID_VECTOR) def _check_both_consistent(self, tg2sr, sr2tg): self._check_vect_valid(tg2sr) self._check_vect_valid(sr2tg) res = np.arange(tg2sr.shape[0]) - assert np.all(tg2sr[sr2tg] == res) - assert np.all(sr2tg[tg2sr] == res) + if np.any(tg2sr[sr2tg] != res): + raise BackendError("np.any(tg2sr[sr2tg] != res)") + if np.any(sr2tg[tg2sr] != res): + raise BackendError("np.any(sr2tg[tg2sr] != res)") def assert_grid_correct_after_powerflow(self): # we don't assert that `self.source_backend.assert_grid_correct_after_powerflow()` diff --git a/grid2op/Converter/IdToAct.py b/grid2op/Converter/IdToAct.py index a06098a7f..7ab82e6ce 100644 --- a/grid2op/Converter/IdToAct.py +++ b/grid2op/Converter/IdToAct.py @@ -280,7 +280,8 @@ def init_converter(self, all_actions=None, **kwargs): nb = len(all_actions) # assert I can compute the "len" for i in range(nb): act = all_actions[i] # assert I can use the `[]` operator - assert isinstance(act, BaseAction) # assert what's in there is a BaseAction + if not isinstance(act, BaseAction): + raise RuntimeError("The action provided is not a valid grid2op action") except Exception as exc_: raise RuntimeError("Impossible to load the action provided.") from exc_ # does not copy here (to save memory in case of shared memory setting) diff --git a/grid2op/Download/DownloadDataset.py b/grid2op/Download/DownloadDataset.py index 3f0366ccf..477e614e6 100644 --- a/grid2op/Download/DownloadDataset.py +++ b/grid2op/Download/DownloadDataset.py @@ -16,12 +16,15 @@ from grid2op.Exceptions import Grid2OpException try: + import urllib.parse import urllib.request -except Exception as e: - raise RuntimeError("Impossible to find library urllib. Please install it.") +except Exception as exc_: # pragma: no cover + raise RuntimeError(f"Impossible to find library urllib. Please install it, error was:\n{exc_}.") URL_GRID2OP_DATA = "https://github.com/Tezirg/Grid2Op/releases/download/{}/{}" + DATASET_TAG_v0_1_0 = "datasets-v0.1.0" + DICT_URL_GRID2OP_DL = { "rte_case14_realistic": URL_GRID2OP_DATA.format( DATASET_TAG_v0_1_0, "rte_case14_realistic.tar.bz2" @@ -31,14 +34,17 @@ ), "l2rpn_2019": URL_GRID2OP_DATA.format(DATASET_TAG_v0_1_0, "l2rpn_2019.tar.bz2"), } + LI_VALID_ENV = sorted(['"{}"'.format(el) for el in DICT_URL_GRID2OP_DL.keys()]) +ALLOWED_SCHEMES = {"http", "https"} + # Archive extraction safety limits (S5042) _MAX_UNCOMPRESSED_SIZE = 30 * 1024 * 1024 * 1024 # 30 GB _MAX_COMPRESSION_RATIO = 100 # reject archives that expand more than 100× -class DownloadProgressBar(tqdm): +class DownloadProgressBar(tqdm): # pragma: no cover """ INTERNAL @@ -53,7 +59,7 @@ def update_to(self, b=1, bsize=1, tsize=None): self.update(b * bsize - self.n) -def download_url(url, output_path): +def download_url(url, output_path): # pragma: no cover """ INTERNAL @@ -69,13 +75,19 @@ def download_url(url, output_path): output_path: ``str`` The path where the data will be stored. """ + scheme = urllib.parse.urlparse(url).scheme.lower() + if scheme not in ALLOWED_SCHEMES: + raise ValueError( + f"Unsafe URL scheme '{scheme}'. Only {ALLOWED_SCHEMES} are allowed." + ) + with DownloadProgressBar( unit="B", unit_scale=True, miniters=1, desc=url.split("/")[-1] ) as t: urllib.request.urlretrieve(url, filename=output_path, reporthook=t.update_to) -def _aux_download(url, dataset_name, path_data, ds_name_dl=None): +def _aux_download(url, dataset_name, path_data, ds_name_dl=None): # pragma: no cover """ INTERNAL @@ -168,7 +180,7 @@ def _aux_download(url, dataset_name, path_data, ds_name_dl=None): ) -def main_download(dataset_name, path_data): +def main_download(dataset_name, path_data): # pragma: no cover """ INTERNAL diff --git a/grid2op/Episode/EpisodeReplay.py b/grid2op/Episode/EpisodeReplay.py index 519ba7595..25fd0daf8 100644 --- a/grid2op/Episode/EpisodeReplay.py +++ b/grid2op/Episode/EpisodeReplay.py @@ -17,7 +17,7 @@ from grid2op.Episode.EpisodeData import EpisodeData -class EpisodeReplay(object): +class EpisodeReplay(object): # pragma: no cover """ This class allows to see visually what an agent has done during an episode. It uses for now the "PlotMatplot" as the method to plot the different states of the system. It reads directly data from the runner. diff --git a/grid2op/Exceptions/__init__.py b/grid2op/Exceptions/__init__.py index 03614f934..5d09c0f39 100644 --- a/grid2op/Exceptions/__init__.py +++ b/grid2op/Exceptions/__init__.py @@ -36,6 +36,7 @@ "InvalidLineStatus", "InvalidStorage", "InvalidCurtailment", + "InvalidAction", "AmbiguousAction", "NonFiniteElement", "AmbiguousActionRaiseAlert", @@ -67,6 +68,7 @@ "SomeGeneratorBelowRampmin", "ImpossibleRedispatching", "InvalidBackendCallback", + "RewardException", ] from grid2op.Exceptions.grid2OpException import Grid2OpException @@ -102,7 +104,8 @@ IllegalRedispatching, ) -from grid2op.Exceptions.ambiguousActionExceptions import (NotEnoughGenerators, +from grid2op.Exceptions.ambiguousActionExceptions import (InvalidAction, + NotEnoughGenerators, GeneratorTurnedOffTooSoon, GeneratorTurnedOnTooSoon, InvalidRedispatching, @@ -156,3 +159,6 @@ from grid2op.Exceptions.agentError import AgentError from grid2op.Exceptions.simulatorExceptions import SimulatorError + +from grid2op.Exceptions.rewardExceptions import RewardException + diff --git a/grid2op/Exceptions/ambiguousActionExceptions.py b/grid2op/Exceptions/ambiguousActionExceptions.py index 5eb9def23..46b6da113 100644 --- a/grid2op/Exceptions/ambiguousActionExceptions.py +++ b/grid2op/Exceptions/ambiguousActionExceptions.py @@ -9,6 +9,14 @@ from grid2op.Exceptions.grid2OpException import Grid2OpException +class InvalidAction(Grid2OpException): + """ + This exception indicate that the :class:`grid2op.BaseAction` has most likely not been + properly initialized. + """ + + pass + # ambiguous action class AmbiguousAction(Grid2OpException): """ diff --git a/grid2op/Exceptions/rewardExceptions.py b/grid2op/Exceptions/rewardExceptions.py new file mode 100644 index 000000000..3bad0dab0 --- /dev/null +++ b/grid2op/Exceptions/rewardExceptions.py @@ -0,0 +1,18 @@ +# Copyright (c) 2019-2022, RTE (https://www.rte-france.com) +# See AUTHORS.txt +# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 +# This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + +from grid2op.Exceptions.grid2OpException import Grid2OpException + + +class RewardException(Grid2OpException): + """ + This exception indicate that the reward you are trying to use has + encounter an error. + """ + + pass diff --git a/grid2op/MakeEnv/get_default_env_kwargs.py b/grid2op/MakeEnv/get_default_env_kwargs.py index 1850d0e93..43d5202cc 100644 --- a/grid2op/MakeEnv/get_default_env_kwargs.py +++ b/grid2op/MakeEnv/get_default_env_kwargs.py @@ -567,7 +567,6 @@ def get_default_env_kwargs( build_kwargs=data_feeding_kwargs, msg_error=ERR_MSG_KWARGS["chronics_handler"], ) - ### other rewards other_rewards_cfg = {} if "other_rewards" in config_data and config_data["other_rewards"] is not None: diff --git a/grid2op/Plot/BasePlot.py b/grid2op/Plot/BasePlot.py index c5f9d78f4..f2b84a5ec 100644 --- a/grid2op/Plot/BasePlot.py +++ b/grid2op/Plot/BasePlot.py @@ -24,7 +24,7 @@ from grid2op.Exceptions import PlotError -class BasePlot(GridObjects): +class BasePlot(GridObjects): # pragma: no cover """ INTERNAL diff --git a/grid2op/Plot/EpisodeReplay.py b/grid2op/Plot/EpisodeReplay.py index 0e3b4c1c3..d363e5b72 100644 --- a/grid2op/Plot/EpisodeReplay.py +++ b/grid2op/Plot/EpisodeReplay.py @@ -36,7 +36,7 @@ can_save_gif = False -class EpisodeReplay(object): +class EpisodeReplay(object): # pragma: no cover """ INTERNAL diff --git a/grid2op/Plot/PlotMatplotlib.py b/grid2op/Plot/PlotMatplotlib.py index 665bf223b..72f782aa4 100644 --- a/grid2op/Plot/PlotMatplotlib.py +++ b/grid2op/Plot/PlotMatplotlib.py @@ -43,14 +43,14 @@ from matplotlib.lines import Line2D # type: ignore can_plot = True -except ImportError as exc_: # noqa: F841 +except ImportError as exc_: # noqa: F841 # pragma: no cover can_plot = False pass # TODO add tests there -class PlotMatplotlib(BasePlot): +class PlotMatplotlib(BasePlot): # pragma: no cover """ INTERNAL diff --git a/grid2op/Plot/PlotPlotly.py b/grid2op/Plot/PlotPlotly.py index 30a68cc8c..2844db877 100644 --- a/grid2op/Plot/PlotPlotly.py +++ b/grid2op/Plot/PlotPlotly.py @@ -55,7 +55,7 @@ import seaborn as sns # type: ignore can_plot = True -except ImportError: +except ImportError: # pragma: no cover can_plot = False pass @@ -63,7 +63,7 @@ # Some utilities to plot substation, lines or get the color id for the colormap. -def draw_sub(pos, radius=50, line_color="LightSeaGreen"): +def draw_sub(pos, radius=50, line_color="LightSeaGreen"): # pragma: no cover """ INTERNAL @@ -100,7 +100,7 @@ def draw_sub(pos, radius=50, line_color="LightSeaGreen"): return res -def get_col(rho): +def get_col(rho): # pragma: no cover """ INTERNAL @@ -132,7 +132,7 @@ def get_col(rho): return 6 -def draw_line(pos_sub_or, pos_sub_ex, rho, color_palette, status, line_color="gray"): +def draw_line(pos_sub_or, pos_sub_ex, rho, color_palette, status, line_color="gray"): # pragma: no cover """ INTERNAL @@ -180,7 +180,7 @@ def draw_line(pos_sub_or, pos_sub_ex, rho, color_palette, status, line_color="gr return res -class PlotPlotly(BasePlot): +class PlotPlotly(BasePlot): # pragma: no cover """ INTERNAL diff --git a/grid2op/Plot/PlotPyGame.py b/grid2op/Plot/PlotPyGame.py index ae4d9c770..2cc07d191 100644 --- a/grid2op/Plot/PlotPyGame.py +++ b/grid2op/Plot/PlotPyGame.py @@ -34,7 +34,7 @@ pass -class Point: +class Point: # pragma: no cover # https://codereview.stackexchange.com/questions/70143/drawing-a-dashed-line-with-pygame # constructed using a normal tupple def __init__(self, point_t=(0, 0)): @@ -75,7 +75,7 @@ def from_cplx(cplx): return Point((cplx.real, cplx.imag)) -def _draw_dashed_line(surf, color, start_pos, end_pos, width=1, dash_length=10): +def _draw_dashed_line(surf, color, start_pos, end_pos, width=1, dash_length=10): # pragma: no cover # https://codereview.stackexchange.com/questions/70143/drawing-a-dashed-line-with-pygame origin = Point(start_pos) target = Point(end_pos) @@ -99,7 +99,7 @@ def _draw_arrow( num_arrows=10, length_arrow=10, angle_arrow=30, -): +): # pragma: no cover if positive_flow: origin = Point(start_pos) target = Point(end_pos) @@ -132,7 +132,7 @@ def _draw_arrow( ) -class PlotPyGame(BasePlot): +class PlotPyGame(BasePlot): # pragma: no cover """ INTERNAL diff --git a/grid2op/Plot/Plotting.py b/grid2op/Plot/Plotting.py index 0bf05b490..a545699ae 100644 --- a/grid2op/Plot/Plotting.py +++ b/grid2op/Plot/Plotting.py @@ -14,7 +14,7 @@ from grid2op.Exceptions.plotExceptions import PyGameQuit -class Plotting: +class Plotting: # pragma: no cover """ INTERNAL diff --git a/grid2op/PlotGrid/BasePlot.py b/grid2op/PlotGrid/BasePlot.py index 707c8d349..a15330342 100644 --- a/grid2op/PlotGrid/BasePlot.py +++ b/grid2op/PlotGrid/BasePlot.py @@ -19,7 +19,7 @@ from grid2op.dtypes import dt_float, dt_int -class BasePlot(ABC): +class BasePlot(ABC): # pragma: no cover """ INTERNAL diff --git a/grid2op/PlotGrid/LayoutUtil.py b/grid2op/PlotGrid/LayoutUtil.py index 99d329ab5..a1a73307e 100644 --- a/grid2op/PlotGrid/LayoutUtil.py +++ b/grid2op/PlotGrid/LayoutUtil.py @@ -12,7 +12,7 @@ import math -def layout_obs_sub_only(obs, scale=1000.0): +def layout_obs_sub_only(obs, scale=1000.0): # pragma: no cover n_line = obs.n_line or_sub = obs.line_or_to_subid ex_sub = obs.line_ex_to_subid @@ -52,7 +52,7 @@ def layout_obs_sub_only(obs, scale=1000.0): def layout_obs_sub_load_and_gen( obs, scale=1000.0, use_initial=False, parallel_spacing=3.0 -): +): # pragma: no cover # Create a graph of substations vertices G = nx.Graph() diff --git a/grid2op/PlotGrid/PlotMatplot.py b/grid2op/PlotGrid/PlotMatplot.py index 20aef5336..cd0aee293 100644 --- a/grid2op/PlotGrid/PlotMatplot.py +++ b/grid2op/PlotGrid/PlotMatplot.py @@ -21,7 +21,7 @@ from grid2op.PlotGrid.config import * # all colors -class GenDraw(patches.CirclePolygon): +class GenDraw(patches.CirclePolygon): # pragma: no cover """ INTERNAL @@ -34,7 +34,7 @@ def __init__(self, *args, resolution=5, **kwargs): patches.CirclePolygon.__init__(self, *args, resolution=resolution, **kwargs) -class LoadDraw(patches.CirclePolygon): +class LoadDraw(patches.CirclePolygon): # pragma: no cover """ INTERNAL @@ -47,7 +47,7 @@ def __init__(self, *args, resolution=3, **kwargs): patches.CirclePolygon.__init__(self, *args, resolution=resolution, **kwargs) -class StorageDraw(patches.CirclePolygon): +class StorageDraw(patches.CirclePolygon): # pragma: no cover """ INTERNAL @@ -67,7 +67,7 @@ def __init__(self, *args, resolution=4, **kwargs): # TODO code the load part in the plot_info -class PlotMatplot(BasePlot): +class PlotMatplot(BasePlot): # pragma: no cover """ This class uses the python library "matplotlib" to draw the powergrid. diff --git a/grid2op/PlotGrid/PlotPlotly.py b/grid2op/PlotGrid/PlotPlotly.py index 126e40ce9..c4aec1e4f 100644 --- a/grid2op/PlotGrid/PlotPlotly.py +++ b/grid2op/PlotGrid/PlotPlotly.py @@ -19,7 +19,7 @@ from grid2op.PlotGrid.PlotUtil import PlotUtil as pltu -class PlotPlotly(BasePlot): +class PlotPlotly(BasePlot): # pragma: no cover """ This class uses the python library "plotly" to draw the powergrid. Plotly has the ability to generate diff --git a/grid2op/PlotGrid/PlotUtil.py b/grid2op/PlotGrid/PlotUtil.py index fa28fd698..ccb66d416 100644 --- a/grid2op/PlotGrid/PlotUtil.py +++ b/grid2op/PlotGrid/PlotUtil.py @@ -9,7 +9,7 @@ import numpy as np -class PlotUtil: +class PlotUtil: # pragma: no cover """ INTERNAL diff --git a/grid2op/Reward/_alarmScore.py b/grid2op/Reward/_alarmScore.py index 82cc5a591..9725f76df 100644 --- a/grid2op/Reward/_alarmScore.py +++ b/grid2op/Reward/_alarmScore.py @@ -9,7 +9,7 @@ import numpy as np import copy -from grid2op.Exceptions import Grid2OpException +from grid2op.Exceptions import RewardException from grid2op.Reward import AlarmReward from grid2op.dtypes import dt_float @@ -69,7 +69,7 @@ def __init__(self, logger=None): def initialize(self, env): if not env._has_attention_budget: - raise Grid2OpException( + raise RewardException( 'Impossible to use the "_AlarmScore" with an environment for which this feature ' 'is disabled. Please make sure "env._has_attention_budget" is set to ``True`` or ' "change the reward class with `grid2op.make(..., reward_class=AnyOtherReward)`" @@ -105,7 +105,8 @@ def _lines_disconnected_first(self, disc_lines_at_cascading_time): disc_lines_to_consider_for_score = disc_lines_at_cascading_time == 0 # if we are there, it is because we have identified before that the failure is due to disconnected powerlines - assert (disc_lines_to_consider_for_score).any() + if (disc_lines_to_consider_for_score).all(): + raise RewardException("This reward should be called only if a line has been disconnected.") # we transform the vector so that disconnected lines have a zero, to be coherent with env._disc_lines return 1 - disc_lines_to_consider_for_score diff --git a/grid2op/Runner/runner.py b/grid2op/Runner/runner.py index 245ec2f34..9cfced78b 100644 --- a/grid2op/Runner/runner.py +++ b/grid2op/Runner/runner.py @@ -11,7 +11,7 @@ import copy import numpy as np from multiprocessing import get_start_method, get_context, Pool -from typing import Tuple, List, Union +from typing import Any, Optional, Tuple, List, Type, Union from grid2op.Environment import BaseEnv from grid2op.Action import BaseAction, TopologyAction, DontAct @@ -369,7 +369,7 @@ def __init__( # or GridStateFromFileWithForecasts otherwise backendClass=PandaPowerBackend, backend_kwargs=None, - agentClass=DoNothingAgent, # class used to build the agent + agentClass: Type[BaseAgent]=DoNothingAgent, # class used to build the agent agentInstance=None, verbose=False, gridStateclass_kwargs={}, @@ -1323,12 +1323,12 @@ def _clean_up(self): def run( self, - nb_episode, + nb_episode: int, *, # force kwargs - nb_process=1, - path_save=None, - max_iter=None, - pbar=False, + nb_process: int =1, + path_save: Optional[Union[os.PathLike, str]]=None, + max_iter:Optional[int]=None, + pbar: Any=False, env_seeds=None, agent_seeds=None, episode_id=None, diff --git a/grid2op/Space/GridObjects.py b/grid2op/Space/GridObjects.py index 30d78b24d..71e1b20be 100644 --- a/grid2op/Space/GridObjects.py +++ b/grid2op/Space/GridObjects.py @@ -25,6 +25,10 @@ from packaging import version from typing import Dict, Type, Union, Literal, Any, List, Optional, ClassVar, Tuple +try: + from typing import Self +except ImportError: + from typing_extensions import Self import grid2op from grid2op.dtypes import dt_int, dt_float, dt_bool @@ -2377,35 +2381,37 @@ def assert_grid_correct_cls(cls): def _check_validity_alarm_data(cls): if cls.dim_alarms == 0: # no alarm data - assert ( - cls.alarms_area_names == [] - ), "No alarm data is provided, yet cls.alarms_area_names != []" - assert ( - cls.alarms_lines_area == {} - ), "No alarm data is provided, yet cls.alarms_lines_area != {}" - assert ( - cls.alarms_area_lines == [] - ), "No alarm data is provided, yet cls.alarms_area_lines != []" + if cls.alarms_area_names != []: + raise Grid2OpException("No alarm data is provided, yet cls.alarms_area_names != []") + if cls.alarms_lines_area != {}: + raise Grid2OpException("No alarm data is provided, yet cls.alarms_lines_area != {}") + if cls.alarms_area_lines != []: + raise Grid2OpException("No alarm data is provided, yet cls.alarms_area_lines != []") elif cls.dim_alarms < 0: raise EnvError( f"The number of areas for the alarm feature should be >= 0. It currently is {cls.dim_alarms}" ) else: - assert cls.assistant_warning_type == "zonal" + if cls.assistant_warning_type != "zonal": + raise Grid2OpException("cls.assistant_warning_type shoud be 'zonal'") # the "alarm" feature is supported - assert isinstance( + if not isinstance( cls.alarms_area_names, (list, tuple) - ), "cls.alarms_area_names should be a list or a tuple" - assert isinstance( + ): + raise Grid2OpException("cls.alarms_area_names should be a list or a tuple") + if not isinstance( cls.alarms_lines_area, dict - ), "cls.alarms_lines_area should be a dict" - assert isinstance( + ): + raise Grid2OpException("cls.alarms_lines_area should be a dict") + if not isinstance( cls.alarms_area_lines, (list, tuple) - ), "cls.alarms_area_lines should be a list or a tuple" - assert ( - len(cls.alarms_area_names) == cls.dim_alarms - ), "len(cls.alarms_area_names) != cls.dim_alarms" + ): + raise Grid2OpException("cls.alarms_area_lines should be a list or a tuple") + if ( + len(cls.alarms_area_names) != cls.dim_alarms + ): + raise Grid2OpException("len(cls.alarms_area_names) != cls.dim_alarms") names_to_id = {nm: id_ for id_, nm in enumerate(cls.alarms_area_names)} # check that information in alarms_lines_area and alarms_area_lines match @@ -2413,24 +2419,26 @@ def _check_validity_alarm_data(cls): for area_nm in li_area: area_id = names_to_id[area_nm] all_lines_this_area = cls.alarms_area_lines[area_id] - assert l_nm in all_lines_this_area, ( - f'line "{l_nm}" is said to belong to area "{area_nm}" ' - f"in cls.alarms_lines_area yet when looking for the lines in " - f"this " - f"area in cls.alarms_area_lines, this line is not in there" - ) + if l_nm not in all_lines_this_area: + raise Grid2OpException( + f'line "{l_nm}" is said to belong to area "{area_nm}" ' + f"in cls.alarms_lines_area yet when looking for the lines in " + f"this " + f"area in cls.alarms_area_lines, this line is not in there" + ) for area_id, all_lines_this_area in enumerate(cls.alarms_area_lines): area_nm = cls.alarms_area_names[area_id] for l_nm in all_lines_this_area: - assert area_nm in cls.alarms_lines_area[l_nm], ( - f'line "{l_nm}" is said to belong to area ' - f'"{area_nm}" ' - f"in cls.alarms_area_lines yet when looking for " - f"the areas where this line belong in " - f"cls.alarms_lines_area it appears it does not " - f"belong there." - ) + if area_nm not in cls.alarms_lines_area[l_nm]: + raise Grid2OpException( + f'line "{l_nm}" is said to belong to area ' + f'"{area_nm}" ' + f"in cls.alarms_area_lines yet when looking for " + f"the areas where this line belong in " + f"cls.alarms_lines_area it appears it does not " + f"belong there." + ) # now check that all lines are in at least one area for line, li_area in cls.alarms_lines_area.items(): @@ -2449,28 +2457,32 @@ def _check_validity_alarm_data(cls): @classmethod def _check_validity_alert_data(cls): - # TODO remove assert and raise Grid2opExcpetion instead if cls.dim_alerts == 0: # no alert data - assert ( - cls.alertable_line_names == [] - ), "No alert data is provided, yet cls.alertable_line_names != []" - assert ( - len(cls.alertable_line_ids) == 0 - ), "No alert data is provided, yet len(cls.alertable_line_ids) != 0" + if ( + cls.alertable_line_names != [] + ): + raise Grid2OpException("No alert data is provided, yet cls.alertable_line_names != []") + if ( + len(cls.alertable_line_ids) != 0 + ): + raise Grid2OpException("No alert data is provided, yet len(cls.alertable_line_ids) != 0") elif cls.dim_alerts < 0: raise EnvError( f"The number of lines for the alert feature should be >= 0. It currently is {cls.dim_alerts}" ) else: - assert cls.assistant_warning_type == "by_line" + if cls.assistant_warning_type != "by_line": + raise Grid2OpException("cls.assistant_warning_type should be 'by_line'") # the "alert" feature is supported - assert isinstance( + if not isinstance( cls.alertable_line_names, list - ), "cls.alertable_line_names should be a list" - assert ( - len(cls.alertable_line_names) == cls.dim_alerts - ), "len(cls.alertable_line_names) != cls.dim_alerts" + ): + raise Grid2OpException("cls.alertable_line_names should be a list") + if ( + len(cls.alertable_line_names) != cls.dim_alerts + ): + raise Grid2OpException("len(cls.alertable_line_names) != cls.dim_alerts") try: cls.alertable_line_ids = np.array(cls.alertable_line_ids).astype(dt_int) @@ -2969,7 +2981,7 @@ def _aux_init_grid_from_cls(cls, gridobj, name_res): return cls_res @classmethod - def init_grid(cls, gridobj, force=False, extra_name=None, force_module=None, _local_dir_cls=None): + def init_grid(cls, gridobj, force=False, extra_name=None, force_module=None, _local_dir_cls=None) -> Type[Self]: """ INTERNAL diff --git a/grid2op/Space/SerializableSpace.py b/grid2op/Space/SerializableSpace.py index 088187ea9..0281f61d2 100644 --- a/grid2op/Space/SerializableSpace.py +++ b/grid2op/Space/SerializableSpace.py @@ -11,6 +11,7 @@ import re import json import copy +from typing import Type from grid2op.Exceptions import Grid2OpException from grid2op.Space.space_utils import extract_from_dict, save_to_dict @@ -62,7 +63,11 @@ class SerializableSpace(GridObjects, RandomObject): """ - def __init__(self, gridobj, subtype=object, _init_grid=True, _local_dir_cls=None): + def __init__(self, + gridobj: GridObjects, + subtype: Type[GridObjects], + _init_grid=True, + _local_dir_cls=None): """ subtype: ``type`` diff --git a/grid2op/__init__.py b/grid2op/__init__.py index 16ed9ad9a..86e05cdbc 100644 --- a/grid2op/__init__.py +++ b/grid2op/__init__.py @@ -10,8 +10,9 @@ """ Grid2Op a testbed platform to model sequential decision making in power systems. """ +import sys -__version__ = '1.12.4' +__version__ = '1.12.5.dev0' __all__ = [ "Action", @@ -60,3 +61,4 @@ except ImportError as exc_: # grid2op is most likely not installed in editable mode from source pass + \ No newline at end of file diff --git a/grid2op/check_release.py b/grid2op/check_release.py new file mode 100644 index 000000000..180db750e --- /dev/null +++ b/grid2op/check_release.py @@ -0,0 +1,324 @@ +""" +grid2op/check_release.py +------------------------ +CLI subcommand for verifying the integrity of a local Grid2Op release asset +against known-good digests published on PyPI. + +Invocation (once wired into __main__.py): + grid2op.check_release Grid2Op-1.9.8-py3-none-any.whl + grid2op.check_release Grid2Op-1.9.8.tar.gz --sha256 + grid2op.check_release Grid2Op-1.9.8-py3-none-any.whl --md5 --sha256 --blake2b + grid2op.check_release Grid2Op-1.9.8-py3-none-any.whl --all + grid2op.check_release Grid2Op-1.9.8-py3-none-any.whl --offline digests.json + +Exit codes: + 0 all requested checks passed + 1 one or more checks failed or file not recognised + 2 usage / argument error +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +import urllib.parse +import urllib.error +import urllib.request +from pathlib import Path + +from grid2op.Download.DownloadDataset import ALLOWED_SCHEMES + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +PYPI_VERSION_URL = "https://pypi.org/pypi/{package}/{version}/json" +PACKAGE_NAME = "Grid2Op" + + +# Regex to extract version from a Grid2Op filename, e.g.: +# Grid2Op-1.9.8-py3-none-any.whl → 1.9.8 +# Grid2Op-1.9.8.tar.gz → 1.9.8 +# grid2op-1.10.5.post1-py3-none-any.whl → 1.10.5.post1 +# grid2op-1.12.2.dev0.tar.gz → 1.12.2.dev0 +# Grid2Op-1.9.8-py3-none-any.whl → 1.9.8 +# Grid2Op-1.9.8.tar.gz → 1.9.8 +# Grid2Op-1.7.2rc2.tar.gz → 1.7.2rc2 + +_VERSION_RE = re.compile( + r"^grid2op-(\d+(?:\.\d+)+(?:\.(?:post|dev)\d+|[ab]\d+|rc\d+)?)[-.]", + re.IGNORECASE, +) + +DIGEST_ALGORITHMS = { + "md5": ("md5", hashlib.md5), + "sha256": ("sha256", hashlib.sha256), + "blake2b_256":("blake2b_256", lambda: hashlib.blake2b(digest_size=32)), +} + +CHUNK = 1 << 20 # 1 MiB read chunks + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _compute_digests(path: Path, algorithms: list[str]) -> dict[str, str]: # pragma: no cover + hashers = {alg: DIGEST_ALGORITHMS[alg][1]() for alg in algorithms} + with path.open("rb") as fh: + while chunk := fh.read(CHUNK): + for h in hashers.values(): + h.update(chunk) + return {alg: hashers[alg].hexdigest() for alg in algorithms} + + +def _fetch_pypi_digests(version: str, filename: str) -> dict[str, str] | None: # pragma: no cover + """Return the digest dict for *filename* from the PyPI JSON API, or None.""" + url = PYPI_VERSION_URL.format(package=PACKAGE_NAME, version=version) + scheme = urllib.parse.urlparse(url).scheme.lower() + if scheme not in ALLOWED_SCHEMES: + raise ValueError( + f"Unsafe URL scheme '{scheme}'. Only {ALLOWED_SCHEMES} are allowed." + ) + try: + with urllib.request.urlopen(url, timeout=15) as resp: + data = json.loads(resp.read().decode()) + except urllib.error.HTTPError as exc: + if exc.code == 404: + return None + raise + for f in data.get("urls", []): + if f["filename"] == filename: + return f["digests"] # keys: md5, sha256, blake2b_256 + return None + + +def _load_offline_digests(json_path: Path, filename: str) -> dict[str, str] | None: # pragma: no cover + """Return digest dict from a local JSON file produced by fetch_release_digests.py.""" + data = json.loads(json_path.read_text(encoding="utf-8")) + for _version, entries in data.get("releases", {}).items(): + for entry in entries: + if entry["filename"] == filename: + return entry["digests"] + return None + + +def _colour(text: str, ok: bool) -> str: # pragma: no cover + """ANSI green for pass, red for fail — degrades gracefully on Windows.""" + if sys.stdout.isatty(): + code = "\033[32m" if ok else "\033[31m" + return f"{code}{text}\033[0m" + return text + + +# --------------------------------------------------------------------------- +# Core verification logic +# --------------------------------------------------------------------------- + +def verify( + asset: Path, + algorithms: list[str], + offline_db: Path | None = None, + quiet: bool = False, +) -> bool: # pragma: no cover + """ + Verify *asset* against PyPI (or an offline digest DB). + Returns True if every requested check passes. + """ + filename = asset.name + + # Extract version from filename + m = _VERSION_RE.match(filename) + if not m: + print( + f"ERROR: Cannot determine Grid2Op version from filename '{filename}'.\n" + "Expected a name like 'Grid2Op-1.9.8-py3-none-any.whl' or 'Grid2Op-1.9.8.tar.gz'.", + file=sys.stderr, + ) + return False + + version = m.group(1) + + if not quiet: + print(f"File : {filename}") + print(f"Version : {version}") + + # Fetch expected digests + if offline_db is not None: + if not quiet: + print(f"Source : offline ({offline_db})") + expected = _load_offline_digests(offline_db, filename) + else: + if not quiet: + print("Source : PyPI (live)") + try: + expected = _fetch_pypi_digests(version, filename) + except urllib.error.URLError as exc: + print( + f"ERROR: Could not reach PyPI — {exc.reason}\n" + "Tip: use --offline to check without internet access.", + file=sys.stderr, + ) + return False + + if expected is None: + print( + f"ERROR: '{filename}' was not found on PyPI for version {version}.\n" + "Double-check the filename or use --offline if this is a private build.", + file=sys.stderr, + ) + return False + + # Compute local digests + if not quiet: + print(f"Checking: {', '.join(algorithms)}\n") + + local = _compute_digests(asset, algorithms) + + all_ok = True + for alg in algorithms: + expected_val = expected.get(alg, "") + local_val = local[alg] + ok = expected_val and (local_val == expected_val) + all_ok = all_ok and ok + + label = alg.upper().replace("_", "-") + status = _colour("PASS", ok) if ok else _colour("FAIL", ok) + + if not quiet: + print(f" {label:<16} {status}") + if not ok: + print(f" expected : {expected_val or '(not available)'}") + print(f" got : {local_val}") + + if not quiet: + print() + if all_ok: + print(_colour("✓ All checks passed.", True)) + else: + print(_colour("✗ One or more checks FAILED. Do not use this file.", False)) + + return all_ok + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + +def build_parser() -> argparse.ArgumentParser: # pragma: no cover + parser = argparse.ArgumentParser( + prog="grid2op.check_release", + description=( + "Verify the integrity of a Grid2Op release asset (.whl or .tar.gz) " + "by comparing its digests against those published on PyPI." + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Check a wheel with BLAKE2b-256 (default) + grid2op.check_release Grid2Op-1.9.8-py3-none-any.whl + + # Check with SHA-256 explicitly + grid2op.check_release Grid2Op-1.9.8-py3-none-any.whl --sha256 + + # Check with all three algorithms + grid2op.check_release Grid2Op-1.9.8-py3-none-any.whl --all + + # Check specific algorithms + grid2op.check_release Grid2Op-1.9.8.tar.gz --md5 --blake2b + + # Check without internet access using a local digest DB + grid2op.check_release Grid2Op-1.9.8-py3-none-any.whl --offline release_digests.json +""", + ) + + parser.add_argument( + "asset", + type=Path, + help="Path to the .whl or .tar.gz file to verify.", + ) + + algo_group = parser.add_argument_group("digest algorithms (default: --blake2b)") + algo_group.add_argument( + "--md5", + action="store_true", + help="Verify MD5 digest (weak, kept for legacy compatibility).", + ) + algo_group.add_argument( + "--sha256", + action="store_true", + help="Verify SHA-256 digest.", + ) + algo_group.add_argument( + "--blake2b", + action="store_true", + help="Verify BLAKE2b-256 digest.", + ) + algo_group.add_argument( + "--all", + dest="all_algos", + action="store_true", + help="Verify all three digests (MD5 + SHA-256 + BLAKE2b-256).", + ) + + parser.add_argument( + "--offline", + metavar="DIGESTS_JSON", + type=Path, + default=None, + help=( + "Path to a local digest DB (JSON produced by fetch_release_digests.py). " + "When omitted, digests are fetched live from PyPI. " + "Use this for air-gapped environments or to avoid network calls." + ), + ) + parser.add_argument( + "--quiet", + action="store_true", + help="Suppress all output; rely solely on the exit code.", + ) + + return parser + + +def main(argv: list[str] | None = None) -> int: # pragma: no cover + parser = build_parser() + args = parser.parse_args(argv) + + # Resolve requested algorithms + if args.all_algos: + algorithms = ["md5", "sha256", "blake2b_256"] + else: + algorithms = [] + if args.md5: + algorithms.append("md5") + if args.sha256: + algorithms.append("sha256") + if args.blake2b: + algorithms.append("blake2b_256") + if not algorithms: + algorithms = ["blake2b_256"] # default + + # Validate file path + asset: Path = args.asset + if not asset.exists(): + print(f"ERROR: File not found: {asset}", file=sys.stderr) + return 1 + if not asset.is_file(): + print(f"ERROR: Not a file: {asset}", file=sys.stderr) + return 1 + + ok = verify( + asset=asset, + algorithms=algorithms, + offline_db=args.offline, + quiet=args.quiet, + ) + return 0 if ok else 1 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/grid2op/command_line.py b/grid2op/command_line.py index 970be2e73..a1df38e12 100644 --- a/grid2op/command_line.py +++ b/grid2op/command_line.py @@ -38,15 +38,15 @@ ] -def main(): +def main(): # pragma: no cover mainEntryPoint() -def download(): +def download(): # pragma: no cover downloadEntryPoint() -def replay(): +def replay(): # pragma: no cover try: from grid2op.Episode.EpisodeReplay import main as replayEntryPoint @@ -59,7 +59,12 @@ def replay(): warnings.warn(warn_msg.format(e)) -def testinstall(): +def check_release(): # pragma: no cover + from grid2op.check_release import main as _main_check_release + _main_check_release() + + +def testinstall(): # pragma: no cover """ Performs basic tests to make sure grid2op is properly installed and working. diff --git a/grid2op/data/l2rpn_case14_sandbox_diff_grid/change_grid_params.py b/grid2op/data/l2rpn_case14_sandbox_diff_grid/change_grid_params.py index 756a0ef4d..1b29d9eee 100644 --- a/grid2op/data/l2rpn_case14_sandbox_diff_grid/change_grid_params.py +++ b/grid2op/data/l2rpn_case14_sandbox_diff_grid/change_grid_params.py @@ -24,8 +24,10 @@ pp.runpp(sim_case) pp.runpp(real_case) -assert sim_case.converged -assert sim_case.res_line.shape[0] == sim_case.line.shape[0] +if not sim_case.converged: + raise RuntimeError("Should have converged") +if sim_case.res_line.shape[0] != sim_case.line.shape[0]: + raise RuntimeError("Should have the same number of lines") print(f"L1 error on p: {np.mean(np.abs(sim_case.res_line['p_from_mw'] - real_case.res_line['p_from_mw'])):.2f}MW") print(f"L1 error on q: {np.mean(np.abs(sim_case.res_line['q_from_mvar'] - real_case.res_line['q_from_mvar'])):.2f}MVAr") pp.to_json(sim_case, "grid_forecast.json") diff --git a/grid2op/gym_compat/gym_act_space.py b/grid2op/gym_compat/gym_act_space.py index 768d2951d..3a8f774ed 100644 --- a/grid2op/gym_compat/gym_act_space.py +++ b/grid2op/gym_compat/gym_act_space.py @@ -15,6 +15,7 @@ BaseMultiProcessEnvironment, ) from grid2op.Action import BaseAction, ActionSpace +from grid2op.Exceptions.grid2OpException import Grid2OpException from grid2op.dtypes import dt_int, dt_bool, dt_float from grid2op.Converter.Converters import Converter from grid2op.gym_compat.utils import GYM_AVAILABLE, GYMNASIUM_AVAILABLE, DictType @@ -368,9 +369,8 @@ def to_gym(self, action: object) -> DictType: gym_action = self._converter.convert_action_to_gym(action) else: # in that case action should be an instance of grid2op BaseAction - assert isinstance( - action, BaseAction - ), "impossible to convert an action not coming from grid2op" + if not isinstance(action, BaseAction): + raise Grid2OpException("impossible to convert an action not coming from grid2op") # TODO this do not work in case of multiple converter, # TODO this should somehow call tmp = self._keys_encoding[internal_k].g2op_to_gym(v) gym_action = self._base_to_gym( diff --git a/grid2op/tests/helper_list_test.py b/grid2op/tests/helper_list_test.py index e85079926..77cec8ee8 100755 --- a/grid2op/tests/helper_list_test.py +++ b/grid2op/tests/helper_list_test.py @@ -3,6 +3,7 @@ li_tested_elsewhere = [ # agent (approx 1 min) "test_Agent", "test_AgentsFast", "test_recopowerlineperarea", + "test_Opponent", "test_baseline_alert", # converter (approx 45s) "test_AgentConverter", "test_Converter", "test_BackendConverter", # Runner / EpisodeData / "score (3 mins)" @@ -31,12 +32,27 @@ "test_multi_steps_env", "test_simenv_blackout", "test_get_default_env_kwargs", + "test_Environment", + "test_EnvironmentCpy", + "test_generate_classes", + "test_basic_env_ls", # alert / alarm ( 1min) "test_AlarmFeature", "test_alert_gym_compat", "test_alert_obs_act", "test_alert_trust_score", "test_AlertReward", + # time_series / backend + "test_ts_handlers", + "test_ChronicsHandler", + "test_env_from_episode", + # backend + "test_basicBackendInterface", + "test_action_set_orig_state", + "test_PandaPowerBackendDefaultFunc", + "test_PandaPowerBackend", + "test_BackendAction" + # TODO simulate # TODO curtailment ] @@ -60,6 +76,15 @@ def print_suite(suite): do_print = False break test_name = "{}.{}.{}".format(testmodule, testsuite, testmethod) + + # sometimes tests are included in other tests + # and so they are executed like "grid2op.tests.test_Agent.TestAgent.test_0_donothing" + # this prevents it + for el in li_tested_elsewhere: + if test_name.startswith("grid2op.tests") and el in test_name: + do_print = False + break + if do_print: print(test_name) diff --git a/grid2op/tests/test_Action.py b/grid2op/tests/test_Action.py index 16e6c3311..af84911b1 100644 --- a/grid2op/tests/test_Action.py +++ b/grid2op/tests/test_Action.py @@ -9,8 +9,10 @@ import copy import json import re +from typing import Type import warnings import unittest +import os import numpy as np from abc import ABC, abstractmethod @@ -18,8 +20,29 @@ from grid2op.tests.helper_path_test import * from grid2op.dtypes import dt_int, dt_float, dt_bool -from grid2op.Exceptions import * -from grid2op.Action import * +from grid2op.Exceptions import ( + Grid2OpException, + InvalidLineStatus, + InvalidBusStatus, + NonFiniteElement, + AmbiguousAction, +) +from grid2op.Action import ( + ActionSpace, + TopologySetAndDispatchAction, + TopologySetAction, + TopologyChangeAndDispatchAction, + TopologyChangeAction, + BaseAction, + DispatchAction, + DontAct, + PowerlineChangeAndDispatchAction, + PowerlineChangeAction, + PowerlineSetAction, + PowerlineSetAndDispatchAction, + TopologyAndDispatchAction, + TopologyAction, +) from grid2op.Rules import RulesChecker, DefaultRules from grid2op.Space import GridObjects from grid2op.Space.space_utils import save_to_dict @@ -27,9 +50,11 @@ # TODO check that if i set the element of a powerline to -1, then it's working as intended (disconnect both ends) -def _get_action_grid_class(): +def _get_action_grid_class(nm_to_add: str=""): GridObjects._clear_class_attribute() - GridObjects.env_name = "test_action_env" + if nm_to_add == "": + nm_to_add = os.path.splitext(os.path.split(__file__)[-1])[0] + GridObjects.env_name = f"test_act_{nm_to_add}" GridObjects.n_busbar_per_sub = 2 GridObjects.detachment_is_allowed = False GridObjects.n_gen = 5 @@ -162,7 +187,7 @@ def _get_action_grid_class(): "sub_13", ], "name_storage": ["storage_0", "storage_1"], - "env_name": "test_action_env", + "env_name": "test_act_test_Action", "sub_info": [3, 7, 5, 6, 5, 6, 3, 2, 5, 3, 3, 3, 4, 3], "load_to_subid": [1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13], "gen_to_subid": [0, 1, 2, 5, 7], @@ -347,7 +372,7 @@ def _get_action_grid_class(): class TestActionBase(ABC): @abstractmethod - def _action_setup(self): + def _action_setup(self) -> Type[BaseAction]: pass def _skipMissingKey(self, key): @@ -421,10 +446,14 @@ def test_assign_attr_from_name(self): act._assign_attr_from_name( "_set_line_status", np.zeros(self.helper_action.n_line) ) + + def test_reset_is_do_nothing(self): + act = self.helper_action.sample() + act.reset() def test_eq_none(self): act = self.helper_action.sample() - assert not (act == None) + assert act is not None def test_eq_diff_grid(self): act = self.helper_action.sample() diff --git a/grid2op/tests/test_BackendAction.py b/grid2op/tests/test_BackendAction.py index 2a0918d85..e7af90f47 100644 --- a/grid2op/tests/test_BackendAction.py +++ b/grid2op/tests/test_BackendAction.py @@ -53,13 +53,13 @@ def apply_action(self, backendAction=None): shunts__, ) = backendAction() - tmp_prod_p = self._get_vector_inj["prod_p"](self._grid) + # tmp_prod_p = self._get_vector_inj["prod_p"](self._grid) if np.any(prod_p.changed): - tmp_prod_p.iloc[prod_p.changed] = prod_p.values[prod_p.changed] + self._grid.gen.iloc[prod_p.changed, self._prod_p_col_id] = prod_p.values[prod_p.changed] - tmp_prod_v = self._get_vector_inj["prod_v"](self._grid) + # tmp_prod_v = self._get_vector_inj["prod_v"](self._grid) if np.any(prod_v.changed): - tmp_prod_v.iloc[prod_v.changed] = ( + self._grid.gen.iloc[prod_v.changed, self._prod_v_col_id] = ( prod_v.values[prod_v.changed] / self.prod_pu_to_kv[prod_v.changed] ) @@ -67,13 +67,13 @@ def apply_action(self, backendAction=None): # handling of the slack bus, where "2" generators are present. self._grid["ext_grid"]["vm_pu"] = 1.0 * tmp_prod_v[self._id_bus_added] - tmp_load_p = self._get_vector_inj["load_p"](self._grid) + # tmp_load_p = self._get_vector_inj["load_p"](self._grid) if np.any(load_p.changed): - tmp_load_p.iloc[load_p.changed] = load_p.values[load_p.changed] + self._grid.load.iloc[load_p.changed, self._load_p_col_id] = load_p.values[load_p.changed] - tmp_load_q = self._get_vector_inj["load_q"](self._grid) + # tmp_load_q = self._get_vector_inj["load_q"](self._grid) if np.any(load_q.changed): - tmp_load_q.iloc[load_q.changed] = load_q.values[load_q.changed] + self._grid.load.iloc[load_q.changed, self._load_q_col_id] = load_q.values[load_q.changed] if type(self).shunts_data_available: shunt_p, shunt_q, shunt_bus = shunts__ @@ -106,10 +106,10 @@ def apply_action(self, backendAction=None): loads_bus = backendAction.get_loads_bus() for load_id, new_bus in loads_bus: if new_bus == -1: - self._grid.load["in_service"][load_id] = False + self._grid.load.iloc[load_id, self._in_service_load_col_id] = False else: - self._grid.load["in_service"][load_id] = True - self._grid.load["bus"][load_id] = ( + self._grid.load.iloc[load_id, self._in_service_load_col_id] = True + self._grid.load.iloc[load_id, self._bus_load_col_id] = ( self.load_to_subid[load_id] + (new_bus - 1) * self._nb_bus_before_for_test ) @@ -117,10 +117,10 @@ def apply_action(self, backendAction=None): gens_bus = backendAction.get_gens_bus() for gen_id, new_bus in gens_bus: if new_bus == -1: - self._grid.gen["in_service"][gen_id] = False + self._grid.gen.iloc[gen_id, self._in_service_gen_col_id] = False else: - self._grid.gen["in_service"][gen_id] = True - self._grid.gen["bus"][gen_id] = ( + self._grid.gen.iloc[gen_id, self._in_service_gen_col_id] = True + self._grid.gen.iloc[gen_id, self._bus_gen_col_id] = ( self.gen_to_subid[gen_id] + (new_bus - 1) * self._nb_bus_before_for_test ) @@ -138,18 +138,20 @@ def apply_action(self, backendAction=None): for line_id, new_bus in lines_or_bus: if line_id < self._nb_line_for_test: dt = self._grid.line - key = "from_bus" line_id_db = line_id + is_col_id = self._in_service_line_col_id + bus_col_id = self._from_bus_line_col_id else: dt = self._grid.trafo - key = "hv_bus" line_id_db = line_id - self._nb_line_for_test + is_col_id = self._in_service_trafo_col_id + bus_col_id = self._hv_bus_trafo_col_id if new_bus == -1: - dt["in_service"][line_id_db] = False + dt.iloc[line_id_db, is_col_id] = False else: - dt["in_service"][line_id_db] = True - dt[key][line_id_db] = ( + dt.iloc[line_id_db, is_col_id] = True + dt.iloc[line_id_db, bus_col_id] = ( self.line_or_to_subid[line_id] + (new_bus - 1) * self._nb_bus_before_for_test ) @@ -158,26 +160,29 @@ def apply_action(self, backendAction=None): for line_id, new_bus in lines_ex_bus: if line_id < self._nb_line_for_test: dt = self._grid.line - key = "to_bus" line_id_db = line_id + is_col_id = self._in_service_line_col_id + bus_col_id = self._to_bus_line_col_id else: dt = self._grid.trafo - key = "lv_bus" line_id_db = line_id - self._nb_line_for_test + is_col_id = self._in_service_trafo_col_id + bus_col_id = self._lv_bus_trafo_col_id if new_bus == -1: - dt["in_service"][line_id_db] = False + dt.iloc[line_id_db, is_col_id] = False else: - dt["in_service"][line_id_db] = True - dt[key][line_id_db] = ( + dt.iloc[line_id_db, is_col_id] = True + dt.iloc[line_id_db, bus_col_id] = ( self.line_ex_to_subid[line_id] + (new_bus - 1) * self._nb_bus_before_for_test ) - bus_is = self._grid.bus["in_service"] + # bus_is = self._grid.bus["in_service"] + bus_id_col = (self._grid.bus.columns == "in_service").nonzero()[0][0] for i, (bus1_status, bus2_status) in enumerate(active_bus): - bus_is[i] = bus1_status # no iloc for bus, don't ask me why please :-/ - bus_is[i + self._nb_bus_before_for_test] = bus2_status + self._grid.bus.iloc[i, bus_id_col] = bus1_status # no iloc for bus, don't ask me why please :-/ + self._grid.bus.iloc[i + self._nb_bus_before_for_test, bus_id_col] = bus2_status class TestXXXBus(unittest.TestCase): @@ -199,7 +204,7 @@ def setUp(self) -> None: ) seed = 0 self.nb_test = 10 - self.max_iter = 30 + self.max_iter = 10 self.envref.seed(seed) self.envtest.seed(seed) diff --git a/grid2op/tests/test_ChronicsHandler.py b/grid2op/tests/test_ChronicsHandler.py index 1bb3903e2..63cf284b6 100644 --- a/grid2op/tests/test_ChronicsHandler.py +++ b/grid2op/tests/test_ChronicsHandler.py @@ -6,6 +6,7 @@ # SPDX-License-Identifier: MPL-2.0 # This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. +import copy import pdb import warnings import pandas as pd @@ -1113,6 +1114,25 @@ def test_stopiteration_chunksize(self): raise RuntimeError("This should have thrown a StopIteration exception") except StopIteration: pass + + def test_can_make_env(self): + """test I can make an env with proper action for this feature""" + with warnings.catch_warnings(): + warnings.filterwarnings("ignore") + env = grid2op.make("rte_case14_test", + test=True, + _add_to_name=type(self).__name__, + chronics_class=Multifolder, + names_chronics_to_grid=self.names_chronics_to_backend, + chronics_path=self.path) + obs = env.reset(seed=0, options={"time serie id": 0}) + obs, reward, done, info = env.step(env.action_space({})) + for _ in range(10): + act = env.action_space.sample() + dict_ = act.as_dict() + dict_ser = act.as_serializable_dict() + act2 = env.action_space(dict_ser) + assert act == act2 class TestEnvChunk(HelperTests, unittest.TestCase): diff --git a/grid2op/tests/test_utils.py b/grid2op/tests/test_utils.py index 7220a6970..cfe224684 100644 --- a/grid2op/tests/test_utils.py +++ b/grid2op/tests/test_utils.py @@ -9,6 +9,7 @@ import copy import json import os +import re from hashlib import blake2b from unittest.mock import patch import numpy as np @@ -614,6 +615,42 @@ def test_regex_rejects_path_traversal(self): scores.clear_all() + def test_dotdot_passes_regex_but_containment_check_blocks_it(self): + """Layer 3 (realpath containment check, fix for S2083). + + A bare '..' has no '/' so it passes REGEX_SPLIT, which only blocks + multi-component paths like '../../etc/passwd'. Before the fix, + os.path.join(path_save, '..', META_FILE) would silently read a file + one directory above path_save. The realpath containment check now + catches this case. + """ + # First, confirm that '..' passes the regex — documenting WHY the regex + # alone is not sufficient. + self.assertIsNotNone( + re.match(EpisodeStatistics.REGEX_SPLIT_COMPILED, ".."), + "'..' contains only dots which are in the allowed charset — " + "regex alone cannot block single-component traversal", + ) + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore") + with grid2op.make( + "rte_case5_example", test=True, _add_to_name=type(self).__name__ + ) as env: + scores = ScoreL2RPN2020(env, nb_scenario=2, verbose=0, max_step=10) + my_agent = DoNothingAgent(env.action_space) + + malicious_meta = copy.deepcopy(scores.stat_dn.get_metadata()) + malicious_meta["0"]["scenario_name"] = ".." + + with patch.object( + scores.stat_dn, "get_metadata", return_value=malicious_meta + ): + with self.assertRaises(RuntimeError): + scores.get(my_agent) + + scores.clear_all() + if __name__ == "__main__": unittest.main() diff --git a/grid2op/utils/l2rpn_2020_scores.py b/grid2op/utils/l2rpn_2020_scores.py index 42aab719e..518be94ad 100644 --- a/grid2op/utils/l2rpn_2020_scores.py +++ b/grid2op/utils/l2rpn_2020_scores.py @@ -383,20 +383,35 @@ def get(self, agent, path_save=None, nb_process=1): all_scores = [] ts_survived = [] total_ts = [] + path_save_real = os.path.realpath(path_save) + # Build a map from name → absolute path using the real filesystem so + # that the tainted scenario_name from the metadata JSON is never used + # directly as a path component (satisfies CodeQL py/path-injection). + available_ep_dirs = { + entry: os.path.join(path_save_real, entry) + for entry in os.listdir(path_save_real) + if os.path.isdir(os.path.join(path_save_real, entry)) + } for ep_id in range(self.nb_scenario): this_ep_nm = meta_data_dn[f"{ep_id}"]["scenario_name"] if re.match(EpisodeStatistics.REGEX_SPLIT_COMPILED, this_ep_nm) is None: raise RuntimeError( f'The grid2op statistics name should match the regex "{EpisodeStatistics.REGEX_SPLIT}", it is currently {this_ep_nm}.' ) + if this_ep_nm not in available_ep_dirs: + raise RuntimeError( + f"Episode directory for scenario '{this_ep_nm}' was not found under {path_save}." + ) + # ep_dir comes from os.listdir (filesystem), not from the tainted metadata value. + ep_dir = available_ep_dirs[this_ep_nm] with open( - os.path.join(path_save, this_ep_nm, EpisodeData.META_FILE), + os.path.join(ep_dir, EpisodeData.META_FILE), "r", encoding="utf-8", ) as f: this_epi_meta = json.load(f) with open( - os.path.join(path_save, this_ep_nm, EpisodeData.OTHER_REWARDS_FILE), + os.path.join(ep_dir, EpisodeData.OTHER_REWARDS_FILE), "r", encoding="utf-8", ) as f: diff --git a/pyproject.toml b/pyproject.toml index 7e3826674..1b66da21c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -117,12 +117,16 @@ chronix2grid = [ "ChroniX2Grid>=1.2.0.post1", "pypsa<0.25" # otherwise does not work (need fix in chronix2grid) ] +dev = [ + "pre-commit" +] [project.scripts] "grid2op.main" = "grid2op.command_line:main" "grid2op.download" = "grid2op.command_line:download" "grid2op.replay" = "grid2op.command_line:replay" "grid2op.testinstall" = "grid2op.command_line:testinstall" +"grid2op.check_release" = "grid2op.command_line:check_release" [tool.setuptools] include-package-data = false @@ -131,6 +135,7 @@ include-package-data = false grid2op = [ "data/**/*", "data/**/.multimix", + "data/release_digest.json" ] [tool.setuptools.exclude-package-data] diff --git a/scripts/check-dco-msg.sh b/scripts/check-dco-msg.sh new file mode 100755 index 000000000..946c1aca3 --- /dev/null +++ b/scripts/check-dco-msg.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# check-dco-msg.sh - Called by pre-commit at commit-msg stage. +# $1 = path to the commit message file (provided by pre-commit automatically) + +if ! grep -qE "^Signed-off-by: .+ <.+@.+>" "$1"; then + echo "" + echo "ERROR: Missing DCO Signed-off-by trailer." + echo " Amend with: git commit --amend -s" + echo "" + exit 1 +fi diff --git a/scripts/check-dco.sh b/scripts/check-dco.sh new file mode 100755 index 000000000..54547db65 --- /dev/null +++ b/scripts/check-dco.sh @@ -0,0 +1,115 @@ +#!/bin/sh +# check-dco.sh - Verify every commit in the branch has a DCO Signed-off-by trailer. +# +# Usage (standalone): sh scripts/check-dco.sh [] +# Usage (pre-commit): configured as a commit-msg or post-commit stage hook +# +# The script checks that every commit between and HEAD carries a +# "Signed-off-by: Name " line that matches the committer identity. +# When called with no argument it checks only the latest commit (HEAD), which +# is the right behaviour when wired as a commit-msg hook via pre-commit. + +set -e + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +die() { + printf "${RED}ERROR:${NC} %s\n" "$*" >&2 + exit 1 +} + +warn() { + printf "${YELLOW}WARN:${NC} %s\n" "$*" >&2 +} + +ok() { + printf "${GREEN}OK:${NC} %s\n" "$*" +} + +# --------------------------------------------------------------------------- +# Determine the range of commits to check +# --------------------------------------------------------------------------- + +if [ -n "$1" ]; then + BASE="$1" + RANGE="${BASE}..HEAD" +else + # When invoked as a commit-msg hook the new commit is not yet finalised; + # fall back to checking HEAD only (the most recently created commit). + RANGE="HEAD~1..HEAD" + # If this is the very first commit in the repo HEAD~1 does not exist. + if ! git rev-parse --verify HEAD~1 >/dev/null 2>&1; then + RANGE="HEAD" + fi +fi + +COMMITS=$(git log --format="%H" "$RANGE" 2>/dev/null) || \ + die "Could not list commits for range '$RANGE'. Is '$BASE' a valid ref?" + +if [ -z "$COMMITS" ]; then + warn "No commits found in range '$RANGE' — nothing to check." + exit 0 +fi + +# --------------------------------------------------------------------------- +# Check each commit +# --------------------------------------------------------------------------- + +FAILED=0 + +for COMMIT in $COMMITS; do + SUBJECT=$(git log -1 --format="%s" "$COMMIT") + AUTHOR_NAME=$(git log -1 --format="%an" "$COMMIT") + AUTHOR_EMAIL=$(git log -1 --format="%ae" "$COMMIT") + MESSAGE=$(git log -1 --format="%B" "$COMMIT") + + # Look for at least one "Signed-off-by: Name " trailer + if echo "$MESSAGE" | grep -qE "^Signed-off-by: .+ <.+@.+>"; then + # Optional stricter check: the sign-off must match the author identity. + EXPECTED="Signed-off-by: ${AUTHOR_NAME} <${AUTHOR_EMAIL}>" + if echo "$MESSAGE" | grep -qF "$EXPECTED"; then + ok "$(git log -1 --format='%h' "$COMMIT") — $SUBJECT" + else + # Sign-off present but identity mismatch — warn but don't fail, + # because contributors sometimes use a different display name. + ACTUAL=$(echo "$MESSAGE" | grep "^Signed-off-by:" | head -1) + warn "$(git log -1 --format='%h' "$COMMIT") — sign-off identity mismatch" + warn " Expected : $EXPECTED" + warn " Found : $ACTUAL" + warn " (commit subject: $SUBJECT)" + fi + else + printf "${RED}FAIL:${NC} %s — %s\n" \ + "$(git log -1 --format='%h' "$COMMIT")" "$SUBJECT" >&2 + printf " Author : %s <%s>\n" "$AUTHOR_NAME" "$AUTHOR_EMAIL" >&2 + printf " Missing: Signed-off-by: %s <%s>\n\n" \ + "$AUTHOR_NAME" "$AUTHOR_EMAIL" >&2 + FAILED=$((FAILED + 1)) + fi +done + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- + +if [ "$FAILED" -gt 0 ]; then + printf "\n${RED}%d commit(s) are missing a DCO Signed-off-by trailer.${NC}\n\n" \ + "$FAILED" >&2 + printf "To fix the most recent commit:\n" + printf " git commit --amend -s\n\n" + printf "To fix an entire branch (replace with your branch point, e.g. origin/master):\n" + printf " git rebase --signoff \n\n" + printf "See https://developercertificate.org/ for details.\n" + exit 1 +fi + +printf "\n${GREEN}All commits are properly signed off.${NC}\n" +exit 0 + diff --git a/scripts/check_dco_msg.py b/scripts/check_dco_msg.py new file mode 100755 index 000000000..205696e9d --- /dev/null +++ b/scripts/check_dco_msg.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""check_dco_msg.py - Called by pre-commit at commit-msg stage. + +Usage: check_dco_msg.py +""" +import re +import sys + +DCO_PATTERN = re.compile(r"^Signed-off-by: .+ <.+@.+>", re.MULTILINE) + +if len(sys.argv) != 2: + print("Usage: check_dco_msg.py ", file=sys.stderr) + sys.exit(1) + +with open(sys.argv[1], encoding="utf-8") as f: + message = f.read() + +if not DCO_PATTERN.search(message): + print() + print("ERROR: Missing DCO Signed-off-by trailer.") + print(" Amend with: git commit --amend -s") + print() + sys.exit(1) diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 000000000..71cee0182 --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,17 @@ +sonar.organization=grid2op +sonar.projectKey=Grid2op_grid2op +# Path to sources +sonar.sources=grid2op +#sonar.exclusions= +#sonar.inclusions= + +# Path to tests +#sonar.tests= +#sonar.test.exclusions= +#sonar.test.inclusions= + +# Source encoding +#sonar.sourceEncoding=UTF-8 + +# Exclusions for copy-paste detection +#sonar.cpd.exclusions= diff --git a/utils/templateDockerFile b/utils/Dockerfile.template similarity index 100% rename from utils/templateDockerFile rename to utils/Dockerfile.template diff --git a/utils/templateDockerFile_light b/utils/Dockerfile.template.light similarity index 100% rename from utils/templateDockerFile_light rename to utils/Dockerfile.template.light diff --git a/utils/templateDockerFile_test b/utils/Dockerfile.template.test similarity index 100% rename from utils/templateDockerFile_test rename to utils/Dockerfile.template.test diff --git a/utils/fetch_release_digests.py b/utils/fetch_release_digests.py new file mode 100644 index 000000000..fd0a540f8 --- /dev/null +++ b/utils/fetch_release_digests.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +""" +fetch_release_digests.py +------------------------ +Fetch MD5, SHA256, and BLAKE2b digests for all Grid2Op release assets +from the PyPI JSON API and write them to a structured JSON file. + +Usage: + python fetch_release_digests.py + python fetch_release_digests.py --package Grid2Op --output digests.json + python fetch_release_digests.py --versions 1.9.6 1.9.7 1.9.8 +""" + +import argparse +import json +import sys +import urllib.request +import urllib.error +from datetime import datetime, timezone +from pathlib import Path + + +PYPI_JSON_URL = "https://pypi.org/pypi/{package}/json" + +PACKAGETYPE_LABEL = { + "bdist_wheel": "whl", + "sdist": "tar.gz", +} + + +def fetch_pypi_data(package: str) -> dict: + url = PYPI_JSON_URL.format(package=package) + try: + with urllib.request.urlopen(url, timeout=15) as resp: + return json.loads(resp.read().decode()) + except urllib.error.HTTPError as exc: + sys.exit(f"ERROR: PyPI returned HTTP {exc.code} for package '{package}'") + except urllib.error.URLError as exc: + sys.exit(f"ERROR: Could not reach PyPI — {exc.reason}") + + +def build_digest_db( + pypi_data: dict, + only_versions: list[str] | None = None, +) -> dict: + """ + Returns a dict structured as: + { + "package": "Grid2Op", + "generated_at": "", + "releases": { + "1.9.8": [ + { + "filename": "Grid2Op-1.9.8-py3-none-any.whl", + "format": "whl", + "upload_time": "2024-01-26T10:15:04Z", + "yanked": false, + "digests": { + "md5": "...", + "sha256": "...", + "blake2b_256": "..." + } + }, + ... + ], + ... + } + } + """ + package_name = pypi_data["info"]["name"] + releases_raw = pypi_data["releases"] + + versions = sorted(releases_raw.keys()) if only_versions is None else only_versions + unknown = set(versions) - set(releases_raw.keys()) + if unknown: + sys.exit(f"ERROR: Unknown version(s): {', '.join(sorted(unknown))}") + + releases_out: dict[str, list] = {} + + for version in versions: + files = releases_raw[version] + if not files: + # version exists but has no files (rare edge case) + releases_out[version] = [] + continue + + entries = [] + for f in files: + fmt = PACKAGETYPE_LABEL.get(f.get("packagetype", ""), f.get("packagetype", "unknown")) + entries.append( + { + "filename": f["filename"], + "format": fmt, + "upload_time": f.get("upload_time_iso_8601", f.get("upload_time", "")), + "yanked": f.get("yanked", False), + "yanked_reason": f.get("yanked_reason"), + "digests": { + "md5": f["digests"].get("md5", ""), + "sha256": f["digests"].get("sha256", ""), + "blake2b_256": f["digests"].get("blake2b_256", ""), + }, + } + ) + releases_out[version] = entries + + return { + "package": package_name, + "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "releases": releases_out, + } + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Fetch PyPI release digests for a package and write them to JSON." + ) + parser.add_argument( + "--package", + default="Grid2Op", + help="PyPI package name (default: Grid2Op)", + ) + parser.add_argument( + "--output", + default="release_digests.json", + help="Output JSON file path (default: release_digests.json)", + ) + parser.add_argument( + "--versions", + nargs="+", + metavar="VERSION", + default=None, + help="Restrict output to specific version(s). Default: all versions.", + ) + parser.add_argument( + "--indent", + type=int, + default=2, + help="JSON indentation level (default: 2)", + ) + args = parser.parse_args() + + print(f"Fetching PyPI data for '{args.package}'...") + pypi_data = fetch_pypi_data(args.package) + + print("Building digest database...") + db = build_digest_db(pypi_data, only_versions=args.versions) + + n_versions = len(db["releases"]) + n_files = sum(len(v) for v in db["releases"].values()) + print(f" → {n_versions} version(s), {n_files} file(s)") + + out_path = Path(args.output) + out_path.write_text(json.dumps(db, indent=args.indent), encoding="utf-8") + print(f"Written to {out_path.resolve()}") + + +if __name__ == "__main__": + main() diff --git a/utils/make_release.py b/utils/make_release.py index a1ad48546..9a74bb2e5 100644 --- a/utils/make_release.py +++ b/utils/make_release.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2020, RTE (https://www.rte-france.com) +# Copyright (c) 2019-2026, RTE (https://www.rte-france.com) # See AUTHORS.txt # This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. # If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, @@ -20,7 +20,7 @@ import time -def start_subprocess_print(li, sleepbefore=2, cwd=None): +def __start_subprocess_print(li, sleepbefore=2, cwd=None): print("Will execute command after {}s: \n\t{}".format(sleepbefore, " ".join(li))) time.sleep(sleepbefore) subprocess.run(li, cwd=cwd) @@ -28,7 +28,7 @@ def start_subprocess_print(li, sleepbefore=2, cwd=None): def modify_and_push_docker(version, # grid2op version path, - templateDockerFile_to_use="templateDockerFile", + templateDockerFile_to_use="Dockerfile.template", docker_versions=[], docker_tags=[]): # Dockerfile @@ -44,9 +44,9 @@ def modify_and_push_docker(version, # grid2op version # Create new docker containers for vers_ in docker_versions: - start_subprocess_print( + __start_subprocess_print( ["docker", "build"] + docker_tags + ["-t", "{}/grid2op:{}".format(dockeruser, vers_), "."], cwd=path) - start_subprocess_print(["docker", "push", "{}/grid2op:{}".format(dockeruser, vers_)], cwd=path) + __start_subprocess_print(["docker", "push", "{}/grid2op:{}".format(dockeruser, vers_)], cwd=path) if __name__ == "__main__": @@ -72,7 +72,10 @@ def modify_and_push_docker(version, # grid2op version "script \"update_version\": version should be formated as XX.YY.ZZ (eg 0.3.1). " "Please modify \"--version\" argument") from exc_ - regex_version = "[0-9]+\.[0-9]+\.[0-9]+(.post[0-9]+){0,1}(.rc[0-9]+){0,1}(.pre[0-9]+){0,1}(.dev[0-9]+){0,1}" + if not re.match(r'^[a-zA-Z0-9_-]+$', dockeruser): + raise ValueError(f"Invalid dockeruser: {dockeruser!r}") + + regex_version = r"[0-9]+\.[0-9]+\.[0-9]+(.post[0-9]+){0,1}(.rc[0-9]+){0,1}(.pre[0-9]+){0,1}(.dev[0-9]+){0,1}" # TODO use the official regex ! # see https://semver.org/ and https://regex101.com/r/Ly7O1x/3/ # regex_version = r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$" @@ -142,7 +145,7 @@ def modify_and_push_docker(version, # grid2op version f.write(new_setup) # Stage in git - start_subprocess_print(["git", "add", setup_path]) + __start_subprocess_print(["git", "add", setup_path]) # grid2op/__init__.py with open(grid2op_init, "r") as f: @@ -155,7 +158,7 @@ def modify_and_push_docker(version, # grid2op version f.write(new_setup) # Stage in git - start_subprocess_print(["git", "add", grid2op_init]) + __start_subprocess_print(["git", "add", grid2op_init]) # docs/conf.py docs_conf = os.path.join(path, "docs", "conf.py") @@ -170,10 +173,10 @@ def modify_and_push_docker(version, # grid2op version with open(docs_conf, "w") as f: f.write(new_setup) # Stage in git - start_subprocess_print(["git", "add", docs_conf]) + __start_subprocess_print(["git", "add", docs_conf]) # Dockerfile - template_dockerfile = os.path.join(path, "utils", "templateDockerFile") + template_dockerfile = os.path.join(path, "utils", "Dockerfile.template") dockerfile = os.path.join(path, "Dockerfile") with open(template_dockerfile, "r") as f: new_setup = f.read() @@ -185,7 +188,7 @@ def modify_and_push_docker(version, # grid2op version if not is_prerelease: # Stage in git - start_subprocess_print(["git", "add", dockerfile]) + __start_subprocess_print(["git", "add", dockerfile]) # generate some logs, for backward compatibility # NB this generation is part of the test run, so it's safe to re generate the log when each version is released @@ -206,7 +209,7 @@ def modify_and_push_docker(version, # grid2op version pbar=True, max_iter=100) # Stage in git - start_subprocess_print(["git", "add", f'{os.path.join(PATH_PREVIOUS_RUNNER, f"res_agent_{version}")}/*']) + __start_subprocess_print(["git", "add", f'{os.path.join(PATH_PREVIOUS_RUNNER, f"res_agent_{version}")}/*']) # Commit # start_subprocess_print(["git", "commit", "-s", "-S", "-m", "Release v{}".format(version)]) @@ -228,18 +231,18 @@ def modify_and_push_docker(version, # grid2op version sys.exit() # Wait for user to push changes pushed = input("Please push changes: 'git push && git push --tags' - then press any key") - # TODO refacto these, no need to have 3 times almost the same "templatedockerfile" + # TODO refacto these, no need to have 3 times almost the same "Dockerfile.template" # update docker for test version # TODO remove the "-e" in this docker file, and copy paste the data in data_test in the appropriate folder # that you can get with a python call modify_and_push_docker(version, path=path, - templateDockerFile_to_use="templateDockerFile_test", + templateDockerFile_to_use="Dockerfile.template.test", docker_versions=["test"], docker_tags=["--no-cache"]) # update docker for "light" modify_and_push_docker(version, path=path, - templateDockerFile_to_use="templateDockerFile_light", + templateDockerFile_to_use="Dockerfile.template.light", docker_versions=[f"{version}-light"], docker_tags=["--no-cache"]) # update version for competition and regular version diff --git a/utils/trigger_readthedocs.io.py b/utils/trigger_readthedocs.io.py deleted file mode 100644 index ad3ce17a4..000000000 --- a/utils/trigger_readthedocs.io.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (c) 2019-2020, RTE (https://www.rte-france.com) -# See AUTHORS.txt -# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. -# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, -# you can obtain one at http://mozilla.org/MPL/2.0/. -# SPDX-License-Identifier: MPL-2.0 -# This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. - -# This files allows to automatically update the documentation of grid2op on the -# readthedocs.io website. It should not be used for other purpose. - -import argparse -import json -import os -import re -import time - -try: - import requests as rq -except: - raise RuntimeError("Impossible to find library urllib. Please install it.") - -import pdb - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Update the version of grid2op in the python files.') - parser.add_argument('--version', default=None, - help='The new version to update.') - parser.add_argument('--path', default=os.path.abspath("."), - help='The path of the root directory of Grid2op (default {}'.format(os.path.abspath("."))) - - args = parser.parse_args() - path = args.path - version = args.version - - if args.version is None: - raise RuntimeError("script \"update_version\" should be called with a version number.") - - try: - maj_, min_, minmin_ = version.split(".") - except: - raise RuntimeError("script \"update_version\": version should be formated as XX.YY.ZZ (eg 0.3.1). Please modify \"--version\" argument") - - if re.match('^[0-9]+\.[0-9]+\.[0-9]+$', version) is None: - raise RuntimeError("script \"update_version\": version should be formated as XX.YY.ZZ (eg 0.3.1) and not {}. Please modify \"--version\" argument".format(version)) - - if not os.path.exists(".readthedocstoken.json"): - raise RuntimeError("Impossible to find credential for buildthedocs. Stopping there. Make sur to put them on \".readthedocstoken.json\"") - - with open(".readthedocstoken.json", "r") as f: - dict_credentials = json.load(f) - - token = dict_credentials["token"] - hdr = {"Authorization": "Token {}".format(token)} - - # curl \ - # -X POST \ - # -H "Authorization: Token " https://readthedocs.org/api/v3/projects/pip/versions/latest/builds/ - - # list existing version on read the doc: - url_existing_version = "https://readthedocs.org/api/v3/projects/grid2op/versions/" - req = rq.get(url_existing_version, headers=hdr) - resp = req.json() - li_existing_version = set() - for el in resp["results"]: - li_existing_version.add(el['slug']) - - # update new versions - url_version = "https://readthedocs.org/api/v3/projects/grid2op/versions/{version_slug}/builds/" - - for vers_ in ["v{}".format(version), "stable", "latest"]: - if vers_ in li_existing_version: - req = rq.post(url_version.format(version_slug=vers_), headers=hdr) - print("Version {} properly updated".format(vers_)) - time.sleep(5) - else: - raise RuntimeError("Version \"{}\" is not part of the read the doc version," - "please create it before updating it.".format(vers_))