Initial cve-core release

Author: afoote-mitre

.env-EXAMPLE

@@ -0,0 +1,43 @@
+################################################
+### cve-core ###################################
+################################################
+
+# ----- ----- runtime ----- ----- ----- -----
+CVES_BASE_DIRECTORY=cves
+CVES_RECENT_ACTIVITIES_FILENAME=recent_activities.json
+CVES_DEFAULT_UPDATE_LOOKBACK_IN_MINS=180
+CVES_DEFAULT_DELTA_LOG_HISTORY_IN_DAYS=30
+CVES_MAX_ALLOWABLE_CVE_YEAR=2026
+GIT_MAX_FILESIZE_MB=100
+# ----- ----- for testing only ----- ----- -----
+CVES_TEST_BASE_DIRECTORY=test/pretend_github_repository
+
+################################################
+### CVE Search specific ########################
+#################################################
+### LOCAL: OpenSearch variables  ----- ----- ----- 
+# The following values are setup for a local OpenSearch instance
+#  used for development and testing
+# These OpenSearch-specific variables must be modified in
+#  each OpenSearch operating environment.
+# Please ask the infrastructure team for new values if needed
+#  so they can update the infrastructure-as-code for consistency.
+OpenSearchCveIndex=cve-index-local
+# OpenSearchAllowUnknownSslCerts=true
+OpenSearchDomainEndpoint=https://admin:admin@localhost:9200
+
+
+################################################
+### cve services api ###########################
+################################################
+
+# ----- production environment ----- ----- -----
+# CVE Services API-specific ----- -----
+CVE_SERVICES_URL=https://cveawg.mitre.org
+CVE_SERVICES_RECORDS_PER_PAGE=500
+CVE_ORG_URL=https://www.cve.org
+
+# SECRET: user/role specific: DO NOT COMMIT ----- -----
+CVE_API_ORG=<yourOrg>
+CVE_API_USER=<yourUser>
+CVE_API_KEY=<serverToken>

.eslintignore

@@ -0,0 +1 @@
+/**/*.js

.eslintrc.json

@@ -0,0 +1,37 @@
+{
+  "env": {
+    "browser": false,
+    "es6": true,
+    "node": true
+  },
+  "parser": "@typescript-eslint/parser",
+  "parserOptions": {
+    "project": "tsconfig.json",
+    "sourceType": "module",
+    "ecmaVersion": 2020
+  },
+  "plugins": [
+    "@typescript-eslint",
+    "jest"
+  ],
+  "extends": [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/recommended",
+    "plugin:jest/recommended",
+    "prettier"
+  ],
+  "rules": {
+    // The following rule is enabled only to supplement the inline suppression
+    // examples, and because it is not a recommended rule, you should either
+    // disable it, or understand what it enforces.
+    // https://typescript-eslint.io/rules/explicit-function-return-type/
+    "@typescript-eslint/explicit-function-return-type": "warn",
+    "@typescript-eslint/no-empty-function": "warn",
+    "@typescript-eslint/no-inferrable-types": "off",
+    "@typescript-eslint/no-unused-vars": "warn",
+    // @todo temporarily turn off
+    "jest/no-export": "off",
+    // @todo changing this temporarily to warning (from error, which is the default)
+    "prefer-const": "warn"
+  }
+}

.github/dependabot.yml

@@ -0,0 +1,13 @@
+version: 2
+updates:
+  - package-ecosystem: "npm"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    # Raise pull requests for version updates
+    # against the `develop` branch
+    target-branch: "develop"
+    # Labels on pull requests for version updates only
+    labels:
+      - "dependabot"
+

.github/workflows/codeql.yml

@@ -0,0 +1,78 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: 'CodeQL'
+
+on:
+  push:
+    # run CodeQL on all branches on push
+    branches: ['**']
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: ['main', 'develop']
+  schedule:
+    - cron: '27 5 * * 3'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ['javascript']
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby', 'swift' ]
+        # Use only 'java' to analyze code written in Java, Kotlin or both
+        # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #     echo "Run, Build Application using script"
+    #     ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2
+      with:
+        category: "/language:${{matrix.language}}"

.gitignore

@@ -0,0 +1,57 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+temp/logs
+
+# work directories
+node_modules/
+.history
+
+# Coverage
+coverage
+
+# Transpiled files
+build/
+dist/
+dist.*/
+release/
+lib-cjs/
+lib-esm/
+
+# VS Code
+# .vscode
+# !.vscode/tasks.js
+
+# JetBrains IDEs
+.idea/
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Misc
+.DS_Store
+devel/
+# test/pretend_github_repository*/log
+*.old
+# test/pretend_github_repository*/20*
+*\ copy
+*\ copy.*
+*.old/
+
+Notes.md
+
+# deployment specific
+.env
+.env.*
+
+# development and testing files
+aws.temp
+bulk/converted*.jsonl
+cves/deltaLog.json
+data
+deltas/
+preview_cves/

ChangeLog.md

@@ -0,0 +1,84 @@
+# Change Log
+
+## 2.0.0-rc14
+  - initial version of `cve-core` as a peer project to other `cve-projects`.  Can be used as part of a monorepo
+  - search using `axios`, NodeJS-native `fetch` and `@opensearch-project/opensearch` libraries
+    - CVE-, CWE-, and CAPAC- IDs
+    - CVE YEAR
+    - basic version strings (e.g., "v3.2.5", "v3.2.5-RC1")
+    - basic IPv4 and IPv6
+    - URLs
+    - compound words (e.g., "docker-compose", "microsoft word")
+    - hyphenated words (e.g., "man-in-the-middle")
+    - software names (e.g., "Node.JS", ".NET")
+    - file extension (e.g., "matvar_struct.c")
+    - repeating non-language characters (e.g., "aaaaa" is ok, but "?????" is replaced by "")
+    - can run as AWS Lambda Layer
+  - new adapters
+    - CVE Services reader
+    - CVE Search reader
+    - CVE file reader
+    - file reader/writer
+    - console input for interacting with a user in a CLI
+  - CveResult class with standardized errors and messages (this version is aimed at the search service)
+  - object (JSON) comparer using `json-difference` library
+  - JSON replacer that alphabetizes keys when serializing using JSON.stringify()
+
+## Older Milestones from the older `cveUtils`/`cvelist-bulk-download` repositories
+
+Note that the following milestones were in other repositories, which contained a superset of the source code in this npm library.  The milestones below are meant only for historic reference, in case a full history of an implementation is needed.
+
+### 1.2.0 - deployed 2024-07-18 (tag `2024-07-18_v1.2.0`)
+  - baseline for the `cve-core` npm library
+  - changes for cisa adp, reference ingest
+  - axios-retry for network retry
+  - optimized update.yml to use fetch-depth: 1
+  - CVES_MAX_ALLOWABLE_CVE_YEAR environment variable set to 2025
+  - GIT_MAX_FILESIZE_MB environment variable set to 100
+  - initial refactoring of core classes to separate I/O functions from business logic classes (work in progress)
+  - minimized 3rd party dependency in IsoDateString class to minimize footprint for AWS Lambda
+  - import specific lodash functions instead of the full lodash to minimize footprint for AWS Lambda
+  - dependabot PRs defaults to develop branch
+  - cveUtils/GitLab PR 32
+
+### 1.1.1 - 2024-06-03
+  - hotfix for large commit messages in anticipation of CISA adding ADP containers to a large number of CVEs on 6/4/2024.
+  - tested but not used on cvelistV5
+
+### 1.1.0 - 2023-09-26 (tag `2023-09-26_v1.1.0`)
+  - Delta files in /cves (delta.json and deltaLog.json), replacing recent_activities.json
+
+### 1.0.0 - 2023-05-26 (tag `2023-04-25_v1.0.0`)
+  - Official version using public domain code in https://github.com/CVEProject/cvelist-bulk-download
+
+
+### `Sprint-0` - 2023-04-20 (tag `2023-04-20_initial_cveUtils_on_github`)
+  - initial version selectively copied from internal MITRE gitlab to https://github.com/hkong-mitre/cvelist-bulk-download
+  - https://github.com/hkong-mitre/cvelist-bulk-download/commit/207b9f2b82908afbd8d9d2270969f6781f9d39e4
+  - (note date is different):  https://gitlab.mitre.org/hkong/cve_utils/-/tags/2023-04-25_to_github_hkong-mitre_cvelist_bulk_download
+
+
+### 2023-03-29
+  - official version used in GitHub actions that updated /cves when cvelistV5 was announced at CNA Summit 2023
+  - https://gitlab.mitre.org/hkong/cve_utils/-/tags/2023-03-29-cveproject_cvelistV5_dist_(similar)
+
+
+### 2023-03-10
+  - code during team code walkthru
+  - https://gitlab.mitre.org/hkong/cve_utils/-/tags/2023_03_10_code_walkthrough_with_team
+
+
+### 2023-03-06
+  - first version deployed to cvelistV5 for testing (using `preview_cves` instead of `cves`)
+  - https://gitlab.mitre.org/hkong/cve_utils/-/tags/2023_03_06_deployed_to_cveproject_cvelistv5
+
+
+## Additional Information
+
+This project uses (either verbatim or modified from) the following projects:
+
+1. [jsynowiec/node-typescript-boilerplate](https://github.com/jsynowiec/node-typescript-boilerplate) as a starter (8/26/2022).
+   - but not using [Volta][volta]
+2. [Quicktype](https://quicktype.io/) to convert CVE schemas to usable Typescript classes. Specifically, all classes in `src/generated/quicktype` are all generated this way:
+   - `Cve5`: https://raw.githubusercontent.com/CVEProject/cve-services/dev/schemas/cve/create-full-cve-record-request.json
+3. [recommended tsconfig](https://github.com/tsconfig/bases#centralized-recommendations-for-tsconfig-bases)