Typescript Migration: Tests (#1095)

* feat: convert errors.js to typescript  fix: fix issue with ava test runner not identifying the error typescript file and finding temporary workaround during migration.

* feat: creating base STAC types fix: updating linter overrides fro typescript

* feat: convert geo-utils and logger to typescript  feat: use BBox type from geojson lib

* feat: typing stac-utils fix: tweaking types based on experience in stac-utils.ts

* feat: deleted s3-utils.js which was slightly different from s3-utils.ts.  Confirmed using s3-utils.ts passes all tests

* feat: rounding out typing for s3-utils.ts

* feat: typing sns file

* feat: typing aws clients

* feat: converting database clients to typescript

* complete typing of asset-proxy and asset-buckets files tests: flesh out input object in test to be a more complete StacItem to pass an expected logic gate

* feat: first quick pass at typing Searchparams and DbOperation

* feat: updating changelog

* logger: change to 'info' not 'error' in case of index already existing

* feat: typing database.ts and updating types as required

* feat: cleaning up types, remove unused, consolidating where appropriate

* updating changelog

* feat: typing a few remainig untyped params

* feat: moved some utils to api-utils.ts and typed them  feat: changed api.js to api.ts

* feat: typing api.ts, adjusting types as needed

* feat: adding missing type, adding return type in a few spots where it was missing

* updating changelog

* PR feedback

* feat: typing ingest lambda and related files, adjusting types as needed

* converting lambdas to typescript

* feat: finishing typing lambdas, extending base express Request type, adding types as necessary

* updating changelog

* feat: converting test helpers to typescript

* typing unit tests

* feat: tweaking package and linter rules for typesript test conversion  feat: some of the LLM converted tests

* feat: rest of the system tests converted to typescript

* fix: PR feedback

* merge changes from api PR

* PR feedback changs after reviewing Claude suggestions

* updating .nsprc file

* feat: partial commit of typed tests after setting config to typecheck tests as well

* feat: fixing typing issues from removing implictAny: true setting

* feat: more tidying up of tests typescript conversion, linter fixes

* linter fix

---------

Co-authored-by: Matthew Hanson <matt.a.hanson@gmail.com>

Author: theodorehreuter

.eslintrc.json

@@ -154,7 +154,8 @@
   "overrides": [
     {
       "files": [
-        "test-*.js"
+        "test-*.js",
+        "test-*.ts"
       ],
       "rules": {
         "max-len": [

package.json

@@ -21,7 +21,7 @@
     "test:system:coverage": "c8 npm run test:system",
     "test:unit": "NODE_OPTIONS='--import=tsx' ava tests/unit/*.[tj]s --no-worker-threads",
     "test:unit:coverage": "c8 npm run test:unit",
-    "typecheck": "tsc",
+    "typecheck": "tsc && tsc -p tsconfig.test.json",
     "audit-prod": "npx better-npm-audit audit --production",
     "audit": "npx better-npm-audit audit",
     "deploy": "sls deploy",
@@ -33,7 +33,10 @@
   },
   "ava": {
     "verbose": true,
-    "timeout": "1m"
+    "timeout": "1m",
+    "extensions": [
+      "ts"
+    ]
   },
   "publishConfig": {
     "access": "public"

src/lambdas/api/app.ts

@@ -1,4 +1,3 @@
-// @ts-nocheck
 import cors from 'cors'
 import createError, { HttpError } from 'http-errors'
 import express, { Request, Response, NextFunction } from 'express'

src/lambdas/api/index.ts

@@ -25,7 +25,7 @@ const internalServerError: Readonly<APIGatewayProxyResult> = Object.freeze({
     'content-type': 'text/plain'
   },
   body: 'Internal Server Error'
-})
+}) as APIGatewayProxyResult
 
 let appInstance: express.Express | undefined
 

src/lib/api-utils.ts

@@ -39,9 +39,19 @@ export const extractLimit = function (params: APIParameters): number {
   return DEFAULT_LIMIT
 }
 
+type PrecisionField =
+  | 'grid_geohash_frequency_precision'
+  | 'grid_geohex_frequency_precision'
+  | 'grid_geotile_frequency_precision'
+  | 'centroid_geohash_grid_frequency_precision'
+  | 'centroid_geohex_grid_frequency_precision'
+  | 'centroid_geotile_grid_frequency_precision'
+  | 'geometry_geohash_grid_frequency_precision'
+  | 'geometry_geotile_grid_frequency_precision'
+
 export const extractPrecision = function (
   params: APIParameters,
-  name: string,
+  name: PrecisionField,
   min: number,
   max: number
 ): number {
@@ -435,7 +445,8 @@ export const isCollectionIdAllowed = function (
           || allowedCollectionIds.includes('*')
 }
 
-export const extractSortby = function (params: APIParameters): string[] {
+export const extractSortby = function (params: APIParameters):
+string[] | { field: string; direction: string; }[] | undefined {
   let sortbyRules
   const { sortby } = params
   if (sortby) {

src/lib/api.ts

@@ -284,15 +284,14 @@ const wrapResponseInFeatureCollection = function (
     numberMatched,
     numberReturned,
     features,
-    links
-  }
-
-  if (process.env['ENABLE_CONTEXT_EXTENSION']) {
-    fc['context'] = {
-      matched: numberMatched,
-      returned: numberReturned,
-      limit
-    }
+    links,
+    ...(process.env['ENABLE_CONTEXT_EXTENSION'] ? {
+      context: {
+        matched: numberMatched,
+        returned: numberReturned,
+        limit
+      }
+    } : {})
   }
 
   return fc
@@ -307,23 +306,24 @@ const buildPaginationLinks = function (
   filter: Cql2Filter,
   endpoint: string,
   httpMethod: string,
-  _sortby: string[],
+  _sortby: string[] | { field: string; direction: string; }[] | undefined,
   items: StacItem[],
-  lastItemSort: string | null
+  lastItemSort: string | null | undefined
 ): Link[] {
   if (items.length) {
-    const dictToURI = (dict: Partial<APIParameters>) => (
+    const dictToURI = (dict: Record<string, unknown>) => (
       Object.keys(dict).map(
         (p) => {
           let value = dict[p]
           if (typeof value === 'object' && value !== null) {
             if (p === 'sortby') {
               const sortFields: string[] = []
-              for (let i = 0; i < value.length; i += 1) {
-                if (value[i]['direction'] === 'asc') {
-                  sortFields.push(value[i]['field'])
+              const sortArray = value as Array<{ field: string; direction: string }>
+              for (let i = 0; i < sortArray.length; i += 1) {
+                if (sortArray[i]!['direction'] === 'asc') {
+                  sortFields.push(sortArray[i]!['field'])
                 } else {
-                  sortFields.push('-'.concat(value[i]['field']))
+                  sortFields.push('-'.concat(sortArray[i]!['field']))
                 }
               }
               value = sortFields.join(',')
@@ -333,7 +333,7 @@ const buildPaginationLinks = function (
               value = JSON.stringify(value)
             }
           }
-          const query = encodeURIComponent(value)
+          const query = encodeURIComponent(String(value))
           return `${encodeURIComponent(p)}=${query}`
         }
       ).join('&')
@@ -367,7 +367,7 @@ const buildPaginationLinks = function (
 const searchItems = async function (
   backend: Backend,
   httpMethod: string,
-  collectionId: string,
+  collectionId: string | null,
   endpoint: string,
   parameters: APIParameters,
   headers: IncomingHttpHeaders
@@ -498,7 +498,7 @@ const searchItems = async function (
     links.push({
       rel: 'collection',
       type: 'application/json',
-      href: collectionEndpoint
+      href: collectionEndpoint!
     })
   }
 
@@ -537,7 +537,7 @@ const agg = function (
 const aggregate = async function (
   backend: Backend,
   httpMethod: string,
-  collectionId: string,
+  collectionId: string | null,
   endpoint: string,
   parameters: APIParameters,
   headers: IncomingHttpHeaders
@@ -1083,19 +1083,19 @@ const getCollections = async function (
         title: 'Root Catalog'
       },
     ],
+    ...(process.env['ENABLE_CONTEXT_EXTENSION'] ? {
+      context: {
+        page: 1,
+        limit: COLLECTION_LIMIT,
+        matched: collections.length,
+        returned: collections.length
+      }
+    } : {})
   }
-
-  // note: adding this to the Collections response is not
+  // note: adding 'context' this to the Collections response is not
   // part of the Context Extension, and was just a proprietary
   // behavior of this implemenation
-  if (process.env['ENABLE_CONTEXT_EXTENSION']) {
-    resp['context'] = {
-      page: 1,
-      limit: COLLECTION_LIMIT,
-      matched: collections && collections.length,
-      returned: collections && collections.length
-    }
-  }
+
   return resp
 }
 

src/lib/asset-buckets.ts

@@ -31,7 +31,7 @@ export class AssetBuckets {
    * @param {string} bucketOption - Bucket option (NONE, ALL, ALL_BUCKETS_IN_ACCOUNT, LIST)
    * @param {string[]|null} bucketNames - Array of bucket names (required for LIST option)
    */
-  constructor(bucketOption, bucketNames) {
+  constructor(bucketOption: string, bucketNames: string[] | null) {
     this.bucketOption = bucketOption
     this.bucketNames = bucketNames
     this.bucketCache = {}
@@ -42,7 +42,7 @@ export class AssetBuckets {
    * @param {string[]|null} bucketNames - Array of bucket names (required for LIST option)
    * @returns {Promise<AssetBuckets>} Initialized AssetBuckets instance
    */
-  static async create(bucketOption, bucketNames) {
+  static async create(bucketOption: string, bucketNames: string[] | null) {
     const instance = new AssetBuckets(bucketOption, bucketNames)
     await instance._initBuckets()
     return instance

src/lib/asset-proxy.ts

@@ -43,7 +43,11 @@ export class AssetProxy {
    * @param {number} urlExpiry - Pre-signed URL expiry time in seconds
    * @param {string} bucketOption - Bucket option (NONE, ALL, ALL_BUCKETS_IN_ACCOUNT, LIST)
    */
-  constructor(buckets, urlExpiry, bucketOption) {
+  constructor(
+    buckets: AssetBuckets,
+    urlExpiry: number,
+    bucketOption: string
+  ) {
     this.buckets = buckets
     this.urlExpiry = urlExpiry
     this.isEnabled = bucketOption !== BucketOptionEnum.NONE
@@ -81,7 +85,7 @@ export class AssetProxy {
     collectionId: string | null,
     itemId: string | null
   ): {assets: Assets, wasProxied: boolean} {
-    const proxiedAssets = {}
+    const proxiedAssets: Assets = {}
     let wasProxied = false
 
     for (const [assetKey, asset] of Object.entries(assets)) {

src/lib/database.ts

@@ -9,14 +9,15 @@ import { bboxToPolygon } from './geo-utils.js'
 import {
   Cql2Filter,
   Cql2Value,
-  DateQuery,
   DateTimeRange,
   DbQueryParameters,
   FieldsFilter,
   ItemProperties,
   OpenSearchBody,
   OpenSearchFilterQuery,
+  OpValue,
   PartialItemUpdate,
+  QueryOperators,
   QueryParameters,
   RangeQuery,
   SearchParameters,
@@ -34,7 +35,7 @@ variable which is the URL to the search database host line
 
 const COLLECTIONS_INDEX = process.env['COLLECTIONS_INDEX'] || 'collections'
 const DEFAULT_INDICES = ['*', '-.*', '-collections']
-const OP = {
+export const OP = {
   AND: 'and',
   OR: 'or',
   NOT: 'not',
@@ -49,7 +50,8 @@ const OP = {
   BETWEEN: 'between',
   LIKE: 'like',
   S_INTERSECTS: 's_intersects',
-}
+} as const
+
 const RANGE_TRANSLATION = {
   '<': 'lt',
   '<=': 'lte',
@@ -80,7 +82,7 @@ const MAX_COLLECTIONS_IN_QUERY_PATH = 10
 let collectionToIndexMapping: Record<string, string> | null = null
 let unrestrictedIndices: string[] | null = null
 
-export const isIndexNotFoundError = (e) => (
+export const isIndexNotFoundError = (e: unknown) => (
   e instanceof Error
     && e.name === 'ResponseError'
     && e.message.includes('index_not_found_exception'))
@@ -129,7 +131,7 @@ function buildRangeQuery(
  * assumes a valid RFC3339 datetime or interval
  * validation was previously done by api.extractDatetime
  */
-export function buildDatetimeQuery(parameters: QueryParameters): DateQuery {
+export function buildDatetimeQuery(parameters: QueryParameters): OpenSearchFilterQuery | undefined {
   let dateQuery
   const { datetime } = parameters
   if (datetime) {
@@ -140,7 +142,7 @@ export function buildDatetimeQuery(parameters: QueryParameters): DateQuery {
       if (end && end !== '..') datetimeRange.lte = end
       dateQuery = {
         range: {
-          'properties.datetime': datetimeRange
+          'properties.datetime': datetimeRange as Record<string, unknown>
         }
       }
     } else {
@@ -244,7 +246,7 @@ function sIntersects(
 /**
  * build the opensearch property queries
  */
-function buildQueryExtQuery(query: QueryParameters): OpenSearchFilterQuery {
+function buildQueryExtQuery(query: Record<string, QueryOperators>): OpenSearchFilterQuery {
   const eq = 'eq'
   const inop = 'in'
   const startsWith = 'startsWith'
@@ -260,6 +262,7 @@ function buildQueryExtQuery(query: QueryParameters): OpenSearchFilterQuery {
     property: string
   ) => {
     const operatorsObject = query[property]
+    if (!operatorsObject) return accumulator
     const operators = Object.keys(operatorsObject)
 
     // eq
@@ -275,7 +278,7 @@ function buildQueryExtQuery(query: QueryParameters): OpenSearchFilterQuery {
     if (operators.includes(inop)) {
       accumulator.push({
         terms: {
-          [`properties.${property}`]: operatorsObject.in
+          [`properties.${property}`]: operatorsObject.in!
         }
       })
     }
@@ -329,6 +332,7 @@ function buildQueryExtQuery(query: QueryParameters): OpenSearchFilterQuery {
     property: string
   ) => {
     const operatorsObject = query[property]
+    if (!operatorsObject) return accumulator
     const operators = Object.keys(operatorsObject)
 
     // neq
@@ -435,7 +439,7 @@ function buildLeafFilter(filter: Cql2Filter): OpenSearchFilterQuery {
 
 // Routes to recursive or leaf handler based on operator
 function buildFilterExtQuery(filter: Cql2Filter): OpenSearchFilterQuery {
-  const RECURSIVE_OPS = [OP.AND, OP.OR, OP.NOT]
+  const RECURSIVE_OPS: OpValue[] = [OP.AND, OP.OR, OP.NOT]
   if (RECURSIVE_OPS.includes(filter.op)) {
     return buildRecursiveFilter(filter)
   }
@@ -629,7 +633,7 @@ function buildSearchAfter(parameters: QueryParameters): string[] | undefined {
  * a. null
  * b. an empty array
  */
-function fieldsParamIsEmpty(fieldsSpec: object, paramName: string): boolean {
+function fieldsParamIsEmpty(fieldsSpec: Record<string, unknown>, paramName: string): boolean {
   return fieldsSpec.hasOwnProperty(paramName)
     && (fieldsSpec[paramName] === null
     || (Array.isArray(fieldsSpec[paramName]) && !fieldsSpec[paramName].length))
@@ -812,6 +816,7 @@ async function getCollections(
       size: limit,
       from: (page - 1) * limit
     })
+    // @ts-ignore -- OpenSearch response body is of unknown shape
     return response.body['hits'].hits.map((r) => (r._source))
   } catch (e) {
     logger.error('Failure getting collections, maybe none exist?', e)
@@ -897,7 +902,7 @@ export async function constructSearchParams(
     if (!unrestrictedIndices) {
       await populateUnrestrictedIndices()
     }
-    indices = unrestrictedIndices
+    indices = unrestrictedIndices!
   }
   // hash indices
   indices = indices.map((index) => {
@@ -948,6 +953,7 @@ async function search(
   })
 
   const hits = dbResponse.body['hits'].hits
+  // @ts-ignore -- OpenSearch response body is of unknown shape
   const results = hits.map((r) => (r._source))
   const lastItem = hits.at(-1)
   let lastItemSort = null
@@ -1039,9 +1045,9 @@ async function aggregate(
   // include all aggregations specified
   // this will ignore aggregations with the wrong names
   searchParams.body.aggs = Object.keys(ALL_AGGREGATIONS).reduce((o, k) => {
-    if (aggregations.includes(k)) o[k] = ALL_AGGREGATIONS[k]
+    if (aggregations.includes(k)) o[k] = ALL_AGGREGATIONS[k as keyof typeof ALL_AGGREGATIONS]
     return o
-  }, {})
+  }, {} as Record<string, unknown>)
 
   // deprecated centroid