Audits

app.js

@@ -1,10 +1,10 @@
-var express = require('express');
-var path = require('path');
-var morganLogger = require('morgan');
-var cookieParser = require('cookie-parser');
-var bodyParser = require('body-parser');
-var cors = require('cors');
-var dateUtils = require('./js/dateUtils');
+const express = require('express');
+const path = require('path');
+const morganLogger = require('morgan');
+const cookieParser = require('cookie-parser');
+const bodyParser = require('body-parser');
+const cors = require('cors');
+const dateUtils = require('./js/dateUtils');
 
 require('dotenv').config({
   silent: true
@@ -52,6 +52,9 @@ const googleMapRoutes = new GoogleMapRoutes();
 const AuditRoutes = require('./routes/auditRoutes');
 const auditRoutes = new AuditRoutes();
 
+const AuditRoutesV2 = require('./routes/auditRoutesV2');
+const auditRoutesV2 = new AuditRoutesV2();
+
 const ContractRoutes = require('./routes/contractRoutes');
 const contractRoutes = new ContractRoutes();
 
@@ -64,7 +67,7 @@ const ptoCashOutRoutes = new PTOCashOutRoutes();
 const TagRoutes = require('./routes/tagRoutes');
 const tagRoutes = new TagRoutes();
 
-var app = express();
+const app = express();
 
 // view engine setup
 app.set('views', path.join(__dirname, 'views'));
@@ -102,22 +105,20 @@ app.use('/basecamp', basecampRoutes.router);
 app.use('/emsi', emsiRoutes.router);
 app.use('/googleMaps', googleMapRoutes.router);
 app.use('/audits', auditRoutes.router);
+app.use('/auditsV2', auditRoutesV2.router);
 app.use('/contracts', contractRoutes.router);
 app.use('/highFives', highFiveRoutes.router);
 app.use('/ptoCashOuts', ptoCashOutRoutes.router);
 app.use('/tags', tagRoutes.router);
 // catch 404 and forward to error handler
-app.use(function (req, res, next) {
-  var err = new Error(' No Route Found');
+app.use((_req, _res, next) => {
+  const err = new Error(' No Route Found');
   err.status = 404;
   next(err);
 });
 
 // error handler
-//eslint is disabled because we need 4th param but never use it
-app.use(function (err, req, res, next) {
-
-
+app.use((err, req, res, _next) => {
   // set locals, only providing error in development
   res.locals.message = err.message;
   res.locals.error = req.app.get('env') === 'development' ? err : {};

app.yaml

@@ -1052,7 +1052,7 @@ Resources:
         - Arn
 
   TimesheetSubmissionReminderFunction:
-    Type: "AWS::Serverless::Function" # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction
+    Type: "AWS::Serverless::Function"
     Properties:
       FunctionName: !Join
         - ""
@@ -1141,24 +1141,24 @@ Resources:
                 - logs:PutLogEvents
               Effect: Allow
               Resource: "*"
-        # - Statement:
-        #     - Effect: Allow
-        #       Action:
-        #         - rds-data:*
-        #       Resource:
-        #         Fn::ImportValue: !Join [
-        #             "-",
-        #             [!Ref Stage, expense-app-cluster-arn],
-        #           ] # from database.yaml exports
-        # - Statement:
-        #     - Effect: Allow
-        #       Action:
-        #         - secretsmanager:GetSecretValue
-        #       Resource:
-        #         Fn::ImportValue: !Join [
-        #             "-",
-        #             [!Ref Stage, expense-app-cluster-secret-arn],
-        #           ] # from database.yaml exports
+        - Statement:
+            - Effect: Allow
+              Action:
+                - rds-data:*
+              Resource:
+                Fn::ImportValue: !Join [
+                    "-",
+                    [!Ref Stage, expense-app-cluster-arn],
+                  ] # from database.yaml exports
+        - Statement:
+            - Effect: Allow
+              Action:
+                - secretsmanager:GetSecretValue
+              Resource:
+                Fn::ImportValue: !Join [
+                    "-",
+                    [!Ref Stage, expense-app-cluster-secret-arn],
+                  ] # from database.yaml exports
 
   TimesheetSubmissionReminderFunctionLogGroup:
     Type: AWS::Logs::LogGroup

aurora/.gitignore

@@ -0,0 +1,3 @@
+dist
+tsconfig.tsbuildinfo
+*.tgz

aurora/README.md

@@ -0,0 +1,141 @@
+# The Aurora Database Module
+
+This module acts as a wrapper for all interactions with the app's relational database. It entirely handles building postgres queries, setting up the database connection, communicating with the database, and providing types/models to write safer code.
+
+## Todo List
+
+- Test queries with a local database in a docker container, the testcontainers library is designed for this
+- Connect directly to the database instead of using data api. The direct connection would return clearer error messages and is generally more portable
+  - Benefits:
+    - More portable
+    - Standard way of connecting to the database, integrates better with existing libraries/frameworks
+    - Enables the usage of kysely's pg adapter, which automatically (de)serializes more types between postgres and js
+  - Rough outline of how to implement this:
+    - Put all lambda functions in the vpc
+    - Create security group for lambda functions to access the open internet
+    - Create security group for lambda functions to access the aurora security group (which should not allow ingress from the open internet)
+    - Figure out how to run the app locally. This will likely require aws client vpn to connect to the database locally
+    - Use kysely's pg driver (already included in kysely) instead of kysely-data-api (which can be uninstalled)
+    - Revamp the querying functionality (like `CrudAudit.asInsertable`) to utilize the new types. Write tests to aid in developing this (seriously it made it so much easier to write this whole module)
+
+## Usage
+
+### Installation
+
+This module is a local npm package. It's in package.json for the entire project as well as in the dependency lambda layer. The script `npm run build:aurora` tests and builds this module. It's already in `npm run reinstall`. Calling `npm run build:aurora && npm i` is faster, but reinstall also installs in lambda layers.
+
+In any app that uses this, you must first initialize the module, which is an asynchronous operation. The `STAGE` environment variable must also exist before calling initialize.
+
+```js
+const { config: dotenv } = require('dotenv');
+const { initialize } = require('expense-app-db');
+const { CrudAuditQueries } = require('expense-app-db/queries');
+
+async function main() {
+  await initialize();
+  await CrudAuditQueries.record(...); // cannot be called before awaiting on `initialize`
+}
+
+dotenv(); // load STAGE from .env file
+main();
+```
+
+This initialization process should happen exactly once, sometime very early on in any runtime this module is used in. This includes the main app, standalone scripts, lambda functions, etc.
+
+### Querying
+
+There are prebuild queries in various namespaces in the queries submodule.
+
+```js
+const { CrudAuditQueries, NotifAuditQueries } = require('expense-app-db/queries');
+
+const results = await CrudAuditQueries.select(...);
+```
+
+While `db` is exposed via `getDb()` in the main module, its usage should be avoided. Instead, make a query within the module and call the function.
+
+i.e. avoid this:
+
+```js
+const { getDb } = require('expense-app-db');
+
+const db = getDb();
+const result = await db.selectFrom(...).execute();
+```
+
+### Types
+
+To access the types to type-annotate functions and variables, you can import the types using a JSDoc import:
+
+```js
+/** @import { CrudAuditQueryFilters } from 'expense-app-db/types' */
+```
+
+The models folder contains the real class definitions and some 'enums'
+
+```js
+// I recommend aliasing the enums to avoid confusing them with the types
+const { DynamoTable: DynamoTableEnum, CrudAudit: CrudAuditEnum } = require('expense-app-db/models');
+const audit = new CrudAudit(...);
+```
+
+When writing code, vscode will give you suggestions for types that use these enums. It will suggest the literal string values. Instead of using those, you should still use the enum object and access the corresponding key.
+
+## Examples
+
+- `routes/auditRoutesV2`
+- this module
+- [Kysely's recipes](https://kysely.dev/docs/category/recipes)
+- [Kysely's examples](https://kysely.dev/docs/category/examples)
+- [kysely-data-api tests](https://github.com/sst/kysely-data-api/blob/master/test/data-api-query-compiler.test.ts)
+
+## Contributing
+
+> After making changes to this package, run `npm run reinstall` (or `npm run build:aurora && npm i`, if you're not working with the lambda functions) in the project root (not this directory).
+> When adding files, make sure to expose them as exports in the `index.ts` file found _in the same folder_. Some of them have different structures intentionally, make sure to follow that consistently.
+
+If you want to add (or modify) a table or type, a useful place to do that is `scripts/setup.ts` (comment out the other setup function calls and write a new function to setup only the new additions). Make sure to commit the changes so that the schema is clearly defined and the script can be used later if needed.
+
+### Creating tables in the database
+
+Add the following:
+
+1. A function to create the tables/types/indexes in the database, in `scripts.setup.ts`
+2. A new property in the `Database` type in `types.ts`
+3. Their own type defined similarly to the existing tables
+4. A class in the `models/` folder, and enums if applicable
+5. Queries in the `queries/` folder (refer to the [examples](#examples)). You don't have to anticipate potentially useful queries, rather just implement them as they are needed.
+
+### Creating enums
+
+- Make sure the values in the enum exactly match what they are in the postgres enum type
+- Define a corresponding type in `types.ts` based on the other examples
+
+### Installing into lambda functions
+
+Lambda functions don't install dependencies of symlinked packages. To run a lambda function that depends on this module, you need to install without linking. Reference `timesheet-submission-reminder/invoke-local.sh` to see how this is done. You can copy and modify this script for any new lambda function that uses this module (and add it to .npmignore).
+
+### Tests
+
+In general, tests are most useful for the queries. Types can't really be tested, and models don't needed to be tested unless they're complex enough.
+
+## Dependencies and Documentation
+
+- [Aurora Serverless](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-serverless-v2.html) - the compute and storage resources for the database
+- [RDS Data API](https://docs.aws.amazon.com/rdsdataservice/latest/APIReference/Welcome.html) - interact with aurora through an http endpoint
+- [PostgreSQL](https://www.postgresql.org/docs/current/index.html) - the database engine
+- [TypeScript](https://www.typescriptlang.org/docs/handbook/typescript-in-5-minutes.html)
+- [Kysely](https://kysely.dev/docs/intro) - builds queries and interacts with the database
+- [Kysely Data API](https://www.npmjs.com/package/kysely-data-api) - provides support for using kysely via the rds data api
+  - Note: this depends on Kysely version `0.27.x`. The latest version of Kysely cannot be used unless this is updated or a fork is made
+- [Jest](https://jestjs.io/docs/getting-started) - testing framework
+
+## Other Notes
+
+### Why TypeScript?
+
+To integrate better with kysely's type system. While it's possible to use kysely with js and jsdoc, it gets very messy and convoluted. Using typescript simplified the code when compared to writing in in js, and is possible since this is an entirely separate module.
+
+### Why a Separate Module?
+
+The file in this module are very interdependent on each other. While this is fine for the backend as a whole, it was very messy when trying to install it into the lambda layers.

aurora/jest.config.js

@@ -0,0 +1,12 @@
+const { createDefaultPreset } = require('ts-jest');
+
+const tsJestTransformCfg = createDefaultPreset().transform;
+
+/** @type {import('jest').Config} **/
+module.exports = {
+  testEnvironment: 'node',
+  transform: {
+    ...tsJestTransformCfg
+  },
+  globalSetup: './test/globalSetup.ts'
+};