feat(datasource): implement connection pooling and sequelize specific options support (#27)

* feat(datasource): add connection pooling support

adds ability to prase connection pooling options for postgres, mysql and oracle

GH-26

* feat(datasource): add capability to pass options directly to sequelize

added ability to connect with url

added new property called `sequelizeOptions` in datasource config to
allow direct option forwarding to sequelize instance

GH-26

Author: shubhamp-sf

README.md

@@ -52,6 +52,27 @@ export class PgDataSource
 }
 ```
 
+`SequelizeDataSource` accepts commonly used config in the same way as loopback did. So in most cases you won't need to change your existing configuration. But if you want to use sequelize specific options pass them in `sequelizeOptions` like below:
+
+```ts
+let config = {
+  name: 'db',
+  connector: 'postgresql',
+  sequelizeOptions: {
+    username: 'postgres',
+    password: 'secret',
+    dialectOptions: {
+      ssl: {
+        rejectUnauthorized: false,
+        ca: fs.readFileSync('/path/to/root.crt').toString(),
+      },
+    },
+  },
+};
+```
+
+> Note: Options provided in `sequelizeOptions` will take priority over others, For eg. if you have password specified in both `config.password` and `config.password.sequelizeOptions` the latter one will be used.
+
 ### Step 2: Configure Repository
 
 Change the parent class from `DefaultCrudRepository` to `SequelizeCrudRepository` like below.
@@ -219,7 +240,6 @@ There are three built-in debug strings available in this extension to aid in deb
 Please note, the current implementation does not support the following:
 
 1. Loopback Migrations (via default `migrate.ts`). Though you're good if using external packages like [`db-migrate`](https://www.npmjs.com/package/db-migrate).
-2. Connection Pooling is not implemented yet.
 
 Community contribution is welcome.
 

src/__tests__/fixtures/datasources/config.ts

@@ -8,7 +8,7 @@ type AvailableConfig = Record<
 >;
 
 export const datasourceTestConfig: Record<
-  'primary' | 'secondary',
+  'primary' | 'secondary' | 'url' | 'wrongPassword',
   AvailableConfig
 > = {
   primary: {
@@ -47,4 +47,26 @@ export const datasourceTestConfig: Record<
       file: ':memory:',
     },
   },
+  url: {
+    postgresql: {
+      name: 'using-url',
+      connector: 'postgresql',
+      url: 'postgres://postgres:super-secret@localhost:5002/postgres',
+    },
+    sqlite3: {
+      name: 'using-url',
+      url: 'sqlite::memory:',
+    },
+  },
+  wrongPassword: {
+    postgresql: {
+      name: 'wrongPassword',
+      connector: 'postgresql',
+      url: 'postgres://postgres:super-secret-wrong@localhost:5002/postgres',
+    },
+    sqlite3: {
+      name: 'wrongPassword',
+      url: 'sqlite::memory:',
+    },
+  },
 };

src/__tests__/unit/sequelize.datasource.unit.ts

@@ -1,6 +1,8 @@
 import {expect} from '@loopback/testlab';
 import {SequelizeDataSource} from '../../sequelize';
 import {SupportedLoopbackConnectors} from '../../sequelize/connector-mapping';
+import {datasourceTestConfig} from '../fixtures/datasources/config';
+import {config as primaryDataSourceConfig} from '../fixtures/datasources/primary.datasource';
 
 describe('Sequelize DataSource', () => {
   it('throws error when nosql connectors are supplied', () => {
@@ -16,4 +18,98 @@ describe('Sequelize DataSource', () => {
       expect(result).which.eql('Specified connector memory is not supported.');
     }
   });
+
+  it('accepts url strings for connection', async () => {
+    const dataSource = new SequelizeDataSource(
+      datasourceTestConfig.url[
+        primaryDataSourceConfig.connector === 'postgresql'
+          ? 'postgresql'
+          : 'sqlite3'
+      ],
+    );
+    expect(await dataSource.init()).to.not.throwError();
+    await dataSource.stop();
+  });
+
+  it('throws error if url strings has wrong password', async function () {
+    if (primaryDataSourceConfig.connector !== 'postgresql') {
+      // eslint-disable-next-line @typescript-eslint/no-invalid-this
+      this.skip();
+    }
+    const dataSource = new SequelizeDataSource(
+      datasourceTestConfig.wrongPassword.postgresql,
+    );
+    try {
+      await dataSource.init();
+    } catch (err) {
+      expect(err.message).to.be.eql(
+        'password authentication failed for user "postgres"',
+      );
+    }
+  });
+
+  it('should be able override sequelize options', async function () {
+    if (primaryDataSourceConfig.connector !== 'postgresql') {
+      // eslint-disable-next-line @typescript-eslint/no-invalid-this
+      this.skip();
+    }
+    const dataSource = new SequelizeDataSource({
+      ...datasourceTestConfig.primary.postgresql,
+      user: 'wrong-username', // expected to be overridden
+      sequelizeOptions: {
+        username: datasourceTestConfig.primary.postgresql.user,
+      },
+    });
+    expect(await dataSource.init()).to.not.throwError();
+  });
+
+  it('parses pool options for postgresql', async () => {
+    const dataSource = new SequelizeDataSource({
+      name: 'db',
+      connector: 'postgresql',
+      min: 10,
+      max: 20,
+      idleTimeoutMillis: 18000,
+    });
+
+    const poolOptions = dataSource.getPoolOptions();
+
+    expect(poolOptions).to.have.property('min', 10);
+    expect(poolOptions).to.have.property('max', 20);
+    expect(poolOptions).to.have.property('idle', 18000);
+    expect(poolOptions).to.not.have.property('acquire');
+  });
+
+  it('parses pool options for mysql', async () => {
+    const dataSource = new SequelizeDataSource({
+      name: 'db',
+      connector: 'mysql',
+      connectionLimit: 20,
+      acquireTimeout: 10000,
+    });
+
+    const poolOptions = dataSource.getPoolOptions();
+
+    expect(poolOptions).to.have.property('max', 20);
+    expect(poolOptions).to.have.property('acquire', 10000);
+    expect(poolOptions).to.not.have.property('min');
+    expect(poolOptions).to.not.have.property('idle');
+  });
+
+  it('parses pool options for oracle', async () => {
+    const dataSource = new SequelizeDataSource({
+      name: 'db',
+      connector: 'oracle',
+      minConn: 10,
+      maxConn: 20,
+      timeout: 20000,
+    });
+
+    const poolOptions = dataSource.getPoolOptions();
+
+    expect(poolOptions).to.have.property('min', 10);
+    expect(poolOptions).to.have.property('max', 20);
+    expect(poolOptions).to.have.property('idle', 20000);
+    expect(poolOptions).to.not.have.property('acquire');
+  });
 });

src/sequelize/connector-mapping.ts

@@ -1,4 +1,4 @@
-import {Dialect as AllSequelizeDialects} from 'sequelize';
+import {Dialect as AllSequelizeDialects, PoolOptions} from 'sequelize';
 
 export type SupportedLoopbackConnectors =
   | 'mysql'
@@ -19,3 +19,55 @@ export const SupportedConnectorMapping: {
   sqlite3: 'sqlite',
   db2: 'db2',
 };
+
+/**
+ * Loopback uses different keys for pool options depending on the connector.
+ */
+export const poolConfigKeys = [
+  // mysql
+  'connectionLimit',
+  'acquireTimeout',
+  // postgresql
+  'min',
+  'max',
+  'idleTimeoutMillis',
+  // oracle
+  'minConn',
+  'maxConn',
+  'timeout',
+] as const;
+export type LoopbackPoolConfigKey = (typeof poolConfigKeys)[number];
+
+export type PoolingEnabledConnector = Exclude<
+  SupportedLoopbackConnectors,
+  'db2' | 'sqlite3'
+>;
+
+export const poolingEnabledConnectors: PoolingEnabledConnector[] = [
+  'mysql',
+  'oracle',
+  'postgresql',
+];
+
+type IConnectionPoolOptions = {
+  [connectorName in PoolingEnabledConnector]?: {
+    [sequelizePoolOption in keyof PoolOptions]: LoopbackPoolConfigKey;
+  };
+};
+
+export const ConnectionPoolOptions: IConnectionPoolOptions = {
+  mysql: {
+    max: 'connectionLimit',
+    acquire: 'acquireTimeout',
+  },
+  postgresql: {
+    min: 'min',
+    max: 'max',
+    idle: 'idleTimeoutMillis',
+  },
+  oracle: {
+    min: 'minConn',
+    max: 'maxConn',
+    idle: 'timeout',
+  },
+};

src/sequelize/sequelize.datasource.base.ts

@@ -3,11 +3,17 @@ import {AnyObject} from '@loopback/repository';
 import debugFactory from 'debug';
 import {
   Options as SequelizeOptions,
+  PoolOptions,
   Sequelize,
   Transaction,
   TransactionOptions,
 } from 'sequelize';
 import {
+  ConnectionPoolOptions,
+  LoopbackPoolConfigKey,
+  poolConfigKeys,
+  PoolingEnabledConnector,
+  poolingEnabledConnectors,
   SupportedConnectorMapping as supportedConnectorMapping,
   SupportedLoopbackConnectors,
 } from './connector-mapping';
@@ -35,7 +41,7 @@ export class SequelizeDataSource implements LifeCycleObserver {
   }
 
   sequelize?: Sequelize;
-  sequelizeConfig: SequelizeDataSourceConfig;
+  sequelizeConfig: SequelizeOptions;
   async init(): Promise<void> {
     const {config} = this;
     const {
@@ -60,9 +66,15 @@ export class SequelizeDataSource implements LifeCycleObserver {
       username: user ?? username,
       password,
       logging: queryLogging,
+      pool: this.getPoolOptions(),
+      ...config.sequelizeOptions,
     };
 
-    this.sequelize = new Sequelize(this.sequelizeConfig);
+    if (config.url) {
+      this.sequelize = new Sequelize(config.url, this.sequelizeConfig);
+    } else {
+      this.sequelize = new Sequelize(this.sequelizeConfig);
+    }
 
     await this.sequelize.authenticate();
     debug('Connection has been established successfully.');
@@ -114,11 +126,68 @@ export class SequelizeDataSource implements LifeCycleObserver {
 
     return this.sequelize!.transaction(options);
   }
+
+  getPoolOptions(): PoolOptions | undefined {
+    const config: SequelizeDataSourceConfig = this.config;
+    const specifiedPoolOptions = Object.keys(config).some(key =>
+      poolConfigKeys.includes(key as LoopbackPoolConfigKey),
+    );
+    const supportsPooling =
+      config.connector &&
+      (poolingEnabledConnectors as string[]).includes(config.connector);
+
+    if (!(supportsPooling && specifiedPoolOptions)) {
+      return;
+    }
+    const optionMapping =
+      ConnectionPoolOptions[config.connector as PoolingEnabledConnector];
+
+    if (!optionMapping) {
+      return;
+    }
+
+    const {min, max, acquire, idle} = optionMapping;
+    const options: PoolOptions = {};
+    if (max && config[max]) {
+      options.max = config[max];
+    }
+    if (min && config[min]) {
+      options.min = config[min];
+    }
+    if (acquire && config[acquire]) {
+      options.acquire = config[acquire];
+    }
+    if (idle && config[idle]) {
+      options.idle = config[idle];
+    }
+    return options;
+  }
 }
 
-export type SequelizeDataSourceConfig = SequelizeOptions & {
+export type SequelizeDataSourceConfig = {
   name?: string;
   user?: string;
   connector?: SupportedLoopbackConnectors;
   url?: string;
+  /**
+   * Additional sequelize options that are passed directly to
+   * Sequelize when initializing the connection.
+   * Any options provided in this way will take priority over
+   * other configurations that may come from parsing the loopback style configurations.
+   *
+   * eg.
+   * ```ts
+   * let config = {
+   *   name: 'db',
+   *   connector: 'postgresql',
+   *   sequelizeOptions: {
+   *      dialectOptions: {
+   *        rejectUnauthorized: false,
+   *        ca: fs.readFileSync('/path/to/root.crt').toString(),
+   *      }
+   *   }
+   * };
+   * ```
+   */
+  sequelizeOptions?: SequelizeOptions;
 } & AnyObject;