Merge pull request #8498 from ever-co/develop

feat: remove SQlite3 native package, switch all to Better SQLite!
ever-co · Oct 27, 2024 · 3d6fcb6 · 3d6fcb6
2 parents cafd8c8 + b3c82ca
commit 3d6fcb6
Show file tree

Hide file tree

Showing 26 changed files with 85 additions and 186 deletions.
diff --git a/apps/desktop-timer/package.json b/apps/desktop-timer/package.json
@@ -81,7 +81,6 @@
 		"rxjs": "^7.4.0",
 		"screenshot-desktop": "^1.15.0",
 		"sound-play": "1.1.0",
-		"sqlite3": "^5.1.7",
 		"squirrelly": "^8.0.8",
 		"twing": "^5.0.2",
 		"locutus": "^2.0.30",

diff --git a/apps/desktop-timer/src/index.ts b/apps/desktop-timer/src/index.ts
@@ -14,7 +14,6 @@ import * as Url from 'url';
 import { environment } from './environments/environment';
 
 require('module').globalPaths.push(path.join(__dirname, 'node_modules'));
-require('sqlite3');
 
 process.env = Object.assign(process.env, environment);
 
@@ -378,7 +377,7 @@ app.on('ready', async () => {
 	if (!settings) {
 		launchAtStartup(true, false);
 	}
-	if (['sqlite', 'better-sqlite'].includes(provider.dialect)) {
+	if (['sqlite', 'better-sqlite', 'better-sqlite3'].includes(provider.dialect)) {
 		try {
 			const res = await knex.raw(`pragma journal_mode = WAL;`);
 			console.log(res);

diff --git a/apps/desktop-timer/src/package.json b/apps/desktop-timer/src/package.json
@@ -41,7 +41,6 @@
 		"asarUnpack": [
 			"node_modules/screenshot-desktop/lib/win32",
 			"node_modules/@sentry/electron",
-			"node_modules/sqlite3/lib",
 			"node_modules/better-sqlite3",
 			"node_modules/@sentry/profiling-node/lib"
 		],
@@ -162,7 +161,6 @@
 		"pg-query-stream": "^4.7.1",
 		"screenshot-desktop": "^1.15.0",
 		"sound-play": "1.1.0",
-		"sqlite3": "^5.1.7",
 		"squirrelly": "^8.0.8",
 		"tslib": "^2.6.2",
 		"twing": "^5.0.2",

diff --git a/apps/desktop/README.md b/apps/desktop/README.md
@@ -12,12 +12,6 @@ yarn install
 
 **build executable for mac**
 
-rebuild sqlite3 for mac
-
-```bash
-yarn build:sqlite:mac
-```
-
 build desktop
 
 ```bash
@@ -31,11 +25,6 @@ build:desktop:mac:quick
 ```
 
 **build execute app for windows**
-rebuild sqlite3 for windows
-
-```bash
-yarn build:sqlite:windows
-```
 
 build desktop
 

diff --git a/apps/desktop/package.json b/apps/desktop/package.json
@@ -85,7 +85,6 @@
 		"rxjs": "^7.4.0",
 		"screenshot-desktop": "^1.15.0",
 		"sound-play": "1.1.0",
-		"sqlite3": "^5.1.7",
 		"squirrelly": "^8.0.8",
 		"twing": "^5.0.2",
 		"typeorm": "^0.3.20",

diff --git a/apps/desktop/src/index.ts b/apps/desktop/src/index.ts
@@ -14,8 +14,6 @@ import { environment } from './environments/environment';
 
 require('module').globalPaths.push(path.join(__dirname, 'node_modules'));
 
-require('sqlite3');
-
 Object.assign(process.env, environment);
 
 app.setName(process.env.NAME);
@@ -263,7 +261,7 @@ async function startServer(value, restart = false) {
 	if (value.db === 'sqlite') {
 		process.env.DB_PATH = sqlite3filename;
 		process.env.DB_TYPE = 'sqlite';
-	} else if (value.db === 'better-sqlite') {
+	} else if (value.db === 'better-sqlite' || value.db === 'better-sqlite3') {
 		process.env.DB_PATH = sqlite3filename;
 		process.env.DB_TYPE = 'better-sqlite3';
 	} else {
@@ -432,7 +430,7 @@ app.on('ready', async () => {
 		throw new AppError('MAINWININIT', error);
 	}
 
-	if (['sqlite', 'better-sqlite'].includes(provider.dialect)) {
+	if (['sqlite', 'better-sqlite', 'better-sqlite3'].includes(provider.dialect)) {
 		try {
 			const res = await knex.raw(`pragma journal_mode = WAL;`);
 			console.log(res);

diff --git a/apps/desktop/src/package.json b/apps/desktop/src/package.json
@@ -70,7 +70,6 @@
 		"asarUnpack": [
 			"node_modules/screenshot-desktop/lib/win32",
 			"node_modules/@sentry/electron",
-			"node_modules/sqlite3/lib",
 			"node_modules/better-sqlite3",
 			"node_modules/@sentry/profiling-node/lib"
 		],
@@ -199,7 +198,6 @@
 		"pg": "^8.13.1",
 		"screenshot-desktop": "^1.15.0",
 		"sound-play": "1.1.0",
-		"sqlite3": "^5.1.7",
 		"squirrelly": "^8.0.8",
 		"tslib": "^2.6.2",
 		"twing": "^5.0.2",

diff --git a/apps/server-api/package.json b/apps/server-api/package.json
@@ -81,7 +81,6 @@
 		"pg": "^8.13.1",
 		"pg-query-stream": "^4.7.1",
 		"rxjs": "^7.4.0",
-		"sqlite3": "^5.1.7",
 		"squirrelly": "^8.0.8",
 		"twing": "^5.0.2",
 		"typeorm": "^0.3.20",

diff --git a/apps/server-api/src/index.ts b/apps/server-api/src/index.ts
@@ -23,8 +23,6 @@ import { environment } from './environments/environment';
 
 require('module').globalPaths.push(path.join(__dirname, 'node_modules'));
 
-require('sqlite3');
-
 process.env = Object.assign(process.env, environment);
 
 app.setName(process.env.NAME);
@@ -520,7 +518,7 @@ ipcMain.on('check_database_connection', async (event, arg) => {
 			};
 		} else {
 			databaseOptions = {
-				client: 'sqlite',
+				client: 'better-sqlite3',
 				connection: {
 					filename: sqlite3filename
 				}

diff --git a/apps/server-api/src/package.json b/apps/server-api/src/package.json
@@ -57,7 +57,6 @@
 		"npmRebuild": true,
 		"asarUnpack": [
 			"node_modules/@sentry/electron",
-			"node_modules/sqlite3/lib",
 			"node_modules/better-sqlite3",
 			"node_modules/@sentry/profiling-node/lib"
 		],
@@ -181,7 +180,6 @@
 		"pg-query-stream": "^4.7.1",
 		"pg": "^8.13.1",
 		"sound-play": "1.1.0",
-		"sqlite3": "^5.1.7",
 		"squirrelly": "^8.0.8",
 		"tslib": "^2.6.2",
 		"twing": "^5.0.2",

diff --git a/apps/server/README.md b/apps/server/README.md
@@ -12,12 +12,6 @@ yarn install
 
 **build executable for mac**
 
-rebuild sqlite3 for mac
-
-```bash
-yarn build:sqlite:mac
-```
-
 build desktop
 
 ```bash
@@ -31,11 +25,6 @@ build:desktop:mac:quick
 ```
 
 **build execute app for windows**
-rebuild sqlite3 for windows
-
-```bash
-yarn build:sqlite:windows
-```
 
 build desktop
 

diff --git a/apps/server/package.json b/apps/server/package.json
@@ -81,7 +81,6 @@
 		"pg": "^8.13.1",
 		"pg-query-stream": "^4.7.1",
 		"rxjs": "^7.4.0",
-		"sqlite3": "^5.1.7",
 		"squirrelly": "^8.0.8",
 		"twing": "^5.0.2",
 		"typeorm": "^0.3.20",

diff --git a/apps/server/src/index.ts b/apps/server/src/index.ts
@@ -23,8 +23,6 @@ import { environment } from './environments/environment';
 
 require('module').globalPaths.push(path.join(__dirname, 'node_modules'));
 
-require('sqlite3');
-
 process.env = Object.assign(process.env, environment);
 
 app.setName(process.env.NAME);
@@ -534,7 +532,7 @@ ipcMain.on('check_database_connection', async (event, arg) => {
 			};
 		} else {
 			databaseOptions = {
-				client: 'sqlite',
+				client: 'better-sqlite3',
 				connection: {
 					filename: sqlite3filename
 				}

diff --git a/apps/server/src/package.json b/apps/server/src/package.json
@@ -70,7 +70,6 @@
 		"npmRebuild": true,
 		"asarUnpack": [
 			"node_modules/@sentry/electron",
-			"node_modules/sqlite3/lib",
 			"node_modules/better-sqlite3",
 			"node_modules/@sentry/profiling-node/lib"
 		],
@@ -194,7 +193,6 @@
 		"pg-query-stream": "^4.7.1",
 		"pg": "^8.13.1",
 		"sound-play": "1.1.0",
-		"sqlite3": "^5.1.7",
 		"squirrelly": "^8.0.8",
 		"tslib": "^2.6.2",
 		"twing": "^5.0.2",

diff --git a/package.json b/package.json
@@ -109,8 +109,6 @@
 		"build:desktop:mac:quick": "npm config set cache .cache && yarn electron-builder -c.electronVersion=30.0.1 build --mac --project dist/apps/desktop",
 		"build:desktop:local:quick": "yarn electron dist/apps/desktop --prod",
 		"build:desktop:windows:quick": "npm config set cache .cache && yarn electron-builder -c.electronVersion=30.0.1 build --windows --project dist/apps/desktop",
-		"build:sqlite:mac": "cd node_modules/sqlite3 && HOME=~/.electron-gyp node-pre-gyp rebuild --target=8.3.0 --arch=x64 --dist-url=https://electronjs.org/headers",
-		"build:sqlite:windows": "cd node_modules/sqlite3 && cross-env HOME=~/.electron-gyp node-pre-gyp rebuild --target=8.3.0 --arch=x64 --target_platform=win32 --dist-url=https://electronjs.org/headers",
 		"debug:desktop:app:mac": "lldb ./dist/apps/desktop-packages/mac/GauzyDesktop.app && run",
 		"start:desktop:ui": "yarn run postinstall.web && yarn ng serve desktop-ui",
 		"build:desktop:ui": "yarn run postinstall.web && yarn ng:prod build desktop-ui --prod --base-href ./",

diff --git a/packages/config/package.json b/packages/config/package.json
@@ -51,7 +51,6 @@
 		"@mikro-orm/mysql": "^6.2.3",
 		"@mikro-orm/nestjs": "^5.2.3",
 		"@mikro-orm/postgresql": "^6.2.3",
-		"@mikro-orm/sqlite": "^6.2.3",
 		"@nestjs/common": "^10.3.7",
 		"@nestjs/config": "^3.2.0",
 		"@nestjs/typeorm": "^10.0.2",

diff --git a/packages/config/src/database.ts b/packages/config/src/database.ts
@@ -3,8 +3,7 @@ import * as chalk from 'chalk';
 import { TypeOrmModuleOptions } from '@nestjs/typeorm';
 import { MikroOrmModuleOptions } from '@mikro-orm/nestjs';
 import { EntityCaseNamingStrategy } from '@mikro-orm/core';
-import { SoftDeleteHandler } from "mikro-orm-soft-delete";
-import { SqliteDriver, Options as MikroOrmSqliteOptions } from '@mikro-orm/sqlite';
+import { SoftDeleteHandler } from 'mikro-orm-soft-delete';
 import { BetterSqliteDriver, Options as MikroOrmBetterSqliteOptions } from '@mikro-orm/better-sqlite';
 import { PostgreSqlDriver, Options as MikroOrmPostgreSqlOptions } from '@mikro-orm/postgresql';
 import { Options as MikroOrmMySqlOptions, MySqlDriver } from '@mikro-orm/mysql';
@@ -63,7 +62,9 @@ const dbConnectionTimeout = process.env.DB_CONNECTION_TIMEOUT ? parseInt(process
 
 const idleTimeoutMillis = process.env.DB_IDLE_TIMEOUT ? parseInt(process.env.DB_IDLE_TIMEOUT) : 10000; // 10 seconds
 
-const dbSlowQueryLoggingTimeout = process.env.DB_SLOW_QUERY_LOGGING_TIMEOUT ? parseInt(process.env.DB_SLOW_QUERY_LOGGING_TIMEOUT) : 10000; // 10 seconds default
+const dbSlowQueryLoggingTimeout = process.env.DB_SLOW_QUERY_LOGGING_TIMEOUT
+	? parseInt(process.env.DB_SLOW_QUERY_LOGGING_TIMEOUT)
+	: 10000; // 10 seconds default
 
 const dbSslMode = process.env.DB_SSL_MODE === 'true';
 
@@ -276,52 +277,16 @@ switch (dbType) {
 		break;
 
 	case DatabaseTypeEnum.sqlite:
-		const dbPath = process.env.DB_PATH || path.join(process.cwd(), ...['apps', 'api', 'data'], 'gauzy.sqlite3');
-		console.log('Sqlite DB Path: ' + dbPath);
-
-		// MikroORM DB Config (SQLite3)
-		const mikroOrmSqliteConfig: MikroOrmSqliteOptions = {
-			driver: SqliteDriver,
-			dbName: dbPath,
-			persistOnCreate: true,
-			extensions: [SoftDeleteHandler],
-			namingStrategy: EntityCaseNamingStrategy,
-			debug: getLoggingMikroOptions(process.env.DB_LOGGING) // by default set to false only
-		};
-		mikroOrmConnectionConfig = mikroOrmSqliteConfig;
-
-		// TypeORM DB Config (SQLite3)
-		const typeOrmSqliteConfig: DataSourceOptions = {
-			type: dbType,
-			database: dbPath,
-			logging: 'all',
-			logger: 'file', //Removes console logging, instead logs all queries in a file ormlogs.log
-			synchronize: process.env.DB_SYNCHRONIZE === 'true' // We are using migrations, synchronize should be set to false.
-		};
-		typeOrmConnectionConfig = typeOrmSqliteConfig;
-
-		// Knex DB Config (SQLite3)
-		const knexSqliteConfig: KnexModuleOptions = {
-			config: {
-				client: 'sqlite3',
-				connection: {
-					filename: dbPath
-				},
-				useNullAsDefault: true // Specify whether to use null as the default for unspecified fields
-			}
-		};
-		knexConnectionConfig = knexSqliteConfig;
-
-		break;
-
 	case DatabaseTypeEnum.betterSqlite3:
-		const betterSqlitePath = process.env.DB_PATH || path.join(process.cwd(), ...['apps', 'api', 'data'], 'gauzy.sqlite3');
-		console.log('Better Sqlite DB Path: ' + betterSqlitePath);
+		const sqlitePath =
+			process.env.DB_PATH || path.join(process.cwd(), ...['apps', 'api', 'data'], 'gauzy.sqlite3');
+
+		console.log('Better Sqlite DB Path: ' + sqlitePath);
 
 		// MikroORM DB Config (Better-SQLite3)
 		const mikroOrmBetterSqliteConfig: MikroOrmBetterSqliteOptions = {
 			driver: BetterSqliteDriver,
-			dbName: betterSqlitePath,
+			dbName: sqlitePath,
 			persistOnCreate: true,
 			extensions: [SoftDeleteHandler],
 			namingStrategy: EntityCaseNamingStrategy,
@@ -332,7 +297,7 @@ switch (dbType) {
 		// TypeORM DB Config (Better-SQLite3)
 		const typeOrmBetterSqliteConfig: DataSourceOptions = {
 			type: dbType,
-			database: betterSqlitePath,
+			database: sqlitePath,
 			logging: 'all',
 			logger: 'file', // Removes console logging, instead logs all queries in a file ormlogs.log
 			synchronize: process.env.DB_SYNCHRONIZE === 'true', // We are using migrations, synchronize should be set to false.
@@ -350,7 +315,7 @@ switch (dbType) {
 			config: {
 				client: 'better-sqlite3',
 				connection: {
-					filename: betterSqlitePath
+					filename: sqlitePath
 				},
 				useNullAsDefault: true // Specify whether to use null as the default for unspecified fields
 			}

diff --git a/packages/core/package.json b/packages/core/package.json
@@ -61,7 +61,6 @@
 		"@mikro-orm/mysql": "^6.2.3",
 		"@mikro-orm/nestjs": "^5.2.3",
 		"@mikro-orm/postgresql": "^6.2.3",
-		"@mikro-orm/sqlite": "^6.2.3",
 		"@nestjs/apollo": "^12.1.0",
 		"@nestjs/axios": "^3.0.2",
 		"@nestjs/cache-manager": "^2.2.1",
@@ -119,7 +118,7 @@
 		"atlassian-connect-express": "^8.5.0",
 		"axios": "^1.7.4",
 		"bcrypt": "^5.1.1",
-		"better-sqlite3": "^9.4.3",
+		"better-sqlite3": "^9.5.0",
 		"cache-manager": "^5.3.2",
 		"cache-manager-redis-yet": "^4.1.2",
 		"camelcase": "^6.3.0",
@@ -185,7 +184,6 @@
 		"rimraf": "^3.0.2",
 		"rxjs": "^7.4.0",
 		"sql.js": "^1.5.0",
-		"sqlite3": "^5.1.7",
 		"streamifier": "^0.1.1",
 		"swagger-ui-express": "^5.0.0",
 		"typeorm": "^0.3.20",

diff --git a/packages/core/src/core/utils.ts b/packages/core/src/core/utils.ts
@@ -1,6 +1,5 @@
 import { BadRequestException } from '@nestjs/common';
 import { TypeOrmModuleOptions } from '@nestjs/typeorm';
-import { SqliteDriver } from '@mikro-orm/sqlite';
 import {
 	FindOptions as MikroORMFindOptions,
 	FilterQuery as MikroFilterQuery,
@@ -397,9 +396,7 @@ export function getDBType(dbConnection?: IDBConnectionOptions): any {
 	let dbType: any;
 	switch (dbORM) {
 		case MultiORMEnum.MikroORM:
-			if (dbConnection.driver instanceof SqliteDriver) {
-				dbType = DatabaseTypeEnum.sqlite;
-			} else if (dbConnection.driver instanceof BetterSqliteDriver) {
+			if (dbConnection.driver instanceof BetterSqliteDriver) {
 				dbType = DatabaseTypeEnum.betterSqlite3;
 			} else if (dbConnection.driver instanceof PostgreSqlDriver) {
 				dbType = DatabaseTypeEnum.postgres;

diff --git a/packages/desktop-api/package.json b/packages/desktop-api/package.json
@@ -30,7 +30,6 @@
 		"@grpc/grpc-js": "^1.6.7",
 		"@mikro-orm/core": "^6.2.3",
 		"@mikro-orm/nestjs": "^5.2.3",
-		"@mikro-orm/sqlite": "^6.2.3",
 		"@nestjs/common": "^10.3.7",
 		"@nestjs/core": "^10.3.7",
 		"@nestjs/typeorm": "^10.0.2",