Skip to content

Commit

Permalink
feat: rename model.
Browse files Browse the repository at this point in the history
  • Loading branch information
@hezhengxu2018
hezhengxu2018 committed Jul 31, 2023
1 parent 4458d05 commit b6ddab6
Show file tree
Hide file tree
Showing 12 changed files with 190 additions and 154 deletions.
16 changes: 7 additions & 9 deletions app/core/entity/ProxyModeCachedFiles.ts → app/core/entity/ProxyCache.ts
View file
Original file line number Diff line number Diff line change
@@ -1,36 +1,34 @@
import { Entity, EntityData } from './Entity';
import { EasyData } from '../util/EntityUtil';
interface ProxyModeData extends EntityData {
interface ProxyCacheData extends EntityData {
targetName: string;
fileType: string;
filePath: string;
version?: string;
lastErrorMessage?: string;
}

export type CreateProxyModeData = Omit<EasyData<ProxyModeData, 'id'>, 'id'>;
export type CreateProxyCacheData = Omit<EasyData<ProxyCacheData, 'id'>, 'id'>;

export class ProxyModeCachedFiles extends Entity {
export class ProxyCache extends Entity {
readonly targetName: string;
readonly fileType: string;
readonly filePath: string;
readonly version?: string;
lastErrorMessage?: string;

constructor(data: ProxyModeData) {
constructor(data: ProxyCacheData) {
super(data);
this.targetName = data.targetName;
this.fileType = data.fileType;
this.filePath = data.filePath;
this.version = data.version;
}

public static create(data: CreateProxyModeData): ProxyModeCachedFiles {
public static create(data: CreateProxyCacheData): ProxyCache {
const newData = { ...data, createdAt: new Date(), updatedAt: new Date() };
return new ProxyModeCachedFiles(newData);
return new ProxyCache(newData);
}

public static update(data: ProxyModeCachedFiles): ProxyModeCachedFiles {
public static update(data: ProxyCache): ProxyCache {
data.updatedAt = new Date();
return data;
}
Expand Down
13 changes: 9 additions & 4 deletions app/core/entity/Task.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import path from 'path';
import { Entity, EntityData } from './Entity';
import { EasyData, EntityUtil } from '../util/EntityUtil';
import { TaskType, TaskState } from '../../common/enum/Task';
import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../common/constants';
import dayjs from '../../common/dayjs';
import { HookEvent } from './HookEvent';

Expand Down Expand Up @@ -41,6 +42,12 @@ export type SyncPackageTaskOptions = {
specificVersions?: Array<string>;
};

export type UpdateProxyCacheTaskOptions = {
version?: string,
fileType: string,
filePath: string
};

export interface CreateHookTaskData extends TaskBaseData {
hookEvent: HookEvent;
}
Expand All @@ -61,7 +68,6 @@ export interface CreateSyncPackageTaskData extends TaskBaseData {
}

export interface CreateUpdateProxyCacheTaskData extends TaskBaseData {
targetName: string,
version?: string,
fileType: string,
filePath: string
Expand Down Expand Up @@ -242,7 +248,7 @@ export class Task<T extends TaskBaseData = TaskBaseData> extends Entity {
return task;
}

public static createUpdateProxyCache(targetName: string, options: CreateUpdateProxyCacheTaskData):CreateUpdateProxyCacheTask {
public static createUpdateProxyCache(targetName: string, options: UpdateProxyCacheTaskOptions):CreateUpdateProxyCacheTask {
const data = {
type: TaskType.UpdateProxyCache,
state: TaskState.Waiting,
Expand All @@ -251,14 +257,13 @@ export class Task<T extends TaskBaseData = TaskBaseData> extends Entity {
authorIp: HOST_NAME,
data: {
taskWorker: '',
targetName,
version: options?.version,
fileType: options.fileType,
filePath: options.filePath,
},
};
const task = this.create(data);
task.logPath = `/packages/${targetName}/update-manifests/${dayjs().format('YYYY/MM/DDHHmm')}-${task.taskId}.log`;
task.logPath = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${targetName}/update-manifest-log/${options.fileType}-${dayjs().format('YYYY/MM/DDHHmm')}-${task.taskId}.log`;
return task;
}

Expand Down
105 changes: 42 additions & 63 deletions app/core/service/ProxyModeService.ts → app/core/service/ProxyCacheService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ import { SingletonProto, AccessLevel, Inject } from '@eggjs/tegg';
import { EggHttpClient } from 'egg';
import { downloadToTempfile } from '../../common/FileUtil';
import { NPMRegistry, RegistryResponse } from '../../common/adapter/NPMRegistry';
import { ProxyModeCachedFiles } from '../entity/ProxyModeCachedFiles';
import { ProxyModeCachedFilesRepository } from '../../repository/ProxyModeCachedFilesRepository';
import { TaskRepository } from '../../repository/TaskRepository';
import { ProxyCache } from '../entity/ProxyCache';
import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository';
// import { TaskRepository } from '../../repository/TaskRepository';
import { AbstractService } from '../../common/AbstractService';
import { TaskService } from './TaskService';
import { readFile, rm } from 'node:fs/promises';
Expand All @@ -14,22 +14,24 @@ import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../common/constants';
import { DIST_NAMES } from '../entity/Package';
import type { PackageJSONType } from '../../repository/PackageRepository';
import { TaskType, TaskState } from '../../common/enum/Task';
import { Task } from '../entity/Task';
import { Task, UpdateProxyCacheTaskOptions, CreateUpdateProxyCacheTask } from '../entity/Task';

function isoNow() {
return new Date().toISOString();
}

@SingletonProto({
accessLevel: AccessLevel.PUBLIC,
})
export class ProxyModeService extends AbstractService {
export class ProxyCacheService extends AbstractService {
@Inject()
private readonly httpclient: EggHttpClient;
@Inject()
private readonly npmRegistry: NPMRegistry;
@Inject()
private readonly nfsAdapter: NFSAdapter;
@Inject()
private readonly proxyModeCachedFiles: ProxyModeCachedFilesRepository;
@Inject()
private readonly taskRepository: TaskRepository;
private readonly proxyCacheRepository: ProxyCacheRepository;
@Inject()
private readonly taskService: TaskService;

Expand All @@ -49,7 +51,7 @@ export class ProxyModeService extends AbstractService {
const manifest = await this.getPackageManifestAndCache(fullname, false);
const distTags = manifest['dist-tags'] || {};
const version = distTags[versionOrTag] ? distTags[versionOrTag] : versionOrTag;
const cachedFileInfo = await this.proxyModeCachedFiles.findCachedPackageVersionManifest(fullname, version, isFullManifests);
const cachedFileInfo = await this.proxyCacheRepository.findCachedPackageVersionManifest(fullname, version, isFullManifests);
const cachedStoreKey = cachedFileInfo?.filePath;
if (cachedStoreKey) {
const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey);
Expand All @@ -62,7 +64,7 @@ export class ProxyModeService extends AbstractService {
} catch {
// JSON parse error
await this.nfsAdapter.remove(cachedStoreKey);
await this.proxyModeCachedFiles.removePackageVersionStoreKey(fullname, isFullManifests);
await this.proxyCacheRepository.removePackageVersionStoreKey(fullname, isFullManifests);
throw new InternalServerError('manifest in NFS JSON parse error');
}
return nfsPkgVersionManifgest;
Expand All @@ -72,8 +74,8 @@ export class ProxyModeService extends AbstractService {
// not in NFS
const { storeKey, pkgVerisonManifest } = await this.getPackageVersionManifestFromSourceAndCache(fullname, version, isFullManifests);

const cachedFiles = await ProxyModeCachedFiles.create({ targetName: fullname, fileType: isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS, filePath: storeKey });
this.proxyModeCachedFiles.savePackageManifests(cachedFiles);
const cachedFiles = await ProxyCache.create({ targetName: fullname, fileType: isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS, filePath: storeKey });
this.proxyCacheRepository.savePackageManifests(cachedFiles);
return pkgVerisonManifest;
}

Expand All @@ -87,7 +89,7 @@ export class ProxyModeService extends AbstractService {
}


const cachedFileInfo = await this.proxyModeCachedFiles.findCachedPackageManifest(fullname, isFullManifests);
const cachedFileInfo = await this.proxyCacheRepository.findCachedPackageManifest(fullname, isFullManifests);
const cachedStoreKey = cachedFileInfo?.filePath;
if (cachedStoreKey) {
const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey);
Expand All @@ -100,17 +102,17 @@ export class ProxyModeService extends AbstractService {
} catch {
// JSON parse error
await this.nfsAdapter.remove(cachedStoreKey);
// TODO: remove
await this.proxyCacheRepository.removePackageVersionStoreKey(fullname, isFullManifests);
throw new InternalServerError('manifest in NFS JSON parse error');
}
return nfsPkgManifgest;
}
this.proxyModeCachedFiles.removePackageStoreKey(fullname, isFullManifests);
this.proxyCacheRepository.removePackageStoreKey(fullname, isFullManifests);
}

const { storeKey, pkgManifest } = await this.getPackageManifestFromSourceAndCache(fullname, isFullManifests);
const cachedFiles = await ProxyModeCachedFiles.create({ targetName: fullname, fileType: isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS, filePath: storeKey });
this.proxyModeCachedFiles.savePackageManifests(cachedFiles);
const cachedFiles = await ProxyCache.create({ targetName: fullname, fileType: isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS, filePath: storeKey });
this.proxyCacheRepository.savePackageManifests(cachedFiles);
return pkgManifest;
}

Expand Down Expand Up @@ -173,16 +175,8 @@ export class ProxyModeService extends AbstractService {
return { storeKey, proxyBytes, pkgManifest };
}

public async createTask(targetName, options) {
const existsTask = await this.taskRepository.findTaskByTargetName(targetName, TaskType.UpdateProxyCache);
if (existsTask) {
return existsTask;
}
try {
return await this.taskService.createTask(Task.createSyncBinary(targetName, options), false);
} catch (e) {
this.logger.error('[ProxyModeService.createTask] targetName: %s, error: %s', targetName, e);
}
public async createTask(targetName: string, options: UpdateProxyCacheTaskOptions): Promise<CreateUpdateProxyCacheTask> {
return await this.taskService.createTask(Task.createUpdateProxyCache(targetName, options), false) as CreateUpdateProxyCacheTask;
}

public async findTask(taskId: string) {
Expand All @@ -199,42 +193,27 @@ export class ProxyModeService extends AbstractService {

public async executeTask(task: Task) {
const logs: string[] = [];
await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
// const binaryName = task.targetName as BinaryName;
// const binaryAdapter = await this.getBinaryAdapter(binaryName);
// const logUrl = `${this.config.cnpmcore.registry}/-/binary/${binaryName}/syncs/${task.taskId}/log`;
// let logs: string[] = [];
// logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 Start sync binary "${binaryName}" 🚧🚧🚧🚧🚧`);
// if (!binaryAdapter) {
// task.error = 'unknow binaryName';
// logs.push(`[${isoNow()}] ❌ Synced "${binaryName}" fail, ${task.error}, log: ${logUrl}`);
// logs.push(`[${isoNow()}] ❌❌❌❌❌ "${binaryName}" ❌❌❌❌❌`);
// this.logger.error('[BinarySyncerService.executeTask:fail] taskId: %s, targetName: %s, %s',
// task.taskId, task.targetName, task.error);
// await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
// return;
// }

// await this.taskService.appendTaskLog(task, logs.join('\n'));
// logs = [];
// this.logger.info('[BinarySyncerService.executeTask:start] taskId: %s, targetName: %s, log: %s',
// task.taskId, task.targetName, logUrl);
// try {
// await this.syncDir(binaryAdapter, task, '/');
// logs.push(`[${isoNow()}] 🟢 log: ${logUrl}`);
// logs.push(`[${isoNow()}] 🟢🟢🟢🟢🟢 "${binaryName}" 🟢🟢🟢🟢🟢`);
// await this.taskService.finishTask(task, TaskState.Success, logs.join('\n'));
// this.logger.info('[BinarySyncerService.executeTask:success] taskId: %s, targetName: %s, log: %s',
// task.taskId, task.targetName, logUrl);
// } catch (err: any) {
// task.error = err.message;
// logs.push(`[${isoNow()}] ❌ Synced "${binaryName}" fail, ${task.error}, log: ${logUrl}`);
// logs.push(`[${isoNow()}] ❌❌❌❌❌ "${binaryName}" ❌❌❌❌❌`);
// this.logger.error('[BinarySyncerService.executeTask:fail] taskId: %s, targetName: %s, %s',
// task.taskId, task.targetName, task.error);
// this.logger.error(err);
// await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
// }
const targetName = task.targetName;
const { fileType, version } = (task as CreateUpdateProxyCacheTask).data;
logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 Start update "${targetName}-${fileType}" 🚧🚧🚧🚧🚧`);
try {
if (fileType === DIST_NAMES.ABBREVIATED || fileType === DIST_NAMES.MANIFEST) {
const isFull = fileType === DIST_NAMES.MANIFEST;
await this.getPackageVersionManifestFromSourceAndCache(targetName, version!, isFull);
} else {
const isFull = fileType === DIST_NAMES.FULL_MANIFESTS;
await this.getPackageManifestFromSourceAndCache(targetName, isFull);
}
} catch (error) {
task.error = error;
logs.push(`[${isoNow()}] ❌ ${task.error}, log: ${task.logPath}`);
logs.push(`[${isoNow()}] ❌❌❌❌❌ ${targetName}-${fileType} ${version} ❌❌❌❌❌`);
await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
this.logger.info('[PackageSyncerService.executeTask:fail-404] taskId: %s, targetName: %s, %s',
task.taskId, task.targetName, task.error);
return;
}
await this.taskService.finishTask(task, TaskState.Success, logs.join('\n'));
}

}
6 changes: 3 additions & 3 deletions app/port/controller/package/DownloadPackageVersionTar.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import { AbstractController } from '../AbstractController';
import { FULLNAME_REG_STRING, getScopeAndName } from '../../../common/PackageUtil';
import { NFSAdapter } from '../../../common/adapter/NFSAdapter';
import { PackageManagerService } from '../../../core/service/PackageManagerService';
import { ProxyModeService } from '../../../core/service/ProxyModeService';
import { ProxyCacheService } from '../../../core/service/ProxyCacheService';
import { PackageSyncerService } from '../../../core/service/PackageSyncerService';
import { SyncMode } from '../../../common/constants';

Expand All @@ -23,7 +23,7 @@ export class DownloadPackageVersionTarController extends AbstractController {
@Inject()
private packageManagerService: PackageManagerService;
@Inject()
private proxyModeService: ProxyModeService;
private proxyCacheService: ProxyCacheService;
@Inject()
private packageSyncerService: PackageSyncerService;
@Inject()
Expand Down Expand Up @@ -99,7 +99,7 @@ export class DownloadPackageVersionTarController extends AbstractController {
}

async #getTgzBuffer(ctx: EggContext, fullname: string, version: string) {
const { tgzBuffer } = await this.proxyModeService.getPackageVersionTarAndTempFilePath(fullname, ctx.url);
const { tgzBuffer } = await this.proxyCacheService.getPackageVersionTarAndTempFilePath(fullname, ctx.url);
const task = await this.packageSyncerService.createTask(fullname, {
authorIp: ctx.ip,
authorId: `pid_${process.pid}`,
Expand Down
6 changes: 3 additions & 3 deletions app/port/controller/package/ShowPackageController.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import { isSyncWorkerRequest } from '../../../common/SyncUtil';
import { PackageManagerService } from '../../../core/service/PackageManagerService';
import { CacheService } from '../../../core/service/CacheService';
import { SyncMode } from '../../../common/constants';
import { ProxyModeService } from '../../../core/service/ProxyModeService';
import { ProxyCacheService } from '../../../core/service/ProxyCacheService';
import { calculateIntegrity } from '../../../common/PackageUtil';

@HTTPController()
Expand All @@ -23,7 +23,7 @@ export class ShowPackageController extends AbstractController {
@Inject()
private cacheService: CacheService;
@Inject()
private proxyModeService: ProxyModeService;
private proxyCacheService: ProxyCacheService;

@HTTPMethod({
// GET /:fullname
Expand Down Expand Up @@ -71,7 +71,7 @@ export class ShowPackageController extends AbstractController {
let result: { etag: string; data: any, blockReason: string };
if (this.config.cnpmcore.syncMode === SyncMode.proxy) {
// proxy mode
const { pkgManifest } = await this.proxyModeService.getPackageManifestFromSourceAndCache(fullname, isFullManifests);
const { pkgManifest } = await this.proxyCacheService.getPackageManifestFromSourceAndCache(fullname, isFullManifests);
const nfsBytes = Buffer.from(JSON.stringify(pkgManifest));
const { shasum: etag } = await calculateIntegrity(nfsBytes);
result = { data: pkgManifest, etag, blockReason: '' };
Expand Down
8 changes: 4 additions & 4 deletions app/port/controller/package/ShowPackageVersionController.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import { AbstractController } from '../AbstractController';
import { getScopeAndName, FULLNAME_REG_STRING } from '../../../common/PackageUtil';
import { isSyncWorkerRequest } from '../../../common/SyncUtil';
import { PackageManagerService } from '../../../core/service/PackageManagerService';
import { ProxyModeService } from '../../../core/service/ProxyModeService';
import { ProxyCacheService } from '../../../core/service/ProxyCacheService';
import { Spec } from '../../../port/typebox';
import { SyncMode } from '../../../common/constants';

Expand All @@ -21,7 +21,7 @@ export class ShowPackageVersionController extends AbstractController {
@Inject()
private packageManagerService: PackageManagerService;
@Inject()
private proxyModeService: ProxyModeService;
private proxyCacheService: ProxyCacheService;

@HTTPMethod({
// GET /:fullname/:versionSpec
Expand All @@ -39,7 +39,7 @@ export class ShowPackageVersionController extends AbstractController {
let { blockReason, manifest, pkg } = await this.packageManagerService.showPackageVersionManifest(scope, name, versionSpec, isSync, isFullManifests);
if (!pkg) {
if (this.config.cnpmcore.syncMode === SyncMode.proxy) {
manifest = await this.proxyModeService.getPackageVersionManifest(fullname, versionSpec, isFullManifests);
manifest = await this.proxyCacheService.getPackageVersionManifest(fullname, versionSpec, isFullManifests);
} else {
const allowSync = this.getAllowSync(ctx);
throw this.createPackageNotFoundErrorWithRedirect(fullname, undefined, allowSync);
Expand All @@ -51,7 +51,7 @@ export class ShowPackageVersionController extends AbstractController {
}
if (!manifest) {
if (this.config.cnpmcore.syncMode === SyncMode.proxy) {
manifest = await this.proxyModeService.getPackageVersionManifest(fullname, versionSpec, isFullManifests);
manifest = await this.proxyCacheService.getPackageVersionManifest(fullname, versionSpec, isFullManifests);
} else {
throw new NotFoundError(`${fullname}@${versionSpec} not found`);
}
Expand Down
Loading

0 comments on commit b6ddab6

Please sign in to comment.