IMN 704 - Anac Certified attributes importer (#961)

.github/workflows/build-push.yaml

@@ -79,6 +79,8 @@ jobs:
             dockerfile_path: packages/producer-key-readmodel-writer
           - image_name: producer-keychain-readmodel-writer
             dockerfile_path: packages/producer-keychain-readmodel-writer
+          - image_name: anac-certified-attributes-importer
+            dockerfile_path: packages/anac-certified-attributes-importer
           - image_name: one-trust-notices
             dockerfile_path: packages/one-trust-notices
 

package.json

@@ -24,6 +24,7 @@
     "start:key-readmodel-writer": "turbo start --filter pagopa-interop-key-readmodel-writer",
     "start:bff": "turbo start --filter pagopa-interop-backend-for-frontend",
     "start:api-gateway": "turbo start --filter pagopa-interop-api-gateway",
+    "start:anac-certified-attributes-importer": "turbo start --filter pagopa-interop-anac-certified-attributes-importer",
     "start:one-trust-notices": "turbo start --filter pagopa-interop-one-trust-notices",
     "test": "turbo test",
     "build": "turbo build",

packages/anac-certified-attributes-importer/.env

@@ -0,0 +1,27 @@
+LOG_LEVEL=info
+
+READMODEL_DB_HOST="localhost"
+READMODEL_DB_NAME="readmodel"
+READMODEL_DB_USERNAME="root"
+READMODEL_DB_PASSWORD="example"
+READMODEL_DB_PORT=27017
+
+SFTP_HOST=""
+SFTP_PORT=0
+SFTP_USERNAME=""
+SFTP_PASSWORD=""
+SFTP_FILENAME_PREFIX=""
+SFTP_PATH=""
+FORCE_REMOTE_FILE_NAME=""
+
+INTERNAL_JWT_KID=ffcc9b5b-4612-49b1-9374-9d203a3834f2
+INTERNAL_JWT_SUBJECT="dev-refactor.interop-eservice-descriptors-archiver"
+INTERNAL_JWT_ISSUER="dev-refactor.interop.pagopa.it"
+INTERNAL_JWT_AUDIENCE="refactor.dev.interop.pagopa.it/internal"
+INTERNAL_JWT_SECONDS_DURATION=60
+
+TENANT_PROCESS_URL="http://localhost:3500"
+RECORDS_PROCESS_BATCH_SIZE=5
+ANAC_TENANT_ID=""
+
+AWS_CONFIG_FILE=aws.config.local

packages/anac-certified-attributes-importer/Dockerfile

@@ -0,0 +1,44 @@
+FROM node:20.14.0-slim@sha256:5e8ac65a0231d76a388683d07ca36a9769ab019a85d85169fe28e206f7a3208e as build
+
+RUN corepack enable
+
+WORKDIR /app
+COPY package.json /app/
+COPY pnpm-lock.yaml /app/
+COPY pnpm-workspace.yaml /app/
+
+COPY ./packages/anac-certified-attributes-importer/package.json /app/packages/anac-certified-attributes-importer/package.json
+COPY ./packages/commons/package.json /app/packages/commons/package.json
+COPY ./packages/models/package.json /app/packages/models/package.json
+COPY ./packages/api-clients/package.json /app/packages/api-clients/package.json
+
+RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
+
+COPY tsconfig.json /app/
+COPY turbo.json /app/
+COPY ./packages/anac-certified-attributes-importer /app/packages/anac-certified-attributes-importer/
+COPY ./packages/commons /app/packages/commons
+COPY ./packages/models /app/packages/models
+COPY ./packages/api-clients /app/packages/api-clients
+
+RUN pnpm build && \
+    rm -rf /app/node_modules/.modules.yaml && \
+    rm -rf /app/node_modules/.cache && \
+    mkdir /out && \
+    cp -a --parents -t /out \
+      node_modules packages/anac-certified-attributes-importer/node_modules \
+      package*.json packages/anac-certified-attributes-importer/package*.json \
+      packages/commons/ \
+      packages/models/ \
+      packages/api-clients  \
+      packages/anac-certified-attributes-importer/dist && \
+    find /out -exec touch -h --date=@0 {} \;
+
+FROM node:20.14.0-slim@sha256:5e8ac65a0231d76a388683d07ca36a9769ab019a85d85169fe28e206f7a3208e as final 
+
+COPY --from=build /out /app
+
+WORKDIR /app/packages/anac-certified-attributes-importer
+EXPOSE 3000
+
+CMD [ "node", "." ]

packages/anac-certified-attributes-importer/README.md

@@ -0,0 +1,3 @@
+# ANAC certified attributes importer
+
+This job assigns the ANAC certified attributes to the authorized tenants

packages/anac-certified-attributes-importer/aws.config.local

@@ -0,0 +1,4 @@
+[default]
+aws_access_key_id=testawskey
+aws_secret_access_key=testawssecret
+region=eu-central-1

packages/anac-certified-attributes-importer/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "pagopa-interop-anac-certified-attributes-importer",
+  "private": true,
+  "version": "1.0.0",
+  "description": "PagoPA Interoperability anac-certified-attributes-importer job",
+  "main": "dist",
+  "type": "module",
+  "scripts": {
+    "test": "vitest",
+    "lint": "eslint . --ext .ts,.tsx",
+    "lint:autofix": "eslint . --ext .ts,.tsx --fix",
+    "format:check": "prettier --check src",
+    "format:write": "prettier --write src",
+    "start": "node --loader ts-node/esm -r 'dotenv-flow/config' --watch ./src/index.ts",
+    "build": "tsc",
+    "check": "tsc --project tsconfig.check.json"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "Apache-2.0",
+  "devDependencies": {
+    "@pagopa/eslint-config": "3.0.0",
+    "@types/node": "20.14.6",
+    "@types/ssh2-sftp-client": "9.0.4",
+    "pagopa-interop-commons-test": "workspace:*",
+    "prettier": "2.8.8",
+    "testcontainers": "10.9.0",
+    "ts-node": "10.9.2",
+    "typescript": "5.4.5",
+    "vitest": "1.6.0"
+  },
+  "dependencies": {
+    "csv": "^6.3.2",
+    "dotenv-flow": "4.1.0",
+    "pagopa-interop-commons": "workspace:*",
+    "pagopa-interop-models": "workspace:*",
+    "pagopa-interop-api-clients": "workspace:*",
+    "ssh2-sftp-client": "^9.1.0",
+    "zod": "3.23.8"
+  }
+}

packages/anac-certified-attributes-importer/src/config/config.ts

@@ -0,0 +1,34 @@
+import {
+  APIEndpoint,
+  LoggerConfig,
+  ReadModelDbConfig,
+  TokenGenerationConfig,
+} from "pagopa-interop-commons";
+import { z } from "zod";
+import { SftpConfig } from "./sftpConfig.js";
+
+const AnacCertifiedAttributesImporterConfig = LoggerConfig.and(
+  ReadModelDbConfig
+)
+  .and(SftpConfig)
+  .and(TokenGenerationConfig)
+  .and(
+    z
+      .object({
+        TENANT_PROCESS_URL: APIEndpoint,
+        RECORDS_PROCESS_BATCH_SIZE: z.number(),
+        ANAC_TENANT_ID: z.string(),
+      })
+      .transform((c) => ({
+        tenantProcessUrl: c.TENANT_PROCESS_URL,
+        recordsProcessBatchSize: c.RECORDS_PROCESS_BATCH_SIZE,
+        anacTenantId: c.ANAC_TENANT_ID,
+      }))
+  );
+
+export type AnacCertifiedAttributesImporterConfig = z.infer<
+  typeof AnacCertifiedAttributesImporterConfig
+>;
+
+export const config: AnacCertifiedAttributesImporterConfig =
+  AnacCertifiedAttributesImporterConfig.parse(process.env);

packages/anac-certified-attributes-importer/src/config/sftpConfig.ts

@@ -0,0 +1,23 @@
+import { z } from "zod";
+
+export const SftpConfig = z
+  .object({
+    SFTP_HOST: z.string(),
+    SFTP_PORT: z.coerce.number().min(1001),
+    SFTP_USERNAME: z.string(),
+    SFTP_PASSWORD: z.string(),
+    SFTP_FILENAME_PREFIX: z.string(),
+    SFTP_PATH: z.string(),
+    FORCE_REMOTE_FILE_NAME: z.string().optional(),
+  })
+  .transform((c) => ({
+    host: c.SFTP_HOST,
+    port: c.SFTP_PORT,
+    username: c.SFTP_USERNAME,
+    password: c.SFTP_PASSWORD,
+    fileNamePrefix: c.SFTP_FILENAME_PREFIX,
+    folderPath: c.SFTP_PATH,
+    forceFileName: c.FORCE_REMOTE_FILE_NAME,
+  }));
+
+export type SftpConfig = z.infer<typeof SftpConfig>;

packages/anac-certified-attributes-importer/src/index.ts

@@ -0,0 +1,37 @@
+import crypto from "crypto";
+import {
+  InteropTokenGenerator,
+  ReadModelRepository,
+  RefreshableInteropToken,
+  logger,
+} from "pagopa-interop-commons";
+import { config } from "./config/config.js";
+import { SftpClient } from "./service/sftpService.js";
+import { ReadModelQueries } from "./service/readmodelQueriesService.js";
+import { TenantProcessService } from "./service/tenantProcessService.js";
+import { importAttributes } from "./service/processor.js";
+
+const sftpClient: SftpClient = new SftpClient(config);
+const readModelClient = ReadModelRepository.init(config);
+const readModelQueries: ReadModelQueries = new ReadModelQueries(
+  readModelClient
+);
+
+const tokenGenerator = new InteropTokenGenerator(config);
+const refreshableToken = new RefreshableInteropToken(tokenGenerator);
+const tenantProcess = new TenantProcessService(config.tenantProcessUrl);
+
+const loggerInstance = logger({
+  serviceName: "anac-certified-attributes-importer",
+  correlationId: crypto.randomUUID(),
+});
+
+await importAttributes(
+  sftpClient,
+  readModelQueries,
+  tenantProcess,
+  refreshableToken,
+  config.recordsProcessBatchSize,
+  config.anacTenantId,
+  loggerInstance
+);

packages/anac-certified-attributes-importer/src/model/csvRowModel.ts

@@ -0,0 +1,45 @@
+import { z } from "zod";
+
+const Row = z.object({
+  codiceFiscaleGestore: z.string().min(1),
+  denominazioneGestore: z.string().min(1),
+  PEC: z.string().min(1),
+  ANAC_incaricato: z
+    .string()
+    .transform((value) => value.toUpperCase() === "TRUE"),
+  ANAC_abilitato: z
+    .string()
+    .transform((value) => value.toUpperCase() === "TRUE"),
+  ANAC_in_convalida: z
+    .string()
+    .transform((value) => value.toUpperCase() === "TRUE"),
+});
+
+export const NonPaRow = Row.transform((value) => ({
+  cf_gestore: value.codiceFiscaleGestore,
+  denominazione: value.denominazioneGestore,
+  domicilio_digitale: value.PEC,
+  anac_incaricato: value.ANAC_incaricato,
+  anac_abilitato: value.ANAC_abilitato,
+  anac_in_convalida: value.ANAC_in_convalida,
+}));
+
+export type NonPaRow = z.infer<typeof NonPaRow>;
+
+export const PaRow = Row.extend({
+  codiceIPA: z.string().min(1),
+}).transform((value) => ({
+  cf_gestore: value.codiceFiscaleGestore,
+  denominazione: value.denominazioneGestore,
+  domicilio_digitale: value.PEC,
+  codice_ipa: value.codiceIPA,
+  anac_incaricato: value.ANAC_incaricato,
+  anac_abilitato: value.ANAC_abilitato,
+  anac_in_convalida: value.ANAC_in_convalida,
+}));
+
+export type PaRow = z.infer<typeof PaRow>;
+
+export const CsvRow = PaRow.or(NonPaRow);
+
+export type CsvRow = z.infer<typeof CsvRow>;

packages/anac-certified-attributes-importer/src/model/interopContextModel.ts

@@ -0,0 +1,8 @@
+import { z } from "zod";
+
+export const InteropContext = z.object({
+  correlationId: z.string(),
+  bearerToken: z.string(),
+});
+
+export type InteropContext = z.infer<typeof InteropContext>;

packages/anac-certified-attributes-importer/src/model/processorModel.ts

@@ -0,0 +1,18 @@
+import { ExternalId } from "pagopa-interop-models";
+import { CsvRow } from "./csvRowModel.js";
+
+export type BatchParseResult = {
+  processedRecordsCount: number;
+  records: CsvRow[];
+};
+
+export type AttributeIdentifiers = {
+  id: string;
+  externalId: ExternalId;
+};
+
+export type AnacAttributes = {
+  anacAbilitato: AttributeIdentifiers;
+  anacInConvalida: AttributeIdentifiers;
+  anacIncaricato: AttributeIdentifiers;
+};