(WIP) Add Logging #175

package.json

@@ -12,6 +12,7 @@
     "@oclif/command": "^1",
     "@oclif/config": "^1",
     "@oclif/dev-cli": "^1",
+    "@oclif/parser": "^3.8.7",
     "@oclif/plugin-help": "^5",
     "@oclif/plugin-warn-if-update-available": "^2.0.3",
     "@oclif/test": "^1",
@@ -28,6 +29,7 @@
     "@types/tmp": "^0.2.2",
     "@types/uuid": "^8.3.4",
     "@types/xml2js": "^0.4.9",
+    "@types/winston": "^2.4.4",
     "aws-sdk": "^2.1074.0",
     "chai": "^4",
     "csv-parse": "^4.16.0",

src/commands/convert/asff2hdf.ts

@@ -1,9 +1,12 @@
-import {Command, flags} from '@oclif/command'
+import BaseCommand from '../../utils/base-command'
+import {OutputFlags} from '@oclif/parser'
+import {flags} from '@oclif/command'
 import fs from 'fs'
 import {ASFFMapper as Mapper} from '@mitre/hdf-converters'
-import {checkSuffix} from '../../utils/global'
+import {checkSuffix, convertFullPathToFilename} from '../../utils/global'
+import {getHDFSummary} from '../../utils/logging'
 
-export default class ASFF2HDF extends Command {
+export default class ASFF2HDF extends BaseCommand {
   static usage = 'convert:asff2hdf -i <asff-finding-json> [--securityhub <standard-1-json> ... <standard-n-json>] -o <hdf-scan-results-json>'
 
   static description = 'Translate a AWS Security Finding Format JSON into a Heimdall Data Format JSON file'
@@ -12,20 +15,35 @@ export default class ASFF2HDF extends Command {
     'saf convert:asff2hdf -i asff-findings.json --sh <standard-1-json> ... <standard-n-json> -o output-hdf-name.json']
 
   static flags = {
-    help: flags.help({char: 'h'}),
-    input: flags.string({char: 'i', required: true}),
+    ...BaseCommand.flags,
     securityhub: flags.string({required: false, multiple: true}),
-    output: flags.string({char: 'o', required: true}),
   }
 
   async run() {
-    const {flags} = this.parse(ASFF2HDF)
+    const flags = this.parsedFlags as OutputFlags<typeof ASFF2HDF.flags>
+
     let securityhub
     if (flags.securityhub) {
-      securityhub = flags.securityhub.map(file => fs.readFileSync(file, 'utf-8'))
+      this.logger.verbose('Reading ASFF standards')
+      securityhub = flags.securityhub.map((file: fs.PathOrFileDescriptor) => fs.readFileSync(file, 'utf-8'))
     }
 
-    const converter = new Mapper(fs.readFileSync(flags.input, 'utf-8'), securityhub)
-    fs.writeFileSync(checkSuffix(flags.output), JSON.stringify(converter.toHdf()))
+    // Read Data
+    this.logger.verbose(`Reading ASFF file: ${flags.input}`)
+    const inputDataText = fs.readFileSync(flags.input, 'utf-8')
+
+    // Strip Extra .json from output filename
+    const fileName = checkSuffix(flags.output)
+    this.logger.verbose(`Output Filename: ${fileName}`)
+
+    // Convert the data
+    const converter = new Mapper(inputDataText, securityhub)
+    this.logger.info('Starting conversion from ASFF to HDF')
+    const converted = converter.toHdf()
+
+    // Write to file
+    this.logger.info(`Output File "${convertFullPathToFilename(fileName)}": ${getHDFSummary(converted)}`)
+    fs.writeFileSync(fileName, JSON.stringify(converted))
+    this.logger.verbose(`HDF successfully written to ${fileName}`)
   }
 }

src/commands/convert/aws_config2hdf.ts

@@ -1,24 +1,27 @@
-import {Command, flags} from '@oclif/command'
+import BaseCommand, {omitFlags} from '../../utils/base-command'
+import {OutputFlags} from '@oclif/parser'
+import {flags} from '@oclif/command'
 import fs from 'fs'
 import {AwsConfigMapper as Mapper} from '@mitre/hdf-converters'
 import {ExecJSON} from 'inspecjs'
-import {checkSuffix} from '../../utils/global'
+import {checkSuffix, convertFullPathToFilename} from '../../utils/global'
+import {getHDFSummary} from '../../utils/logging'
+import _ from 'lodash'
 
-export default class AWSConfig2HDF extends Command {
+export default class AWSConfig2HDF extends BaseCommand {
   static usage = 'convert:aws_config2hdf -a, --accessKeyId=accessKeyId -r, --region=region -s, --secretAccessKey=secretAccessKey -t, --sessionToken=sessionToken -o, --output=OUTPUT'
 
   static description = 'Pull Configuration findings from AWS Config and convert into a Heimdall Data Format JSON file'
 
   static examples = ['saf convert:aws_config2hdf -a ABCDEFGHIJKLMNOPQRSTUV -s +4NOT39A48REAL93SECRET934 -r us-east-1 -o output-hdf-name.json']
 
   static flags = {
-    help: flags.help({char: 'h'}),
+    ...omitFlags(['input']),
     accessKeyId: flags.string({char: 'a', required: false}),
     secretAccessKey: flags.string({char: 's', required: false}),
     sessionToken: flags.string({char: 't', required: false}),
     region: flags.string({char: 'r', required: true}),
     insecure: flags.boolean({char: 'i', required: false, default: false, description: 'Disable SSL verification, this is insecure.'}),
-    output: flags.string({char: 'o', required: true}),
   }
 
   // Refs may not be defined if no resources were found
@@ -44,8 +47,13 @@ export default class AWSConfig2HDF extends Command {
   }
 
   async run() {
-    const {flags} = this.parse(AWSConfig2HDF)
+    const flags = this.parsedFlags as OutputFlags<typeof AWSConfig2HDF.flags>
 
+    // Strip Extra .json from output filename
+    const fileName = checkSuffix(flags.output)
+    this.logger.verbose(`Output Filename: ${fileName}`)
+
+    // Convert the data
     const converter = flags.accessKeyId && flags.secretAccessKey ? new Mapper({
       credentials: {
         accessKeyId: flags.accessKeyId || '',
@@ -54,7 +62,14 @@ export default class AWSConfig2HDF extends Command {
       },
       region: flags.region,
     }, !flags.insecure) : new Mapper({region: flags.region}, !flags.insecure)
+    this.logger.info('Starting conversion from AWS Config to HDF')
+
+    const converted = this.ensureRefs(await converter.toHdf())
+    this.logger.info('Converted AWS Config to HDF')
 
-    fs.writeFileSync(checkSuffix(flags.output), JSON.stringify(this.ensureRefs(await converter.toHdf())))
+    // Write to file
+    this.logger.info(`Output File "${convertFullPathToFilename(fileName)}": ${getHDFSummary(converted)}`)
+    fs.writeFileSync(fileName, JSON.stringify(converted))
+    this.logger.verbose(`HDF successfully written to ${fileName}`)
   }
 }

src/commands/convert/burpsuite2hdf.ts

@@ -1,25 +1,40 @@
-import {Command, flags} from '@oclif/command'
+import BaseCommand from '../../utils/base-command'
+import {OutputFlags} from '@oclif/parser'
 import fs from 'fs'
 import {BurpSuiteMapper as Mapper} from '@mitre/hdf-converters'
-import {checkSuffix} from '../../utils/global'
+import {checkSuffix, convertFullPathToFilename} from '../../utils/global'
+import {getHDFSummary} from '../../utils/logging'
 
-export default class Burpsuite2HDF extends Command {
+export default class Burpsuite2HDF extends BaseCommand {
   static usage = 'convert:burpsuite2hdf -i, --input=XML -o, --output=OUTPUT'
 
   static description = 'Translate a BurpSuite Pro XML file into a Heimdall Data Format JSON file'
 
   static examples = ['saf convert:burpsuite2hdf -i burpsuite_results.xml -o output-hdf-name.json']
 
   static flags = {
-    help: flags.help({char: 'h'}),
-    input: flags.string({char: 'i', required: true}),
-    output: flags.string({char: 'o', required: true}),
+    ...BaseCommand.flags,
   }
 
   async run() {
-    const {flags} = this.parse(Burpsuite2HDF)
+    const flags = this.parsedFlags as OutputFlags<typeof Burpsuite2HDF.flags>
 
-    const converter = new Mapper(fs.readFileSync(flags.input, 'utf-8'))
-    fs.writeFileSync(checkSuffix(flags.output), JSON.stringify(converter.toHdf()))
+    // Read Data
+    this.logger.verbose(`Reading Burpsuite Scan: ${flags.input}`)
+    const inputDataText = fs.readFileSync(flags.input, 'utf-8')
+
+    // Strip Extra .json from output filename
+    const fileName = checkSuffix(flags.output)
+    this.logger.verbose(`Output Filename: ${fileName}`)
+
+    // Convert the data
+    const converter = new Mapper(inputDataText)
+    this.logger.info('Starting conversion from Burpsuite to HDF')
+    const converted = converter.toHdf()
+
+    // Write to file
+    this.logger.info(`Output File "${convertFullPathToFilename(fileName)}": ${getHDFSummary(converted)}`)
+    fs.writeFileSync(fileName, JSON.stringify(converted))
+    this.logger.verbose(`HDF successfully written to ${fileName}`)
   }
 }

src/commands/convert/dbprotect2hdf.ts

@@ -1,25 +1,40 @@
-import {Command, flags} from '@oclif/command'
+import BaseCommand from '../../utils/base-command'
+import {OutputFlags} from '@oclif/parser'
 import fs from 'fs'
 import {DBProtectMapper as Mapper} from '@mitre/hdf-converters'
-import {checkSuffix} from '../../utils/global'
+import {checkSuffix, convertFullPathToFilename} from '../../utils/global'
+import {getHDFSummary} from '../../utils/logging'
 
-export default class DBProtect2HDF extends Command {
+export default class DBProtect2HDF extends BaseCommand {
   static usage = 'convert:dbprotect2hdf -i, --input=XML -o, --output=OUTPUT'
 
   static description = 'Translate a DBProtect report in "Check Results Details" XML format into a Heimdall Data Format JSON file'
 
   static examples = ['saf convert:dbprotect2hdf -i check_results_details_report.xml -o output-hdf-name.json']
 
   static flags = {
-    help: flags.help({char: 'h'}),
-    input: flags.string({char: 'i', required: true}),
-    output: flags.string({char: 'o', required: true}),
+    ...BaseCommand.flags,
   }
 
   async run() {
-    const {flags} = this.parse(DBProtect2HDF)
+    const flags = this.parsedFlags as OutputFlags<typeof DBProtect2HDF.flags>
 
-    const converter = new Mapper(fs.readFileSync(flags.input, 'utf-8'))
-    fs.writeFileSync(checkSuffix(flags.output), JSON.stringify(converter.toHdf()))
+    // Read Data
+    this.logger.verbose(`Reading DB Protect file: ${flags.input}`)
+    const inputDataText = fs.readFileSync(flags.input, 'utf-8')
+
+    // Strip Extra .json from output filename
+    const fileName = checkSuffix(flags.output)
+    this.logger.verbose(`Output Filename: ${fileName}`)
+
+    // Convert the data
+    const converter = new Mapper(inputDataText)
+    this.logger.info('Starting conversion from DB Protect to HDF')
+    const converted = converter.toHdf()
+
+    // Write to file
+    this.logger.info(`Output File "${convertFullPathToFilename(fileName)}": ${getHDFSummary(converted)}`)
+    fs.writeFileSync(fileName, JSON.stringify(converted))
+    this.logger.verbose(`HDF successfully written to ${fileName}`)
   }
 }

src/commands/convert/fortify2hdf.ts

@@ -1,25 +1,40 @@
-import {Command, flags} from '@oclif/command'
+import BaseCommand from '../../utils/base-command'
+import {OutputFlags} from '@oclif/parser'
 import * as fs from 'fs'
 import {FortifyMapper as Mapper} from '@mitre/hdf-converters'
-import {checkSuffix} from '../../utils/global'
+import {checkSuffix, convertFullPathToFilename} from '../../utils/global'
+import {getHDFSummary} from '../../utils/logging'
 
-export default class Fortify2HDF extends Command {
+export default class Fortify2HDF extends BaseCommand {
   static usage = 'convert:fortify2hdf -i, --input=FVDL -o, --output=OUTPUT'
 
   static description = 'Translate a Fortify results FVDL file into a Heimdall Data Format JSON file'
 
   static examples = ['saf convert:fortify2hdf -i audit.fvdl -o output-hdf-name.json']
 
   static flags = {
-    help: flags.help({char: 'h'}),
-    input: flags.string({char: 'i', required: true}),
-    output: flags.string({char: 'o', required: true}),
+    ...BaseCommand.flags,
   }
 
   async run() {
-    const {flags} = this.parse(Fortify2HDF)
+    const flags = this.parsedFlags as OutputFlags<typeof Fortify2HDF.flags>
 
-    const converter = new Mapper(fs.readFileSync(flags.input, 'utf-8'))
-    fs.writeFileSync(checkSuffix(flags.output), JSON.stringify(converter.toHdf()))
+    // Read Data
+    this.logger.verbose(`Reading Fortify file: ${flags.input}`)
+    const inputDataText = fs.readFileSync(flags.input, 'utf-8')
+
+    // Strip Extra .json from output filename
+    const fileName = checkSuffix(flags.output)
+    this.logger.verbose(`Output Filename: ${fileName}`)
+
+    // Convert the data
+    const converter = new Mapper(inputDataText)
+    this.logger.info('Starting conversion from Fortify to HDF')
+    const converted = converter.toHdf()
+
+    // Write to file
+    this.logger.info(`Output File "${convertFullPathToFilename(fileName)}": ${getHDFSummary(converted)}`)
+    fs.writeFileSync(fileName, JSON.stringify(converted))
+    this.logger.verbose(`HDF successfully written to ${fileName}`)
   }
 }

src/commands/convert/hdf2asff.ts

@@ -1,4 +1,6 @@
-import {Command, flags} from '@oclif/command'
+import BaseCommand from '../../utils/base-command'
+import {OutputFlags} from '@oclif/parser'
+import {flags} from '@oclif/command'
 import * as fs from 'fs'
 import https from 'https'
 import {FromHdfToAsffMapper as Mapper} from '@mitre/hdf-converters'
@@ -8,46 +10,53 @@ import {checkSuffix, sliceIntoChunks} from '../../utils/global'
 import _ from 'lodash'
 import {BatchImportFindingsRequestFindingList} from 'aws-sdk/clients/securityhub'
 
-export default class HDF2ASFF extends Command {
+export default class HDF2ASFF extends BaseCommand {
   static usage = 'convert:hdf2asff -i, --input=HDF-JSON -o, --output=ASFF-JSON-Folder -a, --accountId=accountId -r, --region=region -t, --target=target -u, --upload'
 
   static description = 'Translate a Heimdall Data Format JSON file into AWS Security Findings Format JSON file(s) and/or upload to AWS Security Hub'
 
   static examples = ['saf convert:hdf2asff -i rhel7-scan_02032022A.json -a 123456789 -r us-east-1 -t rhel7_example_host -o rhel7.asff', 'saf convert:hdf2asff -i rds_mysql_i123456789scan_03042022A.json -a 987654321 -r us-west-1 -t Instance_i123456789 -u', 'saf convert:hdf2asff -i snyk_acme_project5_hdf_04052022A.json -a 2143658798 -r us-east-1 -t acme_project5 -o snyk_acme_project5 -u']
 
   static flags = {
-    help: flags.help({char: 'h'}),
+    ...BaseCommand.flags,
     accountId: flags.string({char: 'a', required: true, description: 'AWS Account ID'}),
     region: flags.string({char: 'r', required: true, description: 'SecurityHub Region'}),
-    input: flags.string({char: 'i', required: true, description: 'Input HDF JSON File'}),
     target: flags.string({char: 't', required: true, description: 'Unique name for target to track findings across time'}),
     upload: flags.boolean({char: 'u', required: false, description: 'Upload findings to AWS Security Hub'}),
-    output: flags.string({char: 'o', required: false, description: 'Output ASFF JSON Folder'}),
     insecure: flags.boolean({char: 'I', required: false, default: false, description: 'Disable SSL verification, this is insecure.'}),
     certificate: flags.string({char: 'C', required: false, description: 'Trusted signing certificate file'}),
   }
 
   async run() {
-    const {flags} = this.parse(HDF2ASFF)
+    const flags = this.parsedFlags as OutputFlags<typeof HDF2ASFF.flags>
 
-    const converted = new Mapper(JSON.parse(fs.readFileSync(flags.input, 'utf-8')), {
+    // Read Data
+    this.logger.verbose(`Reading HDF file: ${flags.input}`)
+    const inputDataText = fs.readFileSync(flags.input, 'utf-8')
+
+    const converter = new Mapper(JSON.parse(inputDataText), {
       awsAccountId: flags.accountId,
       region: flags.region,
       target: flags.target,
       input: flags.input,
-    }).toAsff()
+    })
+    this.logger.info('Starting conversion from HDF to ASFF')
+    const converted = converter.toAsff()
 
     if (flags.output) {
       const convertedSlices = sliceIntoChunks(converted, 100)
       const outputFolder = flags.output?.replace('.json', '') || 'asff-output'
       fs.mkdirSync(outputFolder)
+      this.logger.verbose(`Created output folder: ${outputFolder}`)
       if (convertedSlices.length === 1) {
         const outfilePath = path.join(outputFolder, checkSuffix(flags.output))
         fs.writeFileSync(outfilePath, JSON.stringify(convertedSlices[0]))
+        this.logger.verbose(`ASFF successfully written to ${outfilePath}`)
       } else {
         convertedSlices.forEach((slice, index) => {
           const outfilePath = path.join(outputFolder, `${checkSuffix(flags.output || '').replace('.json', '')}.p${index}.json`)
           fs.writeFileSync(outfilePath, JSON.stringify(slice))
+          this.logger.verbose(`ASFF successfully written to ${outfilePath}`)
         })
       }
     }
@@ -57,7 +66,7 @@ export default class HDF2ASFF extends Command {
       const convertedSlices = sliceIntoChunks(converted, 100)
 
       if (flags.insecure) {
-        console.warn('WARNING: Using --insecure will make all connections to AWS open to MITM attacks, if possible pass a certificate file with --certificate')
+        this.logger.warn('WARNING: Using --insecure will make all connections to AWS open to MITM attacks, if possible pass a certificate file with --certificate')
       }
 
       const clientOptions: AWS.SecurityHub.ClientConfiguration = {
@@ -77,12 +86,12 @@ export default class HDF2ASFF extends Command {
         convertedSlices.map(async chunk => {
           try {
             const result = await client.batchImportFindings({Findings: chunk}).promise()
-            console.log(
+            this.logger.info(
               `Uploaded ${chunk.length} controls. Success: ${result.SuccessCount}, Fail: ${result.FailedCount}`,
             )
             if (result.FailedFindings?.length) {
               console.error(`Failed to upload ${result.FailedCount} Findings`)
-              console.log(result.FailedFindings)
+              this.logger.info(result.FailedFindings)
             }
           } catch (error) {
             if (typeof error === 'object' && _.get(error, 'code', false) === 'NetworkingError') {
@@ -96,13 +105,13 @@ export default class HDF2ASFF extends Command {
         if (profileInfoFinding) {
           profileInfoFinding.UpdatedAt = new Date().toISOString()
           const result = await client.batchImportFindings({Findings: [profileInfoFinding as unknown] as BatchImportFindingsRequestFindingList}).promise()
-          console.info(`Statistics: ${profileInfoFinding.Description}`)
-          console.info(
+          this.logger.info(`Statistics: ${profileInfoFinding.Description}`)
+          this.logger.info(
             `Uploaded Results Set Info Finding(s) - Success: ${result.SuccessCount}, Fail: ${result.FailedCount}`,
           )
           if (result.FailedFindings?.length) {
             console.error(`Failed to upload ${result.FailedCount} Results Set Info Finding`)
-            console.log(result.FailedFindings)
+            this.logger.info(result.FailedFindings)
           }
         }
       })