From 2df8c5539d7cdab698bef1476511b68c3cc52258 Mon Sep 17 00:00:00 2001 From: Marc-Alexandre Ghaly Date: Thu, 18 May 2023 12:17:47 -0400 Subject: [PATCH] feat: Issue 754 : Json flow / split json_trait --- classes/local/step/flow_json.php | 43 ++++++ classes/local/step/json_trait.php | 207 +++++++++++++++++++++++++++++ classes/local/step/reader_json.php | 173 +----------------------- lang/en/tool_dataflows.php | 1 + lib.php | 1 + version.php | 2 +- 6 files changed, 254 insertions(+), 173 deletions(-) create mode 100644 classes/local/step/flow_json.php create mode 100644 classes/local/step/json_trait.php diff --git a/classes/local/step/flow_json.php b/classes/local/step/flow_json.php new file mode 100644 index 00000000..fc5bd513 --- /dev/null +++ b/classes/local/step/flow_json.php @@ -0,0 +1,43 @@ +. + +namespace tool_dataflows\local\step; + +use tool_dataflows\local\execution\iterators\dataflow_iterator; +use tool_dataflows\local\execution\iterators\iterator; + +/** + * + * JSON flow step type + * + * @package tool_dataflows + * @author Ghaly Marc-Alexandre + * @copyright Catalyst IT, 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class flow_json extends flow_step { + /** @var string sort order descending key */ + const DESC = 'desc'; + + /** @var string sort order ascending key */ + const ASC = 'asc'; + + use json_trait; + + public function execute($input = null) { + return $this->parse_json(); + } +} diff --git a/classes/local/step/json_trait.php b/classes/local/step/json_trait.php new file mode 100644 index 00000000..0fcd82bd --- /dev/null +++ b/classes/local/step/json_trait.php @@ -0,0 +1,207 @@ +. + +namespace tool_dataflows\local\step; + +use html_writer; +use Symfony\Component\ExpressionLanguage\ExpressionLanguage; +use tool_dataflows\helper; +use tool_dataflows\local\execution\iterators\iterator; +use tool_dataflows\local\execution\iterators\dataflow_iterator; + +/** + * JSON trait + * + * @package tool_dataflows + * @author Peter Sistrom + * @author Ghaly Marc-Alexandre + * @copyright 2023, Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +trait json_trait { + /** + * Return the definition of the fields available in this form. + * + * @return array + */ + public static function form_define_fields(): array { + return [ + 'pathtojson' => ['type' => PARAM_TEXT], + 'arrayexpression' => ['type' => PARAM_TEXT], + 'arraysortexpression' => ['type' => PARAM_TEXT], + 'sortorder' => ['type' => PARAM_TEXT], + ]; + } + + /** + * Parses json string to php array. + * + * @return mixed + * @throws \moodle_exception + */ + protected function parse_json() { + $config = $this->get_variables()->get('config'); + $jsonstring = $this->get_json_string($config->pathtojson); + + $decodedjson = json_decode($jsonstring); + if (is_null($decodedjson)) { + throw new \moodle_exception(get_string('reader_json:failed_to_decode_json', 'tool_dataflows', $config->pathtojson)); + } + + $arrayexpression = $config->arrayexpression; + $expressionlanguage = new ExpressionLanguage(); + $returnarray = $expressionlanguage->evaluate( + $arrayexpression != '' ? 'data.'.$arrayexpression : 'data', + ['data' => $decodedjson] + ); + + if (is_null($returnarray)) { + throw new \moodle_exception(get_string('reader_json:failed_to_fetch_array', + 'tool_dataflows', $config->arrayexpression)); + } + + $sortbyexpression = $config->arraysortexpression; + + // Sort the parsed array if required. + if ($sortbyexpression !== '') { + return $this->sort_by_config_value($returnarray, $sortbyexpression); + } + + return $returnarray; + } + + /** + * Parses stream to json string. + * + * @param string $path + * @return string json + * @throws \moodle_exception + */ + protected function get_json_string(string $path): string { + $jsonstring = file_get_contents($this->enginestep->engine->resolve_path($path)); + if ($jsonstring === false) { + $this->enginestep->log(error_get_last()['message']); + throw new \moodle_exception(get_string('reader_json:failed_to_open_file', 'tool_dataflows', $path)); + } + + return $jsonstring; + } + + /** + * Sort array by config value. + * + * @param array $array + * @param string $sortbyexpression + */ + public function sort_by_config_value(array $array, string $sortbyexpression): array { + $expressionlanguage = new ExpressionLanguage(); + usort($array, function ($a, $b) use ($sortbyexpression, $expressionlanguage) { + $a = $expressionlanguage->evaluate( + 'data.'.$sortbyexpression, + ['data' => $a] + ); + $b = $expressionlanguage->evaluate( + 'data.'.$sortbyexpression, + ['data' => $b] + ); + return strnatcasecmp($a, $b) * $this->get_sort_order_direction(); + }); + return $array; + } + + /** + * Converts the sort order to an int used to flip the default order + * + * This returns 1 for default sort ASC order and -1 for DESC order + * + * @return int + */ + public function get_sort_order_direction() { + $sortorder = $this->get_variables()->get('config.sortorder'); + if ($sortorder === self::DESC) { + return -1; + } + + return 1; + } + + /** + * Validate the configuration settings. + * + * @param object $config + * @return true|\lang_string[] true if valid, an array of errors otherwise + */ + public function validate_config($config) { + $errors = []; + if (!isset($config->pathtojson)) { + $errors['config_pathtojson'] = get_string('config_field_missing', 'tool_dataflows', 'pathtojson', true); + } else { + $error = helper::path_validate($config->pathtojson); + if ($error !== true) { + $errors['config_pathtojson'] = $error; + } + } + return empty($errors) ? true : $errors; + } + + /** + * Allows each step type to determine a list of optional/required form + * inputs for their configuration + * + * It's recommended you prefix the additional config related fields to avoid + * conflicts with any existing fields. + * + * @param \MoodleQuickForm $mform + */ + public function form_add_custom_inputs(\MoodleQuickForm &$mform) { + // JSON array source. + $mform->addElement('text', 'config_pathtojson', get_string('reader_json:pathtojson', 'tool_dataflows')); + $mform->addElement('static', 'config_json_path_help', '', get_string('path_help', 'tool_dataflows'). + \html_writer::nonempty_tag('pre', get_string('path_help_examples', 'tool_dataflows'))); + + // Array iterator value. + $arrayexample = (object) [ + 'data' => (object) [ + 'list' => [ + 'users' => [ + ['id' => '1', 'userdetails' => ['firstname' => 'Bob', 'lastname' => 'Smith', 'name' => 'Name1']], + ], + ], + ], + 'modified' => [1654058940], + 'errors' => [], + ]; + $jsonexample = html_writer::empty_tag('br'). + html_writer::nonempty_tag('pre', json_encode($arrayexample, JSON_PRETTY_PRINT)); + $expression = html_writer::nonempty_tag('code', 'data.list.users'); + + $mform->addElement('text', 'config_arrayexpression', get_string('reader_json:arrayexpression', 'tool_dataflows')); + $mform->addElement('static', 'config_arrayexpression_help', '', + get_string('reader_json:arrayexpression_help', 'tool_dataflows', + ['jsonexample' => $jsonexample, 'expression' => $expression])); + + // JSON array sort by. + $mform->addElement('text', 'config_arraysortexpression', get_string('reader_json:arraysortexpression', 'tool_dataflows')); + $mform->addElement('static', 'config_arraysortexpression_help', '', + get_string('reader_json:arraysortexpression_help', 'tool_dataflows', + html_writer::nonempty_tag('code', 'userdetails.firstname'))); + + // JSON array sort order (ASC, DESC). + $mform->addElement('select', 'config_sortorder', get_string('reader_json:sortorder', 'tool_dataflows'), + ['asc' => get_string(self::ASC), 'desc' => get_string(self::DESC)]); + $mform->addElement('static', 'config_sortorder_help', '', get_string('reader_json:sortorder_help', 'tool_dataflows')); + } +} diff --git a/classes/local/step/reader_json.php b/classes/local/step/reader_json.php index 4d38451d..f4c56510 100644 --- a/classes/local/step/reader_json.php +++ b/classes/local/step/reader_json.php @@ -38,19 +38,7 @@ class reader_json extends reader_step { /** @var string sort order ascending key */ const ASC = 'asc'; - /** - * Return the definition of the fields available in this form. - * - * @return array - */ - public static function form_define_fields(): array { - return [ - 'pathtojson' => ['type' => PARAM_TEXT], - 'arrayexpression' => ['type' => PARAM_TEXT], - 'arraysortexpression' => ['type' => PARAM_TEXT], - 'sortorder' => ['type' => PARAM_TEXT], - ]; - } + use json_trait; /** * Get the iterator for the step, based on configurations. @@ -61,163 +49,4 @@ public function get_iterator(): iterator { $jsonarray = $this->parse_json(); return new dataflow_iterator($this->enginestep, new \ArrayIterator($jsonarray)); } - - /** - * Parses json string to php array. - * - * @return mixed - * @throws \moodle_exception - */ - protected function parse_json() { - $config = $this->get_variables()->get('config'); - $jsonstring = $this->get_json_string($config->pathtojson); - - $decodedjson = json_decode($jsonstring); - if (is_null($decodedjson)) { - throw new \moodle_exception(get_string('reader_json:failed_to_decode_json', 'tool_dataflows', $config->pathtojson)); - } - - $arrayexpression = $config->arrayexpression; - $expressionlanguage = new ExpressionLanguage(); - $returnarray = $expressionlanguage->evaluate( - $arrayexpression != '' ? 'data.'.$arrayexpression : 'data', - ['data' => $decodedjson] - ); - - if (is_null($returnarray)) { - throw new \moodle_exception(get_string('reader_json:failed_to_fetch_array', - 'tool_dataflows', $config->arrayexpression)); - } - - $sortbyexpression = $config->arraysortexpression; - - // Sort the parsed array if required. - if ($sortbyexpression !== '') { - return $this->sort_by_config_value($returnarray, $sortbyexpression); - } - - return $returnarray; - } - - /** - * Parses stream to json string. - * - * @param string $path - * @return string json - * @throws \moodle_exception - */ - protected function get_json_string(string $path): string { - $jsonstring = file_get_contents($this->enginestep->engine->resolve_path($path)); - if ($jsonstring === false) { - $this->enginestep->log(error_get_last()['message']); - throw new \moodle_exception(get_string('reader_json:failed_to_open_file', 'tool_dataflows', $path)); - } - - return $jsonstring; - } - - /** - * Sort array by config value. - * - * @param array $array - * @param string $sortbyexpression - */ - public function sort_by_config_value(array $array, string $sortbyexpression): array { - $expressionlanguage = new ExpressionLanguage(); - usort($array, function ($a, $b) use ($sortbyexpression, $expressionlanguage) { - $a = $expressionlanguage->evaluate( - 'data.'.$sortbyexpression, - ['data' => $a] - ); - $b = $expressionlanguage->evaluate( - 'data.'.$sortbyexpression, - ['data' => $b] - ); - return strnatcasecmp($a, $b) * $this->get_sort_order_direction(); - }); - return $array; - } - - /** - * Converts the sort order to an int used to flip the default order - * - * This returns 1 for default sort ASC order and -1 for DESC order - * - * @return int - */ - public function get_sort_order_direction() { - $sortorder = $this->get_variables()->get('config.sortorder'); - if ($sortorder === self::DESC) { - return -1; - } - - return 1; - } - - /** - * Validate the configuration settings. - * - * @param object $config - * @return true|\lang_string[] true if valid, an array of errors otherwise - */ - public function validate_config($config) { - $errors = []; - if (!isset($config->pathtojson)) { - $errors['config_pathtojson'] = get_string('config_field_missing', 'tool_dataflows', 'pathtojson', true); - } else { - $error = helper::path_validate($config->pathtojson); - if ($error !== true) { - $errors['config_pathtojson'] = $error; - } - } - return empty($errors) ? true : $errors; - } - - /** - * Allows each step type to determine a list of optional/required form - * inputs for their configuration - * - * It's recommended you prefix the additional config related fields to avoid - * conflicts with any existing fields. - * - * @param \MoodleQuickForm $mform - */ - public function form_add_custom_inputs(\MoodleQuickForm &$mform) { - // JSON array source. - $mform->addElement('text', 'config_pathtojson', get_string('reader_json:pathtojson', 'tool_dataflows')); - $mform->addElement('static', 'config_json_path_help', '', get_string('path_help', 'tool_dataflows'). - \html_writer::nonempty_tag('pre', get_string('path_help_examples', 'tool_dataflows'))); - - // Array iterator value. - $arrayexample = (object) [ - 'data' => (object) [ - 'list' => [ - 'users' => [ - ['id' => '1', 'userdetails' => ['firstname' => 'Bob', 'lastname' => 'Smith', 'name' => 'Name1']], - ], - ], - ], - 'modified' => [1654058940], - 'errors' => [], - ]; - $jsonexample = html_writer::empty_tag('br'). - html_writer::nonempty_tag('pre', json_encode($arrayexample, JSON_PRETTY_PRINT)); - $expression = html_writer::nonempty_tag('code', 'data.list.users'); - - $mform->addElement('text', 'config_arrayexpression', get_string('reader_json:arrayexpression', 'tool_dataflows')); - $mform->addElement('static', 'config_arrayexpression_help', '', - get_string('reader_json:arrayexpression_help', 'tool_dataflows', - ['jsonexample' => $jsonexample, 'expression' => $expression])); - - // JSON array sort by. - $mform->addElement('text', 'config_arraysortexpression', get_string('reader_json:arraysortexpression', 'tool_dataflows')); - $mform->addElement('static', 'config_arraysortexpression_help', '', - get_string('reader_json:arraysortexpression_help', 'tool_dataflows', - html_writer::nonempty_tag('code', 'userdetails.firstname'))); - - // JSON array sort order (ASC, DESC). - $mform->addElement('select', 'config_sortorder', get_string('reader_json:sortorder', 'tool_dataflows'), - ['asc' => get_string(self::ASC), 'desc' => get_string(self::DESC)]); - $mform->addElement('static', 'config_sortorder_help', '', get_string('reader_json:sortorder_help', 'tool_dataflows')); - } } diff --git a/lang/en/tool_dataflows.php b/lang/en/tool_dataflows.php index d89df14b..562d9755 100644 --- a/lang/en/tool_dataflows.php +++ b/lang/en/tool_dataflows.php @@ -142,6 +142,7 @@ $string['step_name_flow_file_put_content'] = 'File put content'; $string['step_name_flow_gpg'] = 'GPG'; $string['step_name_flow_hash_file'] = 'Hash file'; +$string['step_name_flow_json'] = 'JSON'; $string['step_name_flow_set_variable'] = 'Set variable'; $string['step_name_flow_logic_join'] = 'Join'; $string['step_name_flow_logic_switch'] = 'Switch'; diff --git a/lib.php b/lib.php index e77d4ddf..03676c81 100644 --- a/lib.php +++ b/lib.php @@ -74,6 +74,7 @@ function tool_dataflows_step_types() { new step\flow_file_put_content, new step\flow_gpg, new step\flow_hash_file, + new step\flow_json, new step\flow_logic_join, new step\flow_logic_switch, new step\flow_noop, diff --git a/version.php b/version.php index e69c0bf7..c801ee84 100644 --- a/version.php +++ b/version.php @@ -25,7 +25,7 @@ defined('MOODLE_INTERNAL') || die(); -$plugin->version = 2023051500; +$plugin->version = 2023051800; $plugin->release = 2022102600; $plugin->requires = 2017051500; // Our lowest supported Moodle (3.3.0). $plugin->supported = [35, 401]; // Available as of Moodle 3.9.0 or later.