Skip to content

Commit

Permalink
Add Builtin Entity support per Language (#29)
Browse files Browse the repository at this point in the history
  • Loading branch information
adrienball authored Jul 22, 2019
1 parent f6d8815 commit d35c24f
Show file tree
Hide file tree
Showing 16 changed files with 883 additions and 32 deletions.
6 changes: 3 additions & 3 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,15 @@ members = [

[dependencies]
failure = "0.1"
itertools = "0.7"
itertools = "0.8"
lazy_static = "1.0"
regex = "0.2"
regex = "1.1"
serde = "1.0"
serde_derive = "1.0"
serde_json = "1.0"
gazetteer-entity-parser = { git = "https://github.com/snipsco/gazetteer-entity-parser", tag = "0.7.2" }
rustling-ontology = { git = "https://github.com/snipsco/rustling-ontology", tag = "0.19.0" }
snips-nlu-ontology = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.65.0" }
snips-nlu-ontology = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.66.0" }
snips-nlu-utils = { git = "https://github.com/snipsco/snips-nlu-utils", tag = "0.8.0" }

[dev-dependencies]
Expand Down
4 changes: 2 additions & 2 deletions ffi/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ edition = "2018"
failure = "0.1"
ffi-utils = { git = "https://github.com/snipsco/snips-utils-rs", rev = "291ce1d" }
libc = "0.2"
snips-nlu-ontology = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.65.0" }
snips-nlu-ontology-ffi-macros = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.65.0" }
snips-nlu-ontology = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.66.0" }
snips-nlu-ontology-ffi-macros = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.66.0" }
snips-nlu-parsers-ffi-macros = { path = "ffi-macros" }

[lib]
Expand Down
4 changes: 2 additions & 2 deletions ffi/ffi-macros/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ ffi-utils = { git = "https://github.com/snipsco/snips-utils-rs", rev = "291ce1d"
libc = "0.2"
serde = "1.0"
serde_json = "1.0"
snips-nlu-ontology = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.65.0" }
snips-nlu-ontology-ffi-macros = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.65.0" }
snips-nlu-ontology = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.66.0" }
snips-nlu-ontology-ffi-macros = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.66.0" }
snips-nlu-parsers = { path = "../.." }

[lib]
Expand Down
56 changes: 56 additions & 0 deletions ffi/ffi-macros/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
pub use builtin_entity_parser::*;
pub use gazetteer_entity_parser::*;
pub use ontology::*;

mod builtin_entity_parser;
mod gazetteer_entity_parser;
mod ontology;

type Result<T> = ::std::result::Result<T, ::failure::Error>;

Expand Down Expand Up @@ -137,5 +139,59 @@ macro_rules! export_nlu_parsers_c_symbols {
) -> ::ffi_utils::SNIPS_RESULT {
wrap!($crate::destroy_gazetteer_entity_parser(ptr))
}

#[no_mangle]
pub extern "C" fn snips_nlu_parsers_supported_builtin_entities(
language: *const libc::c_char,
results: *mut *const ::ffi_utils::CStringArray,
) -> ::ffi_utils::SNIPS_RESULT {
wrap!($crate::get_supported_builtin_entities(language, results))
}

#[no_mangle]
pub extern "C" fn snips_nlu_parsers_supported_grammar_entities(
language: *const libc::c_char,
results: *mut *const ::ffi_utils::CStringArray,
) -> ::ffi_utils::SNIPS_RESULT {
wrap!($crate::get_supported_grammar_entities(language, results))
}

#[no_mangle]
pub extern "C" fn snips_nlu_parsers_supported_builtin_gazetteer_entities(
language: *const libc::c_char,
results: *mut *const ::ffi_utils::CStringArray,
) -> ::ffi_utils::SNIPS_RESULT {
wrap!($crate::get_supported_builtin_gazetteer_entities(
language, results
))
}

#[no_mangle]
pub extern "C" fn snips_nlu_parsers_builtin_entity_examples(
builtin_entity_kind: *const libc::c_char,
language: *const libc::c_char,
results: *mut *const ::ffi_utils::CStringArray,
) -> ::ffi_utils::SNIPS_RESULT {
wrap!($crate::get_builtin_entity_examples(
builtin_entity_kind,
language,
results
))
}

#[no_mangle]
pub extern "C" fn snips_nlu_parsers_complete_entity_ontology_json(
result: *mut *const libc::c_char,
) -> ::ffi_utils::SNIPS_RESULT {
wrap!($crate::get_complete_entity_ontology_json(result))
}

#[no_mangle]
pub extern "C" fn snips_nlu_parsers_language_entity_ontology_json(
language: *const libc::c_char,
result: *mut *const libc::c_char,
) -> ::ffi_utils::SNIPS_RESULT {
wrap!($crate::get_language_entity_ontology_json(language, result))
}
};
}
100 changes: 100 additions & 0 deletions ffi/ffi-macros/src/ontology.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
use crate::Result;
use ffi_utils::{point_to_string, CReprOf, CStringArray, RawPointerConverter};
use snips_nlu_ontology::{
BuiltinEntityKind, BuiltinGazetteerEntityKind, GrammarEntityKind, IntoBuiltinEntityKind,
Language,
};
use snips_nlu_parsers::ontology::*;
use snips_nlu_parsers::parsable::ParsableEntityKind;
use std::ffi::CStr;
use std::str::FromStr;

pub fn get_supported_builtin_entities(
language: *const libc::c_char,
results: *mut *const CStringArray,
) -> Result<()> {
let language_str = unsafe { CStr::from_ptr(language) }.to_str()?;
let language = Language::from_str(&*language_str.to_uppercase())?;
let entities = BuiltinEntityKind::all()
.iter()
.filter(|e| e.supported_languages().contains(&language))
.map(|e| e.identifier().to_string())
.collect::<Vec<_>>();
let c_entities = CStringArray::c_repr_of(entities)?.into_raw_pointer();
unsafe {
*results = c_entities;
}
Ok(())
}

pub fn get_supported_grammar_entities(
language: *const libc::c_char,
results: *mut *const CStringArray,
) -> Result<()> {
let language_str = unsafe { CStr::from_ptr(language) }.to_str()?;
let language = Language::from_str(&*language_str.to_uppercase())?;
let entities = GrammarEntityKind::all()
.iter()
.filter(|e| e.supported_languages().contains(&language))
.map(|e| e.identifier().to_string())
.collect::<Vec<_>>();
let c_entities = CStringArray::c_repr_of(entities)?.into_raw_pointer();
unsafe {
*results = c_entities;
}
Ok(())
}

pub fn get_supported_builtin_gazetteer_entities(
language: *const libc::c_char,
results: *mut *const CStringArray,
) -> Result<()> {
let language_str = unsafe { CStr::from_ptr(language) }.to_str()?;
let language = Language::from_str(&*language_str.to_uppercase())?;
let entities = BuiltinGazetteerEntityKind::all()
.iter()
.filter(|e| e.supported_languages().contains(&language))
.map(|e| e.identifier().to_string())
.collect::<Vec<_>>();
let c_entities = CStringArray::c_repr_of(entities)?.into_raw_pointer();
unsafe {
*results = c_entities;
}
Ok(())
}

pub fn get_builtin_entity_examples(
builtin_entity_kind: *const libc::c_char,
language: *const libc::c_char,
results: *mut *const CStringArray,
) -> Result<()> {
let entity_kind_str = unsafe { CStr::from_ptr(builtin_entity_kind) }.to_str()?;
let entity_kind = BuiltinEntityKind::from_identifier(&*entity_kind_str)?;
let language_str = unsafe { CStr::from_ptr(language) }.to_str()?;
let language = Language::from_str(&*language_str.to_uppercase())?;
let examples = entity_kind
.examples(language)
.into_iter()
.map(|example| example.to_string())
.collect::<Vec<_>>();
let c_examples = CStringArray::c_repr_of(examples)?.into_raw_pointer();
unsafe {
*results = c_examples;
}
Ok(())
}

pub fn get_complete_entity_ontology_json(ontology_result: *mut *const libc::c_char) -> Result<()> {
let ontology = serde_json::to_string_pretty(&complete_entity_ontology())?;
point_to_string(ontology_result, ontology)
}

pub fn get_language_entity_ontology_json(
language: *const libc::c_char,
ontology_result: *mut *const libc::c_char,
) -> Result<()> {
let language_str = unsafe { CStr::from_ptr(language) }.to_str()?;
let language = Language::from_str(&*language_str.to_uppercase())?;
let ontology = serde_json::to_string_pretty(&language_entity_ontology(language))?;
point_to_string(ontology_result, ontology)
}
4 changes: 2 additions & 2 deletions python/ffi/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,5 @@ failure = "0.1"
libc = "0.2"
ffi-utils = { git = "https://github.com/snipsco/snips-utils-rs", rev = "291ce1d" }
snips-nlu-parsers-ffi-macros = { git = "https://github.com/snipsco/snips-nlu-parsers", tag = "0.4.0-SNAPSHOT" }
snips-nlu-ontology = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.65.0" }
snips-nlu-ontology-ffi-macros = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.65.0" }
snips-nlu-ontology = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.66.0" }
snips-nlu-ontology-ffi-macros = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.66.0" }
2 changes: 1 addition & 1 deletion python/snips_nlu_parsers/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from snips_nlu_parsers.builtin_entities import (
get_all_builtin_entities, get_all_gazetteer_entities,
get_all_grammar_entities, get_all_languages, get_builtin_entity_examples,
get_builtin_entity_shortname, get_ontology_version, get_supported_entities,
get_builtin_entity_shortname, get_supported_entities,
get_supported_gazetteer_entities, get_supported_grammar_entities)
from snips_nlu_parsers.builtin_entity_parser import BuiltinEntityParser
from snips_nlu_parsers.gazetteer_entity_parser import GazetteerEntityParser
22 changes: 6 additions & 16 deletions python/snips_nlu_parsers/builtin_entities.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,20 +19,10 @@
_ALL_GAZETTEER_ENTITIES = None
_ALL_GRAMMAR_ENTITIES = None
_BUILTIN_ENTITIES_SHORTNAMES = dict()
_ONTOLOGY_VERSION = None
_COMPLETE_ENTITY_ONTOLOGY = None
_LANGUAGE_ENTITY_ONTOLOGY = dict()


def get_ontology_version():
"""Get the version of the ontology"""
global _ONTOLOGY_VERSION
if _ONTOLOGY_VERSION is None:
lib.snips_nlu_ontology_version.restype = c_char_p
_ONTOLOGY_VERSION = lib.snips_nlu_ontology_version().decode("utf8")
return _ONTOLOGY_VERSION


def get_all_languages():
"""Lists all the supported languages"""
global _ALL_LANGUAGES
Expand Down Expand Up @@ -114,7 +104,7 @@ def get_supported_entities(language):

if language not in _SUPPORTED_ENTITIES:
with string_array_pointer(pointer(CStringArray())) as ptr:
exit_code = lib.snips_nlu_ontology_supported_builtin_entities(
exit_code = lib.snips_nlu_parsers_supported_builtin_entities(
language.encode("utf8"), byref(ptr))
check_ffi_error(exit_code, "Something went wrong when retrieving "
"supported entities")
Expand All @@ -139,7 +129,7 @@ def get_supported_gazetteer_entities(language):
if language not in _SUPPORTED_GAZETTEER_ENTITIES:
with string_array_pointer(pointer(CStringArray())) as ptr:
exit_code = \
lib.snips_nlu_ontology_supported_builtin_gazetteer_entities(
lib.snips_nlu_parsers_supported_builtin_gazetteer_entities(
language.encode("utf8"), byref(ptr))
check_ffi_error(exit_code, "Something went wrong when retrieving "
"supported gazetteer entities")
Expand All @@ -163,7 +153,7 @@ def get_supported_grammar_entities(language):

if language not in _SUPPORTED_GRAMMAR_ENTITIES:
with string_array_pointer(pointer(CStringArray())) as ptr:
exit_code = lib.snips_nlu_ontology_supported_grammar_entities(
exit_code = lib.snips_nlu_parsers_supported_grammar_entities(
language.encode("utf8"), byref(ptr))
check_ffi_error(exit_code, "Something went wrong when retrieving "
"supported grammar entities")
Expand All @@ -190,7 +180,7 @@ def get_builtin_entity_examples(builtin_entity_kind, language):

if language not in _ENTITIES_EXAMPLES[builtin_entity_kind]:
with string_array_pointer(pointer(CStringArray())) as ptr:
exit_code = lib.snips_nlu_ontology_builtin_entity_examples(
exit_code = lib.snips_nlu_parsers_builtin_entity_examples(
builtin_entity_kind.encode("utf8"),
language.encode("utf8"), byref(ptr))
check_ffi_error(exit_code, "Something went wrong when retrieving "
Expand All @@ -207,7 +197,7 @@ def get_complete_entity_ontology():
global _COMPLETE_ENTITY_ONTOLOGY
if _COMPLETE_ENTITY_ONTOLOGY is None:
with string_pointer(c_char_p()) as ptr:
exit_code = lib.snips_nlu_ontology_complete_entity_ontology_json(byref(ptr))
exit_code = lib.snips_nlu_parsers_complete_entity_ontology_json(byref(ptr))
check_ffi_error(exit_code, "Something went wrong when retrieving "
"complete entity ontology")
json_str = string_at(ptr).decode("utf8")
Expand All @@ -221,7 +211,7 @@ def get_language_entity_ontology(language):
global _LANGUAGE_ENTITY_ONTOLOGY
if language not in _LANGUAGE_ENTITY_ONTOLOGY:
with string_pointer(c_char_p()) as ptr:
exit_code = lib.snips_nlu_ontology_language_entity_ontology_json(
exit_code = lib.snips_nlu_parsers_language_entity_ontology_json(
language.encode("utf8"), byref(ptr))
check_ffi_error(exit_code, "Something went wrong when retrieving "
"language entity ontology")
Expand Down
5 changes: 1 addition & 4 deletions python/snips_nlu_parsers/tests/test_builtin_entities.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from snips_nlu_parsers.builtin_entities import (
get_all_builtin_entities, get_all_gazetteer_entities,
get_all_grammar_entities, get_all_languages, get_builtin_entity_examples,
get_builtin_entity_shortname, get_ontology_version, get_supported_entities,
get_builtin_entity_shortname, get_supported_entities,
get_supported_gazetteer_entities, get_supported_grammar_entities,
get_complete_entity_ontology, get_language_entity_ontology)

Expand Down Expand Up @@ -93,9 +93,6 @@ def test_should_get_supported_grammar_entities(self):
for builtin in supported_entities:
self.assertIsInstance(builtin, str)

def test_should_get_ontology_version(self):
get_ontology_version()

def test_should_get_builtin_entity_examples(self):
for language in get_all_languages():
for builtin_entity in get_supported_entities(language):
Expand Down
Loading

0 comments on commit d35c24f

Please sign in to comment.