Lttoolbox API

You can use lttoolbox as a library in C++ by with #include <lttoolbox/fst_processor.h>, and with a bit more work, you can also use it from Python.

Using as a library in C++

For this example you will need two files, test.dix:

<dictionary>
  <alphabet/>
  <sdefs>
    <sdef n="n"/>
    <sdef n="pl"/>
  </sdefs>
  <section id="main" type="standard">
    <e><p><l>cars</l><r>car<s n="n"/><s n="pl"/></r></p></e>
  </section>
</dictionary>

And test.cc:

#include <stdio.h>
#include <string>
#include <iostream>

#include <lttoolbox/fst_processor.h>
#include <lttoolbox/lt_locale.h>
#include <lttoolbox/ltstr.h>

using namespace std;

FSTProcessor fstp;

int main(int argc, char **argv)
{
	if(argc < 2) { 
		wcout << L"Please specify a transducer" << endl;
		exit(-1);
	}

        LtLocale::tryToSetLocale();
	FILE *t_rl = fopen(argv[1], "r");

	fstp.load(t_rl);
	fclose(t_rl);
	fstp.initBiltrans();

	wstring input = L"^car<n><pl>$";
	wstring trad = fstp.biltrans(input);

	wcout << input << L" --> " << trad << endl;

	return 0;
}

Compile both of the files:

$ lt-comp rl test.dix test_rl.bin
main@standard 6 5

$ g++ -Wall test.cc -o test -llttoolbox3

And then test:

$ ./test test_rl.bin 
^car<n><pl>$ --> ^cars$

Using as a module from Python

This example is based on how libvoikko uses lttoolbox for analysis. We make a C++ library that exposes whatever lttoolbox functions we need to Python, and a little Python class that makes calling those functions a bit more pythonic.

Write the following files:

libltpy.h:

#include <lttoolbox/fst_processor.h>

class Analyser {
	public:
		Analyser(const std::string & directoryName) throw(exception);
		wstring analyse(wstring const &word);
	private:
		FSTProcessor fst;
};

libltpy.cc:

#include "libltpy.hpp"
#include <lttoolbox/fst_processor.h>
#include <lttoolbox/lt_locale.h>

using namespace std;

Analyser::Analyser(const string & analyserpath) throw(exception) {
	FILE * file = fopen(analyserpath.c_str(), "r");
	if (!file) {
		cerr << "Couldn't open analyser file " << analyserpath << endl;
		throw exception();
	}
	fst.load(file);
	fclose(file);
	fst.setCaseSensitiveMode(false);
	fst.setDictionaryCaseMode(true);
	fst.initBiltrans();
}
    
wstring Analyser::analyse(wstring const &word) {
	pair <wstring,int> analysis = fst.biltransWithQueue(word, false);
	// The 'false' means we require no ^ or $ in input/output
	if (analysis.second == 0) {
		return analysis.first;
	}
	else {
		// a partial match:
		return L"@"+analysis.first;
	}
}

extern "C" wstring * analyse(Analyser * a, const wchar_t * word) {
	// It seems Python can only send wchar_t*, but we need a wstring
	size_t wlen = wcslen(word);
	if (wlen == 0) {
		// avoid a bug in biltransWithQueue:
		return 0;
	}
	wstring inputString = L"";
	for (size_t i = 0; i < wlen; i++) {
		inputString.append(1, word[i]);
	}
	wstring * out = new wstring(a->analyse(inputString));
	return out;
}

EXTERN void free_analyses(wstring * analyses) {
	delete analyses;
}

extern "C" Analyser * init(const char ** error, const char * path) {
	LtLocale::tryToSetLocale();
	Analyser * a = 0;
	try {
		a = new Analyser(path);
		
	}
	catch (exception & e) {
		delete a;
		a = 0;
		*error = e.what();
		return 0;
	}
	*error = 0;
	return a;
}

extern "C" void terminate(Analyser * a) {
	delete a;
}

lt.py:

# -*- coding: utf-8 -*-
from ctypes import byref, CDLL, c_char_p, c_wchar_p, c_void_p, POINTER

class FST(object):
	def __init__(self, libpath, fstpath):
		self.__lib = CDLL(libpath)

		self.__lib.init.argtypes = [POINTER(c_char_p), c_char_p]
		self.__lib.init.restype = c_void_p

		self.__lib.terminate.argtypes = [c_void_p]
		self.__lib.terminate.restype = None

		self.__lib.analyse.argtypes = [c_void_p, c_wchar_p]
		self.__lib.analyse.restype = POINTER(c_wchar_p)

		error = c_char_p()
		self.__handle = self.__lib.init(byref(error), fstpath)
		if error.value != None:
			self.__handle = 0
			raise Exception(u"Initialization of fst failed: " + unicode(error.value, "UTF-8"))
	
	def __del__(self):
		if (self.__handle != 0):
			self.__handle = 0
			class DummyLib:
				def __getattr__(obj, name):
					raise Exception("Attempt to use library after terminate() was called")
			self.__lib = DummyLib()
		
	def analyse(self, word):
		ana_p = self.__lib.analyse(self.__handle, word)
		if ana_p:
			analysis = ana_p.contents.value
		else:
			analysis = ""
		self.__lib.free_analyses(ana_p)
		return analysis

Compile the C++ files so you get a file like libltpy.so. The following commands worked for me:

g++ -DGCC_VISIBILITY=1 -g -O2 -fvisibility=hidden -I/usr/local/include/lttoolbox-3.2 -I/usr/local/lib/lttoolbox-3.2/include -c libltpy.cpp -fPIC -DPIC -o libltpy.o

g++  -fPIC -DPIC -shared  libltpy.o -Wl,-rpath -Wl,/usr/local/lib -Wl,-rpath -Wl,/usr/local/lib -L/usr/local/lib /usr/local/lib/liblttoolbox3.so   -Wl,-soname -Wl,libltpy.so.0 -o libltpy.so

However, if you use autotools, you typically just need the following in your Makefile.am:

lib_LTLIBRARIES = libltpy.la
libltpy_la_SOURCES = libltpy.cpp
libltpy_la_LDFLAGS = $(LTTOOLBOX_LIBS)

then autogen.sh and make libltpy.la (see #Real-world examples).

Then use it like:

import lt
fst = lt.FST("libltpy.so", "analyser.bin")
print fst.analyse(u"cars") # should print car<n><pl>

Real-world examples

• apertium-transfer uses lttoolbox from C++ to look up words in the bidix
  • http://apertium.svn.sourceforge.net/viewvc/apertium/trunk/apertium/apertium/transfer.cc?view=markup

• libvoikko uses lttoolbox from C++ to analyse words, and also makes a Python module:
  • http://sourceforge.net/apps/trac/voikko/browser/trunk/libvoikko/src/morphology/LttoolboxAnalyzer.cpp
  • http://sourceforge.net/apps/trac/voikko/browser/trunk/libvoikko/src/morphology/interface.cpp
  • http://sourceforge.net/apps/trac/voikko/browser/trunk/libvoikko/python/libvoikko.py