diff --git a/opscore-utility-process.js b/opscore-utility-process.js new file mode 100644 index 0000000..08ed8ae --- /dev/null +++ b/opscore-utility-process.js @@ -0,0 +1,765 @@ +///////////////////////////////////////////////////////////////////////////////////////////////// +// Name: datastore/ui/electron/opscore-utility-process.js +// Purpose: Node utility process that does the communication with nativa C++ code +// Created: 2023/12/01 +// Author: Sergio Ferreira +// Copyright: (c) 2023-2024 ITO 33 SA +////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * Utility process where opscore native modules are loaded + * and where it provides database access that then is available to + * be executed by the renderer + * + * It acts as a JSON RPC server, from where it receives the + * intention of the client to execute a Swig exported function with + * paraneters. + * + * Execute the native function and return the result using JSON RPC + * If an exception ocurr, return it to the client using JSON RPC + * + * TODO : This should be bullet proof agains crashes. When it crash, renderer just lost the connection + */ + +const path = require('node:path'); +const jayson = require('jayson'); +const { exit } = require('node:process'); +const { parseArgs } = require('node:util'); + +/** + * Implement of the execution of a function called by the client. + * + * The call to C++ is synchronous, so we just need to worry to + * receive and return something. + * + * Jayson (JSON RPC) implementation take care of the order + * in the channel + */ +class UtilityServerStub { + mod = null; + functionCallConfig = null; + server = null; + logger = null; + + constructor() { + } + + start() { + this.parseArguments(); + this.initLog(); + this.logger.info('STARTED'); + this.handleProcessLifecycle(); + this.mod = new OpscoreNativeModules(this.logger); + this.removeSocketIfExists(); + this.server = new jayson.server(this.createFunctionCallConfig()); + this.server.tcp().listen(this.getPipePath()); + } + + /** + * Initialize the logging + */ + initLog() { + const winston = require('winston'); + if ( ! this.loglevel ) this.loglevel = "info"; + let transports; + if ( this.logFileName ) { + let logName = `/tmp/${this.logFileName}`; + if (process.platform === 'win32') + logName = `c:\\Windows\\Temp\\${logFileName}`; + transports = [ new winston.transports.File({ + filename: logName + }) ]; + } else { + transports = [ + new winston.transports.Console({ + level: this.loglevel, + handleExceptions: true, + json: false, + colorize: true + }) + ] + } + const winstonConfig = { + level: this.loglevel, + format: winston.format.json(), + transports + }; + + this.logger = winston.createLogger(winstonConfig); + } + + /** + * Parse command line arguments used to launch utility process + */ + parseArguments() { + let args = process.argv.slice(1); + if ( process.argv[0].endsWith('node') ) + args = process.argv.slice(2); + const options = { + 'loglevel': { + type: 'string', + short: 'l' + }, + 'logfilename': { + type: 'string' + } + }; + try { + const { values, positionals } = parseArgs( {args, options}); + if ( values.loglevel ) { + this.loglevel = values.loglevel + } + if ( values.logfilename ) { + this.logFileName = values.logfilename; + } + } catch(e) { + console.error(e.message); + } +} + + /** + * Add handlers that take care of process lifecycle + * including removing sockets + */ + handleProcessLifecycle() { + process.on('beforeExit', function () { + // console.log('beforeExit fired') + }) + + // This is callled if signals are received (TODO : test in windows) + process.on('exit', function () { + exitUtilityProcess('EXIT', 0, 'Exit Fired'); + }) + + // signals + process.on('SIGUSR1', function () { + exitUtilityProcess('SIGNAL', 1, 'SIGUSR'); + }) + process.on('SIGTERM', function () { + exitUtilityProcess('SIGNAL', 1, 'SIGTERM'); + }) + process.on('SIGPIPE', function () { + exitUtilityProcess('SIGNAL', 1, 'SIGTERM'); + }) + process.on('SIGHUP', function () { + exitUtilityProcess('SIGNAL', 1, 'SIGHUP'); + }) + process.on('SIGTERM', function () { + exitUtilityProcess('SIGNAL', 1, 'SIGTERM'); + }) + process.on('SIGINT', function () { + exitUtilityProcess('SIGNAL', 1, 'SIGINT'); + }) + process.on('SIGBREAK', function () { + exitUtilityProcess('SIGNAL', 1, 'SIGBREAK'); + }) + } + + /** + * Create a pipe file according to the OS + * @returns The name to be used in the pipe + */ + getPipePath() { + // return(3100) + let socketSufix = ""; + if (process.argv.length >= 3) { + socketSufix = "." + process.argv[2]; + } + if (process.platform === 'win32') + return "\\\\.\\pipe\\opscore" + socketSufix + ".gui"; + return "/tmp/opscore-gui" + socketSufix + ".socket"; + } + + removeSocketIfExists() { + const fs = require('fs'); + if ( fs.existsSync(this.getPipePath())) { + fs.unlinkSync(this.getPipePath()); + } + } + + /** + * Create the object that is passed to Jayson (JSON RPC framework) + * to execute the proper swig function and return the values through JSON RPC + * + * @returns The configuration object + */ + createFunctionCallConfig() { + const writerFunctions = Object.getOwnPropertyNames(Object.getPrototypeOf(this.mod.writer)); + writerFunctions.push(...Object.getOwnPropertyNames(Object.getPrototypeOf(Object.getPrototypeOf(this.mod.writer)))); + writerFunctions.push(...Object.getOwnPropertyNames(Object.getPrototypeOf(Object.getPrototypeOf(Object.getPrototypeOf(this.mod.writer))))) + const jsonRPCCalls = {}; + + const getFunctions = writerFunctions.filter(f => f.startsWith('Get')); + for (const getFunction of getFunctions) { + const functionKey = 'datastore.writer.' + getFunction; + const functionName = getFunction; + jsonRPCCalls[functionKey] = this.getExecutingNativeGetFunction(functionName); + } + + const loadFunctions = writerFunctions.filter(f => f.startsWith('Load')); + for (const loadFunction of loadFunctions) { + const functionKey = 'datastore.writer.' + loadFunction; + const functionName = loadFunction; + jsonRPCCalls[functionKey] = this.getExecutingNativeLoadFunction(functionName); + } + + const saveFunctions = writerFunctions.filter(f => f.startsWith('Save')); + for (const saveFunction of saveFunctions) { + const functionKey = 'datastore.writer.' + saveFunction; + const functionName = saveFunction; + jsonRPCCalls[functionKey] = this.getExecutingNativeSaveFunction(functionName); + } + + const deleteFunctions = writerFunctions.filter(f => f.startsWith('Delete')); + for (const deleteFunction of deleteFunctions) { + const functionKey = 'datastore.writer.' + deleteFunction; + const functionName = deleteFunction; + jsonRPCCalls[functionKey] = this.getExecutingNativeDeleteFunction(functionName); + } + + // Add functions + const addFunctions = writerFunctions.filter(f => f.startsWith('Add')); + for (const addFunction of addFunctions) { + const functionName = 'datastore.writer.' + addFunction; + jsonRPCCalls[functionName] = (args, callback) => { + try { + eval('this.mod.writer.' + addFunction + '("' + args.value + '")'); + callback(null, args.value); + } catch (error) { + this.logger.error(`Error executing function ${addFunction} with parameters: ${args}`); + this.logger.error(`message - ${error.message}, stack trace - ${error.stack}`); + const retException = { code: 5, message: error.message }; + callback(retException, null); + } + } + } + + // Enumerateds + jsonRPCCalls['datastore.enumerated'] = (args, callback) => { + if ( this.mod.error ) { + callback(this.mod.error, null); + } + try { + const enumValues = new SwigEnumerated(this.mod,args).getEnumValues(); + callback(null, enumValues); + } catch (error) { + callback(error, null); + } + } + + // Top level functions + const datastoreFunctions = [ "StringToExtIdsProvidersType" ]; + for (const datastoreFunction of datastoreFunctions) { + jsonRPCCalls['datastore.function.' + datastoreFunction] = (args, callback) => { + try { + const retVal = eval('this.mod.datastore.' + 'StringToExtIdsProvidersType' + '(' + this.genArgsForEval(args) + ')'); + callback(null, retVal); + } catch (error) { + const retException = { code: 6, message: error.message }; + callback(retException, null); + } + } + } + + + // XML Dump + jsonRPCCalls['datastore.XMLDumper'] = (args, callback) => { + if ( this.mod.error ) { + callback(this.mod.error, null); + } + try { + const xmlString = this.dumpXML(args); + callback(null, xmlString); + } catch (error) { + callback(error, null); + } + } + + jsonRPCCalls['PING'] = (args, callback) => { + callback(null, 'PONG'); + } + + return jsonRPCCalls; + } + + /** + * Generate a string containing the args that will be necessary + * to execute a function with eval + * + * @param {*} args The args to be generated + * @return The arguments string + */ + genArgsForEval(args) { + if ( !args ) return ""; + let evalArgs = ""; + for (const arg in args ) { + if ( evalArgs !== "") evalArgs += ','; + if (isNaN(args[arg])) { + evalArgs += '"' + args[arg] + '"'; + } else { + evalArgs += args[arg]; + } + } + return evalArgs; + } + + /** + * Execute a get function that get the list of objects + * @param {*} functionName Name of the get function + * + * @returns The objects returned by swig as array + */ + executeGet = (functionName, args) => { + const swigObjects = eval('this.mod.writer.' + functionName + '(' + this.genArgsForEval(args) + ')'); + let arrayObjects; + if (swigObjects.constructor.name.toString().endsWith('Collection')) { + arrayObjects = []; + for (let i = 0; i < swigObjects.size(); i++) { + // Create a copy of the element since swigObjects will be garbage collected + // that invalidates the C++ item in the container + arrayObjects.push(JSON.parse(JSON.stringify(swigObjects.get(i)))); + } + } else if (typeof swigObjects[Symbol.iterator] === 'function') { + arrayObjects = Array.from(swigObjects); + } else { + arrayObjects = [swigObjects]; + } + this.logger.silly(`Function ${functionName} going to return : ${JSON.stringify(arrayObjects)}`); + return arrayObjects; + } + + /** + * Execute one of the SWIG exported functions that load an object + * @param {*} functionName A string containing the name of the function + * @param {*} args Arguments used to load the objecty (normally id) + * @returns The returned values + */ + executeLoad = (functionName, args) => { + return eval('this.mod.writer.' + functionName + '(args).json()'); + } + + /** + * Execute one of the SWIG exported functions that save an object in the datastore + * @param {*} functionName A string containing the name of the function + * @param {*} args The aditional information to save. Should have the name + * of the class and the object to save in the form { className, loadFunction, objectData } + * @returns The returned values + */ + executeSave = (functionName, args) => { + let swigObject; + this.logger.debug(`Executing ${functionName} with ${JSON.stringify(args)}`); + if ( ! args.loadFunction ) { + swigObject = eval('new this.mod.datastore.' + args.className + '()'); + } else { + swigObject = eval('this.mod.writer.' + args.loadFunction + '(args.objectData.id)'); + } + swigObject.fromJson(JSON.stringify(args.objectData)); + eval('this.mod.writer.' + functionName + '(swigObject)'); + this.logger.debug(`Executed ${functionName} for object id=${swigObject.id}`); + return swigObject.id; + } + + /** + * Execute one of the SWIG exported functions that remove an object from datastore + * @param {*} functionName A string containing the name of the function + * @param {*} args The aditional information to save. Should have the name + * of the class and the object to save in the form { className, objectData } + * @returns The returned values + */ + executeDelete = (functionName, args) => { + let evalArgs; + if (isNaN(args)) { evalArgs = '"' + args + '"'; } + else { evalArgs = args; } + eval('this.mod.writer.' + functionName + '(' + evalArgs + ')'); + } + + /** + * This function returns a function that do the proper + * execution of writer Get function + * @param nativeFunctionName + */ + getExecutingNativeGetFunction = function(nativeFunctionName) { + return (args, callback) => { + if ( this.mod.error ) { + callback(this.mod.error, null); + } + try { + const objects = this.executeGet(nativeFunctionName,args); + callback(null, objects); + } catch (error) { + const retException = { code: 1, message: error.message }; + callback(retException, null); + } + } + } + + /** + * Return a reference to the function that will execute the SWIG corresponding + * load function + * + * @param {*} nativeFunctionName String containing the name of the function + * @returns The reference to the load function + */ + getExecutingNativeLoadFunction = function(nativeFunctionName) { + return (args, callback) => { + if ( this.mod.error ) { + callback(this.mod.error, null); + } + try { + const objects = this.executeLoad(nativeFunctionName, args.id); + this.logger.silly(`Function ${nativeFunctionName} going to return : ${JSON.stringify(objects)}`); + callback(null, objects); + } catch (error) { + const retException = { code: 2, message: error.message }; + this.logger.debug(`Error executing Function ${nativeFunctionName}: ${error.message}`); + callback(retException, null); + } + } + } + + /** + * Return a reference to the function that will execute the SWIG corresponding + * save function + * + * @param {*} nativeFunctionName String containing the name of the function + * @returns The reference to the load function + */ + getExecutingNativeSaveFunction = function(nativeFunctionName) { + return (args, callback) => { + if ( this.mod.error ) { + callback(this.mod.error, null); + } + try { + const objects = this.executeSave(nativeFunctionName, args); + callback(null, objects); + } catch (error) { + const retException = { code: 3, message: error.message }; + this.logger.debug(`Error executing Function ${nativeFunctionName}: ${error.message}`); + callback(retException, null); + } + } + } + + /** + * Return a reference to the function that will execute the SWIG corresponding + * save function + * + * @param {*} nativeFunctionName String containing the name of the function + * @returns The reference to the load function + */ + getExecutingNativeDeleteFunction = function(nativeFunctionName) { + return (args, callback) => { + if ( this.mod.error ) { + callback(this.mod.error, null); + } + try { + const objects = this.executeDelete(nativeFunctionName, args.id); + callback(null, objects); + } catch (error) { + const retException = { code: 4, message: error.message }; + this.logger.debug(`Error executing Function ${nativeFunctionName}: ${error.message}`); + callback(retException, null); + } + } + } + + /** + * Dump a list of object IDs to a string containing XML + * + * @param {*} args Contain an array with a list of ids and + * the name of the addXML specific function + */ + dumpXML(args) { + const xmlDumper = new this.mod.datastore.XMLDumper(this.mod.writer); + if (args.ids === undefined) { + throw new Error('Need an object id to add to XML dumper') + } + let evalArgs + args.ids.forEach((id) => { + if (isNaN(id)) { evalArgs = '"' + id + '"'; } + else { evalArgs = id; } + eval('xmlDumper.' + args.addIdFunctionName + '(' + evalArgs + ')'); + }); + return xmlDumper.GetXML(); + } + +} + +/** + * Utilities to Handle the enumerateds exported by swig + */ +class SwigEnumerated { + nativeModule = null; + typePrefix = null; + subNamespace = null; + useEnumNameAsId = false; + useEnumNameWithoutPrefixAsId = false; // Strip prefix from the description + + constructor(nativeModule, args) { + this.nativeModule = nativeModule; + this.typePrefix = args.typePrefix; + this.subNamespace = args.subNamespace; + this.useEnumNameAsId = args.useEnumNameAsId; + this.useEnumNameWithoutPrefixAsId = args.useEnumNameWithoutPrefixAsId; + } + + /** + * Obtain the enumerated values according to the prefix + */ + getEnumValues() { + const values = []; + for (const enumSwigName in this.getEnumNamespace()) { + if (!enumSwigName.startsWith(this.typePrefix)) { + continue; + } + const enumAsObj = { + id: this.getId(enumSwigName), + name: this.getEnumName(enumSwigName) + } + values.push(enumAsObj); + } + return values; + } + + /** + * Transform the string of the enum in something + * readable and mostly used in the lists + * (Can be overriden in the DAO(s)) + */ + getEnumName(swigEnumName) { + const s = swigEnumName.substring(this.typePrefix.length, swigEnumName.length); + let name = s.replace(/([a-z])([A-Z])/g, '$1 $2') + .replace(/([A-Z])([a-z])/g, ' $1$2') + .replace(/\ +/g, ' ') + .replace(/([0-9]+)([A-Z])/g, '$1 $2') + .replace(/([a-z])([0-9]+)/g, '$1 $2') + .trim(); + name = name.replace(/_/g, ' ').trim(); + return name; + } + + /** + * Some enums are defined inside classes in C++ + * To ontain it, we need to define the namespace where it exists + */ + getEnumNamespace() { + if ( this.subNamespace ) { + return this.nativeModule.datastore[this.subNamespace]; + } + return this.nativeModule.datastore; + } + + /** + * Get the unique id that will be used to uniquely identify a value + * @param propertyName Name of the property + * @returns The ID used + */ + getId(propertyName) { + if (this.objects) { + return this.objects; + } + if ( this.useEnumNameWithoutPrefixAsId ) { + return propertyName.slice(this.typePrefix.length); + } + if ( this.useEnumNameAsId ) { + return propertyName; + } + if ( this.subNamespace ) { + return this.nativeModule.datastore[this.subNamespace][propertyName]; + } + return this.nativeModule.datastore[propertyName]; + } +} + + /** + * Allows writing for/of iteration loops on top of datastore iterators like StrIdIterator + * or NumIdIterator. + * + * Example: + * for ( const country of makeIterator(reader.GetSortedCountries()) ) { + * console.log(country.Name); + * console.log(country.Id); + * } + * + * Another example running through the iterator: + * + * const iter = makeIterator(reader.GetSortedCountries()); + * let result = iter.next(); + * while ( ! result.done ) { + * console.log(result.value.Name); + * console.log(result.value.Id); + * result = iter.next(); + * } + * + * @param baseIterator iterator coming from the Reader (ie. GetSortedCountries/GetSortedCurrencies etc) + * @returns an iterator ready to use in for/of loops. + */ + function makeIterator(baseIterator) { + baseIterator.reset(); + + const iter = { + next() { + const result = baseIterator.next(); + return { value: result, done: result == null } + }, + [Symbol.iterator]() { + return this; + } + }; + return iter; + } + +/** + * Encapsulate the basic operations with opscore native modules + */ +class OpscoreNativeModules { + datastore = null; + common = null; + writer = null; + error = null; + logger = null; + iteratorGetters = [ + 'GetCountriesByCurrency', + 'GetDepositaryReceiptsByEquity', + 'GetGovernmentBillsByCountry', + 'GetGovernmentBondsByCountry', + 'GetListingsByEquity', + 'GetModelInputsFor', + 'GetRateInstruments', + 'GetUniverseByIssuer', + 'GetYieldCurvesForCurrency', + 'GetYieldCurvesForCurrencyWithResidenceType', + 'GetSortedBondWithWarrants', + 'GetSortedBonds', + 'GetSortedCBOptions', + 'GetSortedCDSs', + 'GetSortedCallWarrants', + 'GetSortedCashRates', + 'GetSortedCommonStocks', + 'GetSortedConvertibleBonds', + 'GetSortedCountries', + 'GetSortedCurrencies', + 'GetSortedCurrencyForwards', + 'GetSortedCurrencyRates', + 'GetSortedDefaultUniverseByIssuer', + 'GetSortedDeltaOneInputs', + 'GetSortedDepositaryReceipts', + 'GetSortedEDSs', + 'GetSortedEquitiesForIssuer', + 'GetSortedEquityBaskets', + 'GetSortedEquityListings', + 'GetSortedGovernmentBills', + 'GetSortedGovernmentBonds', + 'GetSortedIRSwaps', + 'GetSortedIssuers', + 'GetSortedListedRegulatoryModels', + 'GetSortedMandatories', + 'GetSortedMarkets', + 'GetSortedModelInputs', + 'GetSortedMoneyMarkets', + 'GetSortedOptions', + 'GetSortedPrivateEquities', + 'GetSortedPrivateRegulatoryModels', + 'GetSortedReferenceCDSs', + 'GetSortedReferenceIRFutures', + 'GetSortedReferenceIRSwaps', + 'GetSortedRegulatoryCapitals', + 'GetSortedUniverseByEquity', + 'GetSortedYieldCurves' + ]; + + + /** + * Inject iterators on functions the only have + * an iteration with get(idx) + */ + injectIterators() { + let proto = this.datastore['Writer'].prototype; + for (const getter in proto) { + if (this.iteratorGetters.indexOf(getter) != -1) { + const oldFn = proto[getter]; + const wrapper = function() { + return makeIterator(oldFn.apply(this, arguments)); + }; + proto[getter] = wrapper; + } + } + } + + + + /** + * Load the native modules and create a Writer Object + * + * TODO : If the native module was not loaded, some error should be + * sent to main and this one should be properly show to the user + */ + constructor(logger) { + this.logger = logger; + const modLocation = path.join(__dirname, '/../pkg/datastore.node'); + try { + this.datastore = require(modLocation); + this.logger.info(`MODULE_LOADED: ${modLocation}`); + } catch (error) { + this.logger.error(`Error loading native module at: ${modLocation}`); + this.logger.error(`message - ${error.message}, stack trace - ${error.stack}`); + exitUtilityProcess('ERROR_LOADING_MODULE', 1, error.message); + } + + try { + this.writer = new this.datastore.Writer(); + this.logger.info(`Connected to the database`) + this.injectIterators(); + } catch (error) { + const msg = 'Error connecting to database : ' + error.message; + this.logger.error(`Error connecting to the database (createe Writer()): ${modLocation}`); + this.logger.error(`message - ${error.message}, stack trace - ${error.stack}`); + exitUtilityProcess('ERROR_CONNECTING', 1, msg); + } + } +} + +/** + * Exit from the process in the most gracious maner it can + * @param {*} reason The reason why is going + * @param {*} status Exist status that should be returned by the process + */ +function exitUtilityProcess(reason, status, aditionalInfo) { + switch(reason) { + case 'ERROR_LOADING_MODULE': + console.log(reason); + if ( aditionalInfo ) + console.error(aditionalInfo); + break; + case 'ERROR_CONNECTING': + console.log(reason); + console.log(aditionalInfo); + if ( aditionalInfo ) + console.error(aditionalInfo); + break; + case 'SIGNAL': + console.log('SIGNAL ' + reason + ' RECEIVED'); + exit(status); + break; + case 'EXIT': + console.log(reason); + } + utilityServerStub.removeSocketIfExists(); + process.exitCode = status; + utilityServerStub.logger.on('finish', function () { + logger.end(); + }); +} + +// Native Module Utility Main +const utilityServerStub = new UtilityServerStub(); +try { + utilityServerStub.start(); +} catch (e) { + if (utilityServerStub.logger) { + utilityServerStub.logger.error('Error ocurred : ' + e.message); + } else { + console.error('Error ocurred : ' + e.message); + } +} +