Merge pull request #87 from biothings/kg-refactor

Allow Predicates, SmartAPI, & ops to be passed in

README.md

@@ -83,7 +83,28 @@ pnpm i @biothings-explorer/smartapi-kg
       meta_kg.constructMetaKGSync();
     ```
 
-  - Option 9: Load Meta-KG with an api list
+  - Option 9: Load Meta-KG from data you specify
+
+    ```javascript
+      const smartapiSpecs = {} // This should contain your smartapi specs: could be from file read, redis, etc.
+      const predicates = [] // This should contain predicates data on TRAPI APIs (optional)
+
+      let meta_kg = new MetaKG();
+      meta_kg.constructMetaKGSync(includeReasoner = true, { predicates, smartapiSpecs });
+    ```
+
+  - Option 10: Load Meta-KG from operations you specify
+
+    ```javascript
+      const meta_kg_old = new MetaKG();
+      meta_kg_old.constructMetaKGSync();
+      const ops = meta_kg_old.ops; // ops can be transfered over a network/threads since it just stores JSON
+
+      // loading a new MetaKG from the operations
+      const meta_kg = new MetaKG(undefined, undefined, ops);
+    ```
+
+  - Option 11: Load Meta-KG with an api list
     ```javascript
     meta_kg.constructMetaKGSync(includeReasoner=true, {apiList: [
       {

src/index.ts

@@ -5,6 +5,7 @@ import { BuilderOptions, FilterCriteria } from "./types";
 import { ft } from "./filter";
 import path from "path";
 import Debug from "debug";
+import QueryOperationObject from "./parser/query_operation";
 const debug = Debug("bte:smartapi-kg:MetaKG");
 
 export * from "./types";
@@ -17,9 +18,12 @@ export default class MetaKG {
   /**
    * constructor to build meta knowledge graph from SmartAPI Specifications
    */
-  constructor(path: string = undefined, predicates_path: string = undefined) {
+  constructor(path: string = undefined, predicates_path: string = undefined, ops: SmartAPIKGOperationObject[] = []) {
     // store all meta-kg operations
-    this._ops = [];
+    ops.forEach(op => {
+      op.query_operation = QueryOperationObject.unfreeze(op.query_operation);
+    });
+    this._ops = ops;
     this.path = path;
     this.predicates_path = predicates_path;
   }
@@ -62,6 +66,23 @@ export default class MetaKG {
     return this.ops;
   }
 
+  /**
+   * Filters the whole meta kg based on apiList, teamName, tag or component
+   * @param {Object} options - filtering options
+   */
+  filterKG(options: BuilderOptions = {}) {
+    if (options.smartAPIID) {
+      this._ops = this._ops.filter(op => op.association.smartapi.id === options.smartAPIID);
+    } else if (options.teamName) {
+      this._ops = this._ops.filter(op => op.association?.["x-translator"]?.teamName === options.teamName);
+    } else if (options.tag) {
+      this._ops = this._ops.filter(op => op.tags?.includes(options.tag));
+    } else if (options.component) {
+      this._ops = this._ops.filter(op => op.association?.["x-translator"]?.component === options.component);
+    }
+  }
+
+
   /**
    * Filter the Meta-KG operations based on specific criteria
    * @param {Object} - filtering criteria, each key represents the field to be quried

src/load/all_specs_sync_loader.ts

@@ -7,14 +7,22 @@ const debug = Debug("bte:smartapi-kg:AllSpecsSyncLoader");
 
 export default class AllSpecsSyncLoader extends BaseLoader {
   private _file_path: string;
-  constructor(path: string) {
+  private _smartapiSpecs?: SmartAPISpec | SmartAPIQueryResult;
+  constructor(path: string, smartapiSpecs?: SmartAPISpec | SmartAPIQueryResult) {
     super();
     this._file_path = path;
+    this._smartapiSpecs = smartapiSpecs;
   }
   protected fetch(): SmartAPIQueryResult {
-    debug(`Fetching from file path: ${this._file_path}`);
-    const file = fs.readFileSync(this._file_path, "utf-8");
-    const data = JSON.parse(file) as SmartAPIQueryResult | SmartAPISpec;
+    let data: SmartAPIQueryResult | SmartAPISpec;
+    if (this._smartapiSpecs) {
+        data = this._smartapiSpecs;
+    } else {
+        debug(`Fetching from file path: ${this._file_path}`);
+        const file = fs.readFileSync(this._file_path, "utf-8");
+        data = JSON.parse(file) as SmartAPIQueryResult | SmartAPISpec;
+    }
+
     let result;
     if (!("hits" in data)) {
       result = {

src/load/api_list_specs_sync_loader.ts

@@ -7,8 +7,8 @@ const debug = Debug("bte:smartapi-kg:APIListSpecsSyncLoader");
 export default class APIListSpecsSyncLoader extends AllSpecsSyncLoader {
   private _apiList: apiListObject | undefined;
 
-  constructor(path: string, apiList?: apiListObject) {
-    super(path);
+  constructor(path: string, apiList?: apiListObject, smartapiSpecs?: SmartAPISpec | SmartAPIQueryResult) {
+    super(path, smartapiSpecs);
     this._apiList = apiList;
   }
 

src/load/component_specs_sync_loader.ts

@@ -5,8 +5,8 @@ import AllSpecsSyncLoader from "./all_specs_sync_loader";
 export default class ComponentSpecsSyncLoader extends AllSpecsSyncLoader {
   private _component: string;
 
-  constructor(component: string, path: string) {
-    super(path);
+  constructor(component: string, path: string, smartapiSpecs?: SmartAPISpec | SmartAPIQueryResult) {
+    super(path, smartapiSpecs);
     this._component = component;
   }
 

src/load/single_spec_sync_loader.ts

@@ -8,8 +8,8 @@ export default class SingleSpecSyncLoader extends AllSpecsSyncLoader {
   private _smartAPIID: string;
   private _apiList: apiListObject | undefined;
 
-  constructor(smartAPIID: string, path: string, apiList?: apiListObject) {
-    super(path);
+  constructor(smartAPIID: string, path: string, apiList?: apiListObject, smartapiSpecs?: SmartAPISpec | SmartAPIQueryResult) {
+    super(path, smartapiSpecs);
     this._smartAPIID = smartAPIID;
     this._apiList = apiList;
   }

src/load/sync_loader_factory.ts

@@ -5,7 +5,7 @@ import TagSpecsSyncLoader from "./tag_specs_sync_loader";
 import ComponentSpecsSyncLoader from "./component_specs_sync_loader";
 import APIListSpecsSyncLoader from "./api_list_specs_sync_loader";
 import { SmartAPISpec } from "../parser/types";
-import { apiListObject } from "../types";
+import { SmartAPIQueryResult, apiListObject } from "../types";
 import Debug from "debug";
 const debug = Debug("bte:smartapi-kg:SyncLoader");
 
@@ -16,25 +16,26 @@ export const syncLoaderFactory = (
   component: string = undefined,
   apiList: apiListObject = undefined,
   path: string,
+  smartapiSpecs?: SmartAPISpec | SmartAPIQueryResult,
 ): SmartAPISpec[] => {
   let loader;
   if (!(typeof smartAPIID === "undefined")) {
-    loader = new SingleSpecSyncLoader(smartAPIID, path, apiList);
+    loader = new SingleSpecSyncLoader(smartAPIID, path, apiList, smartapiSpecs);
     debug("Using single spec sync loader now.");
   } else if (!(typeof teamName === "undefined")) {
-    loader = new TeamSpecsSyncLoader(teamName, path, apiList);
+    loader = new TeamSpecsSyncLoader(teamName, path, apiList, smartapiSpecs);
     debug("Using team spec sync loader now.");
   } else if (!(typeof tag === "undefined")) {
-    loader = new TagSpecsSyncLoader(tag, path);
+    loader = new TagSpecsSyncLoader(tag, path, smartapiSpecs);
     debug("Using tags spec sync loader now.");
   } else if (!(typeof component === "undefined")) {
-    loader = new ComponentSpecsSyncLoader(component, path);
+    loader = new ComponentSpecsSyncLoader(component, path, smartapiSpecs);
     debug("Using component spec sync loader now.");
   } else if (!(typeof apiList === "undefined")) {
-    loader = new APIListSpecsSyncLoader(path, apiList);
+    loader = new APIListSpecsSyncLoader(path, apiList, smartapiSpecs);
     debug("Using api list spec sync loader now.");
   } else {
-    loader = new AllSpecsSyncLoader(path);
+    loader = new AllSpecsSyncLoader(path, smartapiSpecs);
     debug("Using all specs sync loader now.");
   }
   return loader.load();

src/load/tag_specs_sync_loader.ts

@@ -5,8 +5,8 @@ import AllSpecsSyncLoader from "./all_specs_sync_loader";
 export default class TagSpecsSyncLoader extends AllSpecsSyncLoader {
   private _tag: string;
 
-  constructor(tag: string, path: string) {
-    super(path);
+  constructor(tag: string, path: string, smartapiSpecs?: SmartAPISpec | SmartAPIQueryResult) {
+    super(path, smartapiSpecs);
     this._tag = tag;
   }
 

src/load/team_specs_sync_loader.ts

@@ -5,8 +5,8 @@ import APIListSpecsSyncLoader from "./api_list_specs_sync_loader";
 export default class TeamSpecsSyncLoader extends APIListSpecsSyncLoader {
   private _teamName: string;
 
-  constructor(teamName: string, path: string, apiList?: apiListObject) {
-    super(path, apiList);
+  constructor(teamName: string, path: string, apiList?: apiListObject, smartapiSpecs?: SmartAPISpec | SmartAPIQueryResult) {
+    super(path, apiList, smartapiSpecs);
     this._teamName = teamName;
   }
 

src/operations_builder/base_operations_builder.ts

@@ -18,7 +18,7 @@ export default abstract class BaseOperationsBuilder {
         const parser = new API(spec);
         if (!parser.metadata.url) throw new Error("No suitable server present");
         const ops = parser.metadata.operations;
-        allOps = [...allOps, ...ops];
+        allOps.push.apply(allOps, ops)
       } catch (err) {
         // debug(JSON.stringify(spec.paths))
         debug(`[error]: Unable to parse spec, ${spec ? spec.info.title : spec}. Error message is ${err.toString()}`);

src/operations_builder/sync_operations_builder.ts

@@ -18,6 +18,7 @@ export default class SyncOperationsBuilder extends BaseOperationsBuilder {
       this._options.component,
       this._options.apiList,
       this._file_path,
+      this._options.smartapiSpecs
     );
     return this.loadOpsFromSpecs(specs);
   }

src/operations_builder/sync_operations_builder_with_reasoner.ts

@@ -149,6 +149,10 @@ export default class SyncOperationsBuilderWithReasoner extends BaseOperationsBui
   }
 
   private fetch(): PredicatesMetadata[] {
+    if (this._options.predicates) {
+        return this._options.predicates;
+    }
+
     const file = fs.readFileSync(this._predicates_file_path, "utf-8");
     const data = JSON.parse(file) as PredicatesMetadata[];
     return data;
@@ -162,6 +166,7 @@ export default class SyncOperationsBuilderWithReasoner extends BaseOperationsBui
       this._options.component,
       this._options.apiList,
       this._file_path,
+      this._options.smartapiSpecs
     );
     const nonTRAPIOps = this.loadOpsFromSpecs(specs);
     const predicatesMetadata = this.fetch();
@@ -172,6 +177,7 @@ export default class SyncOperationsBuilderWithReasoner extends BaseOperationsBui
       undefined,
       undefined,
       this._file_path,
+      this._options.smartapiSpecs
     ).filter(
       spec =>
         "info" in spec &&
@@ -188,7 +194,7 @@ export default class SyncOperationsBuilderWithReasoner extends BaseOperationsBui
     );
     let TRAPIOps = [] as SmartAPIKGOperationObject[];
     predicatesMetadata.map(metadata => {
-      TRAPIOps = [...TRAPIOps, ...this.parsePredicateEndpoint(metadata)];
+      TRAPIOps.push.apply(TRAPIOps, this.parsePredicateEndpoint(metadata));
     });
     const returnValue = [...nonTRAPIOps, ...TRAPIOps];
     return returnValue;

src/parser/endpoint.ts

@@ -151,7 +151,7 @@ export default class Endpoint {
             operation = this.resolveRefIfProvided(rec);
             operation = Array.isArray(operation) ? operation : [operation];
             for (op of operation) {
-              res = [...res, ...this.parseIndividualOperation({ op, method, pathParams })];
+              res.push.apply(res, this.parseIndividualOperation({ op, method, pathParams }));
             }
           }
         }

src/parser/index.ts

@@ -168,7 +168,7 @@ export default class API implements APIClass {
     if ("paths" in this.smartapiDoc) {
       for (const path of Object.keys(this.smartapiDoc.paths)) {
         const ep = new Endpoint(this.smartapiDoc.paths[path], apiMeta, path);
-        ops = [...ops, ...ep.constructEndpointInfo()];
+        ops.push.apply(ops, ep.constructEndpointInfo());
       }
     }
     return ops;

src/parser/query_operation.ts

@@ -15,6 +15,24 @@ export default class QueryOperationObject implements QueryOperationInterface {
   private _pathParams: string[];
   private _templateInputs: any;
 
+  static unfreeze(obj: any) {
+    const newObj = new QueryOperationObject();
+    newObj._params = obj._params;
+    newObj._requestBody = obj._requestBody;
+    newObj._requestBodyType = obj._requestBodyType;
+    newObj._supportBatch = obj._supportBatch;
+    newObj._batchSize = obj._batchSize;
+    newObj._useTemplating = obj._useTemplating;
+    newObj._inputSeparator = obj._inputSeparator;
+    newObj._path = obj._path;
+    newObj._method = obj._method;
+    newObj._server = obj._server;
+    newObj._tags = obj._tags;
+    newObj._pathParams = obj._pathParams;
+    newObj._templateInputs = obj._templateInputs;
+    return newObj;
+  }
+
   set xBTEKGSOperation(newOp: XBTEKGSOperationObject) {
     this._params = newOp.parameters;
     this._requestBody = newOp.requestBody;

src/types.ts

@@ -22,6 +22,8 @@ export interface BuilderOptions {
   smartAPIID?: string;
   component?: string;
   apiList?: apiListObject;
+  predicates?: PredicatesMetadata[];
+  smartapiSpecs?: SmartAPISpec | SmartAPIQueryResult;
 }
 
 interface PredicatesQueryOperationInterface {