Add extract method (#26)

* add extract method, tests, and example

* version bump

* export IContent type

Author: lucasjacks0n

examples/extractContent.ts

@@ -0,0 +1,56 @@
+import { IndexifyClient } from "../src";
+const fs = require("fs");
+
+// Extract wikipedia article directly from client
+(async () => {
+  // Initialize client
+  const client = await IndexifyClient.createClient();
+
+  // Read html file
+  const html = fs.readFileSync(
+    __dirname + "/../tests/files/steph_curry.html",
+    "utf8"
+  );
+
+  // Call extract method on running wikipedia extractor
+  const response = await client.extract({
+    name: "tensorlake/wikipedia",
+    content: { bytes: html, content_type: "text/plain" },
+  });
+
+  // Output preview
+  console.log(`Number of contents created ${response.content.length}`);
+  response.content.forEach((c, i) => {
+    console.log(`content ${i}`, c.content_type);
+    if (c.content_type === "text/plain" && c.bytes.length) {
+        console.log("Preview:", String.fromCharCode(...c.bytes).slice(0, 100));
+    }
+    if (c.features?.length) {
+        console.log(`Features:`, c.features);
+    }
+  });
+
+  /*
+    Output should look like this following
+
+    Number of contents created 29
+    content 0 text/plain
+    Preview: Curry is the son of Sonya and Dell Curry. He was born in Akron, Ohio, at Summa Akron City Hospital, 
+    Features: [
+    {
+        feature_type: 'metadata',
+        name: 'metadata',
+        data: { headline: 'Early life', title: 'Stephen Curry' }
+    }
+    ]
+    content 1 text/plain
+    Preview: Before Curry even played in his first game for the Wildcats, head coach Bob McKillop praised him at 
+    Features: [
+    {
+        feature_type: 'metadata',
+        name: 'metadata',
+        data: { headline: 'College career', title: 'Stephen Curry' }
+    }
+    ]
+  */
+})();

package.json

@@ -1,6 +1,6 @@
 {
   "name": "getindexify",
-  "version": "0.0.33",
+  "version": "0.0.34",
   "description": "This is the TypeScript client for interacting with the Indexify service.",
   "main": "./dist/index.js",
   "module": "./dist/index.mjs",

src/client.ts

@@ -13,6 +13,8 @@ import {
   IBaseContentMetadata,
   ISchema,
   IMtlsConfig,
+  IContent,
+  IExtractResponse,
 } from "./types";
 import { Agent } from "https";
 import { randomUUID } from "crypto";
@@ -352,6 +354,31 @@ class IndexifyClient {
     return policies;
   }
 
+  async extract({
+    name,
+    input_params,
+    content: { content_type, bytes, features = [], labels = {} },
+  }: {
+    name: string;
+    input_params?: Record<string, string | number>;
+    content: IContent;
+  }): Promise<IExtractResponse> {
+    const resp = await this.client.post(
+      `${DEFAULT_SERVICE_URL}/extractors/extract`,
+      {
+        name,
+        content: {
+          content_type,
+          bytes,
+          features,
+          labels,
+        },
+        input_params: JSON.stringify(input_params),
+      }
+    );
+    return resp.data;
+  }
+
   async ingestRemoteFile(
     url: string,
     mime_type: string,

src/index.ts

@@ -13,6 +13,7 @@ import {
   ITask,
   IDocument,
   ISchema,
+  IContent
 } from "./types";
 
 export {
@@ -30,4 +31,5 @@ export {
   ISearchIndexResponse,
   ITask,
   IDocument,
+  IContent
 };

src/types.ts

@@ -42,7 +42,6 @@ export interface IBaseContentMetadata {
   source: string;
   size: number;
 }
-
 export interface IContentMetadata extends IBaseContentMetadata {
   content_url: string;
 }
@@ -83,6 +82,31 @@ export interface IDocument {
   id?: string;
 }
 
+export interface IFeature {
+  feature_type: "embedding" | "metadata" | "unknown";
+  name: string;
+  data: { [key: string]: any };
+}
+
+export interface IContent {
+  content_type: string;
+  bytes: string | number[];
+  features?: IFeature[];
+  labels?: Record<string, string>;
+}
+
+export interface IContentResp {
+  content_type: string;
+  bytes: number[];
+  features?: IFeature[];
+  labels?: Record<string, string>;
+}
+
+export interface IExtractResponse {
+  features: IFeature[]
+  content: IContentResp[]
+}
+
 export interface ISearchIndexResponse {
   content_id: string;
   text: string;
@@ -99,3 +123,4 @@ export interface IMtlsConfig {
   keyPath: string;
   caPath?: string; // Optional, only if using a custom CA
 }
+

tests/client.test.ts

@@ -1,5 +1,7 @@
 import { IndexifyClient } from "../src";
 import { IExtractionPolicy } from "../src/types";
+import { isAxiosError } from "axios";
+const fs = require("fs");
 
 jest.setTimeout(30000);
 
@@ -173,6 +175,37 @@ test("Ingest remote url", async () => {
   );
 });
 
+test.only("Test Extract Method", async () => {
+  // Test minilm feature extract
+  const client = await IndexifyClient.createClient();
+  const res = await client.extract({
+    name: "tensorlake/minilm-l6",
+    content: { bytes: "testing", content_type: "text/plain" },
+  });
+
+  expect(res.content.length).toBe(0);
+  expect(res.features.length).toBe(1);
+
+  // Test wiki content extraction
+  const html = fs.readFileSync(__dirname + "/files/steph_curry.html", "utf8");
+  const res2 = await client.extract({
+    name: "tensorlake/wikipedia",
+    content: { bytes: html, content_type: "text/plain" },
+  });
+  expect(res2.content.length).toBe(29);
+
+  // Test eighth piece of content
+  const content = res2.content[8];
+  expect(String.fromCharCode(...content.bytes)).toBe(
+    "NCAA  Davidson College  NBA  Golden State Warriors"
+  );
+  expect(content.features?.length).toBe(1);
+  expect(content.features?.[0].feature_type).toBe("metadata");
+  expect(content.content_type).toBe("text/plain");
+  expect(content.features?.[0].data.headline).toBe("Records");
+  expect(content.features?.[0].data.title).toBe("Stephen Curry");
+});
+
 // test.only("MTLS", async () => {
 //   const fs = require("fs")
 //   const https = require("https")