Skip to content

Commit 870c8c7

Browse files
ts-thomasts-thomas
committed
add examples for export/import
1 parent 41b55be commit 870c8c7

File tree

16 files changed

+700
-16
lines changed

16 files changed

+700
-16
lines changed

‎README.md‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ npm install git+https://github.com/nextapps-de/flexsearch/tree/v0.8-preview
1616
- Greatly enhanced performance of the whole text encoding pipeline
1717
- Improved indexing of numeric content (Triplets)
1818
- Intermediate result sets and `Resolver`
19-
- Basic Resolver: `and`, `or`, `xor`, `not`, `limit`, `offset`, `enrich`, `resolve`, Output formatter
19+
- Basic Resolver: `and`, `or`, `xor`, `not`, `limit`, `offset`, `boost`, `resolve`
2020
- Improved charset collection
2121
- New charset preset `soundex` which further reduces memory consumption by also increasing "fuzziness"
2222
- Performance gain when polling tasks to the index by using "Event-Loop-Caches"
@@ -81,10 +81,12 @@ All persistent variants are optimized for larger sized indexes under heavy workl
8181
- [basic-resolver](example/nodejs-commonjs/basic-resolver)
8282
- [basic-worker](example/nodejs-commonjs/basic-worker)
8383
- [basic-worker-extern-config](example/nodejs-commonjs/basic-worker-extern-config)
84+
- [basic-export-import](example/nodejs-commonjs/basic-export-import)
8485
- [document](example/nodejs-commonjs/document)
8586
- [document-persistent](example/nodejs-commonjs/document-persistent)
8687
- [document-worker](example/nodejs-commonjs/document-worker)
8788
- [document-worker-extern-config](example/nodejs-commonjs/document-worker-extern-config)
89+
- [document-export-import](example/nodejs-commonjs/document-export-import)
8890
- [language-pack](example/nodejs-commonjs/language-pack)
8991
- [nodejs-esm](example/nodejs-esm):
9092
- [basic](example/nodejs-esm/basic)
@@ -93,10 +95,12 @@ All persistent variants are optimized for larger sized indexes under heavy workl
9395
- [basic-resolver](example/nodejs-esm/basic-resolver)
9496
- [basic-worker](example/nodejs-esm/basic-worker)
9597
- [basic-worker-extern-config](example/nodejs-esm/basic-worker-extern-config)
98+
- [basic-export-import](example/nodejs-esm/basic-export-import)
9699
- [document](example/nodejs-esm/document)
97100
- [document-persistent](example/nodejs-esm/document-persistent)
98101
- [document-worker](example/nodejs-esm/document-worker)
99102
- [document-worker-extern-config](example/nodejs-esm/document-worker-extern-config)
103+
- [document-export-import](example/nodejs-esm/document-export-import)
100104
- [language-pack](example/nodejs-esm/language-pack)
101105

102106
### Examples Browser
@@ -2315,6 +2319,7 @@ for(let i = 0; i < files.length; i++){
23152319
- You can import language packs by `dist/module/lang/*` when using ESM and by `const EnglishPreset = require("flexsearch/lang/en");` when using CommonJS (Node.js)
23162320
- The method `index.append()` is now deprecated and will be removed in the near future, because it isn't consistent and leads into unexpected behavior when not used properly. You should only use `index.add()` to push contents to the index.
23172321
- Using the `async` variants like `.searchAsync` is now deprecated (but still works), asynchronous responses will always return from Worker-Index and from Persistent-Index, everything else will return a non-promised result. Having both types of methods looks like the developers can choose between them, but they can't.
2322+
- Any of your exports from versions below v0.8 are not compatible to import into v0.8
23182323

23192324
## Not finished yet
23202325

‎example/nodejs-commonjs/basic-export-import/README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
```bash
2+
npm install
3+
```
4+
5+
```bash
6+
node index.js
7+
```

‎example/nodejs-commonjs/basic-export-import/index.js‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
const { Index } = require("flexsearch");
2+
3+
(async function(){
4+
5+
// you will need to keep the index configuration
6+
// they will not export, also every change to the
7+
// configuration requires a full re-index
8+
const config = {
9+
tokenize: "forward"
10+
};
11+
12+
// create a simple index which can store id-content-pairs
13+
let index = new Index(config);
14+
15+
// some test data
16+
const data = [
17+
'cats abcd efgh ijkl mnop qrst uvwx cute',
18+
'cats abcd efgh ijkl mnop qrst cute',
19+
'cats abcd efgh ijkl mnop cute',
20+
'cats abcd efgh ijkl cute',
21+
'cats abcd efgh cute',
22+
'cats abcd cute',
23+
'cats cute'
24+
];
25+
26+
// add data to the index
27+
data.forEach((item, id) => {
28+
index.add(id, item);
29+
});
30+
31+
// perform query
32+
let result = index.search("cute cat");
33+
34+
// display results
35+
result.forEach(i => {
36+
console.log(data[i]);
37+
});
38+
39+
// EXPORT
40+
// -----------------------
41+
42+
const fs = require("fs").promises;
43+
44+
await fs.mkdir("./export/").catch(e => {});
45+
await index.export(async function(key, data){
46+
await fs.writeFile("./export/" + key, data, "utf8");
47+
});
48+
49+
// IMPORT
50+
// -----------------------
51+
52+
// create the same type of index you have used by .export()
53+
// along with the same configuration
54+
index = new Index(config);
55+
56+
// load them in parallel
57+
const files = await fs.readdir("./export/");
58+
await Promise.all(files.map(async file => {
59+
const data = await fs.readFile("./export/" + file, "utf8");
60+
await index.import(file, data);
61+
}));
62+
63+
// perform query
64+
result = index.search("cute cat");
65+
66+
// display results
67+
console.log("-------------------------------------");
68+
result.forEach(i => {
69+
console.log(data[i]);
70+
});
71+
}());

‎example/nodejs-commonjs/basic-export-import/package.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"name": "nodejs-commonjs-basic-export-import",
3+
"dependencies": {
4+
"flexsearch": "github:nextapps-de/flexsearch#v0.8-preview"
5+
}
6+
}

‎example/nodejs-commonjs/document-export-import/README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
```bash
2+
npm install
3+
```
4+
5+
```bash
6+
node index.js
7+
```

‎example/nodejs-commonjs/document-export-import/data.json‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
[
2+
{
3+
"tconst": "tt0000001",
4+
"titleType": "short",
5+
"primaryTitle": "Carmencita",
6+
"originalTitle": "Carmencita",
7+
"isAdult": 0,
8+
"startYear": "1894",
9+
"endYear": "",
10+
"runtimeMinutes": "1",
11+
"genres": [
12+
"Documentary",
13+
"Short"
14+
]
15+
},
16+
{
17+
"tconst": "tt0000002",
18+
"titleType": "short",
19+
"primaryTitle": "Le clown et ses chiens",
20+
"originalTitle": "Le clown et ses chiens",
21+
"isAdult": 0,
22+
"startYear": "1892",
23+
"endYear": "",
24+
"runtimeMinutes": "5",
25+
"genres": [
26+
"Animation",
27+
"Short"
28+
]
29+
},
30+
{
31+
"tconst": "tt0000003",
32+
"titleType": "short",
33+
"primaryTitle": "Pauvre Pierrot",
34+
"originalTitle": "Pauvre Pierrot",
35+
"isAdult": 0,
36+
"startYear": "1892",
37+
"endYear": "",
38+
"runtimeMinutes": "4",
39+
"genres": [
40+
"Animation",
41+
"Comedy",
42+
"Romance"
43+
]
44+
},
45+
{
46+
"tconst": "tt0000004",
47+
"titleType": "short",
48+
"primaryTitle": "Un bon bock",
49+
"originalTitle": "Un bon bock",
50+
"isAdult": 0,
51+
"startYear": "1892",
52+
"endYear": "",
53+
"runtimeMinutes": "12",
54+
"genres": [
55+
"Animation",
56+
"Short"
57+
]
58+
},
59+
{
60+
"tconst": "tt0000005",
61+
"titleType": "short",
62+
"primaryTitle": "Blacksmith Scene",
63+
"originalTitle": "Blacksmith Scene",
64+
"isAdult": 0,
65+
"startYear": "1893",
66+
"endYear": "",
67+
"runtimeMinutes": "1",
68+
"genres": [
69+
"Comedy",
70+
"Short"
71+
]
72+
},
73+
{
74+
"tconst": "tt0000006",
75+
"titleType": "short",
76+
"primaryTitle": "Chinese Opium Den",
77+
"originalTitle": "Chinese Opium Den",
78+
"isAdult": 0,
79+
"startYear": "1894",
80+
"endYear": "",
81+
"runtimeMinutes": "1",
82+
"genres": [
83+
"Short"
84+
]
85+
},
86+
{
87+
"tconst": "tt0000007",
88+
"titleType": "short",
89+
"primaryTitle": "Corbett and Courtney Before the Kinetograph",
90+
"originalTitle": "Corbett and Courtney Before the Kinetograph",
91+
"isAdult": 0,
92+
"startYear": "1894",
93+
"endYear": "",
94+
"runtimeMinutes": "1",
95+
"genres": [
96+
"Short",
97+
"Sport"
98+
]
99+
},
100+
{
101+
"tconst": "tt0000008",
102+
"titleType": "short",
103+
"primaryTitle": "Edison Kinetoscopic Record of a Sneeze",
104+
"originalTitle": "Edison Kinetoscopic Record of a Sneeze",
105+
"isAdult": 0,
106+
"startYear": "1894",
107+
"endYear": "",
108+
"runtimeMinutes": "1",
109+
"genres": [
110+
"Documentary",
111+
"Short"
112+
]
113+
},
114+
{
115+
"tconst": "tt0000009",
116+
"titleType": "movie",
117+
"primaryTitle": "Miss Jerry",
118+
"originalTitle": "Miss Jerry",
119+
"isAdult": 0,
120+
"startYear": "1894",
121+
"endYear": "",
122+
"runtimeMinutes": "45",
123+
"genres": [
124+
"Romance"
125+
]
126+
},
127+
{
128+
"tconst": "tt0000010",
129+
"titleType": "short",
130+
"primaryTitle": "Leaving the Factory",
131+
"originalTitle": "La sortie de l'usine Lumière à Lyon",
132+
"isAdult": 0,
133+
"startYear": "1895",
134+
"endYear": "",
135+
"runtimeMinutes": "1",
136+
"genres": [
137+
"Documentary",
138+
"Short"
139+
]
140+
}
141+
]

‎example/nodejs-commonjs/document-export-import/index.js‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
const { Document, Charset } = require("flexsearch");
2+
const fs = require("fs").promises;
3+
4+
(async function(){
5+
6+
// loading test data
7+
const data = JSON.parse(await fs.readFile(__dirname + "/data.json", "utf8"));
8+
9+
// you will need to keep the index configuration
10+
// they will not export, also every change to the
11+
// configuration requires a full re-index
12+
const config = {
13+
document: {
14+
id: "tconst",
15+
store: true,
16+
index: [{
17+
field: "primaryTitle",
18+
tokenize: "forward",
19+
encoder: Charset.LatinBalance
20+
},{
21+
field: "originalTitle",
22+
tokenize: "forward",
23+
encoder: Charset.LatinBalance
24+
}],
25+
tag: [{
26+
field: "startYear"
27+
},{
28+
field: "genres"
29+
}]
30+
}
31+
};
32+
33+
// create the document index
34+
let document = new Document(config);
35+
36+
// add test data
37+
for(let i = 0; i < data.length; i++){
38+
document.add(data[i]);
39+
}
40+
41+
// perform a query
42+
let result = document.search({
43+
query: "karmen",
44+
tag: {
45+
"startYear": "1894",
46+
"genres": [
47+
"Documentary",
48+
"Short"
49+
]
50+
},
51+
suggest: true,
52+
enrich: true,
53+
merge: true
54+
});
55+
56+
// output results
57+
console.log(result);
58+
59+
// EXPORT
60+
// -----------------------
61+
62+
await fs.mkdir("./export/").catch(e => {});
63+
await document.export(async function(key, data){
64+
await fs.writeFile("./export/" + key, data, "utf8");
65+
});
66+
67+
// IMPORT
68+
// -----------------------
69+
70+
// create the same type of index you have used by .export()
71+
// along with the same configuration
72+
document = new Document(config);
73+
74+
// load them in parallel
75+
const files = await fs.readdir("./export/");
76+
await Promise.all(files.map(async file => {
77+
const data = await fs.readFile("./export/" + file, "utf8");
78+
await document.import(file, data);
79+
}))
80+
81+
// perform query
82+
result = document.search({
83+
query: "karmen",
84+
tag: {
85+
"startYear": "1894",
86+
"genres": [
87+
"Documentary",
88+
"Short"
89+
]
90+
},
91+
suggest: true,
92+
enrich: true,
93+
merge: true
94+
});
95+
96+
// output results
97+
console.log("-------------------------------------");
98+
console.log(result);
99+
}());

‎example/nodejs-commonjs/document-export-import/package.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"name": "nodejs-commonjs-document-export-import",
3+
"dependencies": {
4+
"flexsearch": "github:nextapps-de/flexsearch#v0.8-preview"
5+
}
6+
}

0 commit comments

Comments
 (0)