bfb/examples/upload-config.yaml at master · qdrant/bfb · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
# Example bfb upload config.
#
#   bfb upload --example upload-config -n 1M -b 256 -p 16 -t 8 \
#       --uri http://localhost:6334
#
# The YAML describes the *shape* of the data; the CLI flags above control *how*
# it is uploaded (count, batch size, parallelism, threads, target).

collection:
  name: benchmark
  id: integer                 # integer | uuid
  on_disk_payload: false

  # Where the payload storage lives in RAM (Qdrant 1.19+). Supersedes
  # `on_disk_payload`, which is still sent for older servers.
  payload:
    memory: cached            # cold | cached

  hnsw:
    m: 16
    ef_construct: 100
    on_disk: true
    memory: cold              # cold | cached | pinned; supersedes `on_disk`

  optimizers:
    default_segment_number: 2
    indexing_threshold: 20000

  quantization:
    type: scalar              # none|scalar|binary|binary-2bit|binary-1.5bit|
                              # turbo-1bit|turbo-1.5bit|turbo-2bit|turbo-4bit|
                              # product-x4|x8|x16|x32|x64
    always_ram: true
    memory: pinned            # supersedes `always_ram`

  vectors:
    - name: image
      size: 1024
      distance: cosine        # cosine | dot | euclid | manhattan
      datatype: float32       # float32 | float16 | uint8 | turbo4
      memory: cached          # cold | cached (dense storage cannot be pinned)
      source: random
    - name: text-emb
      size: 384
      source: random
      # Load real vectors from an .fbin file instead. `path` may be a local path
      # or an http(s):// URL, which is downloaded once and cached under
      # $BFB_DATASETS_DIR (default ./datasets).
      # source:
      #   type: file
      #   path: /data/text-emb.fbin
      #   strategy: random-sample   # random-sample | from-start

  sparse_vectors:
    - name: bm25
      memory: pinned              # keep the inverted index resident in RAM
      modifier: idf               # none | idf; required by search `idf_corpus`
      source:
        type: random
        vocab_size: 100000
        length: 1000
        distribution: zipf        # uniform | zipf

  fields:
    - name: color
      type: keyword
      # Build the keyword index with prefix matching, so searches may use
      # `match_prefix` filters on this field.
      prefix: true
      memory: cached
      source:
        type: random
        cardinality: 100
        values_per_point: 1
    - name: price
      type: integer
      range_index: true
      source: { type: random, min: 0, max: 1000000 }
    - name: score
      type: float
      source: { type: random, min: -1.0, max: 1.0 }
    - name: location
      type: geo
      source:
        type: random-clusters
        clusters: 10
    - name: enabled
      type: bool
      source: { type: random, true_ratio: 0.7 }
    - name: created_at
      type: datetime
      is_principal: true
      source: { type: now }
    - name: abstract
      type: text
      on_disk: true
      tokenizer: word
      source:
        type: random
        vocab_size: 100000
        min_length: 10
        max_length: 100
        distribution: zipf
    # Unindexed filler payload (no field index built).
    - name: blob
      type: text
      index: false
      source: { type: random, vocab_size: 1000, min_length: 200, max_length: 512 }