Dense vector field type

A k-nearest neighbor (kNN) search finds the k nearest vectors to a query vector, as measured by a similarity metric.

Dense vector fields can be used to rank documents in script_score queries. This lets you perform a brute-force kNN search by scanning all documents and ranking them by similarity.

In many cases, a brute-force kNN search is not efficient enough. For this reason, the dense_vector type supports indexing vectors into a specialized data structure to support fast kNN retrieval through the knn option in the search API

Unmapped array fields of float elements with size between 128 and 4096 are dynamically mapped as dense_vector with a default similariy of cosine. You can override the default similarity by explicitly mapping the field as dense_vector with the desired similarity.

Indexing is enabled by default for dense vector fields and indexed as int8_hnsw. When indexing is enabled, you can define the vector similarity to use in kNN search:

resp = client.indices.create(
    index="my-index-2",
    mappings={
        "properties": {
            "my_vector": {
                "type": "dense_vector",
                "dims": 3,
                "similarity": "dot_product"
            }
        }
    },
)
print(resp)

response = client.indices.create(
  index: 'my-index-2',
  body: {
    mappings: {
      properties: {
        my_vector: {
          type: 'dense_vector',
          dims: 3,
          similarity: 'dot_product'
        }
      }
    }
  }
)
puts response

const response = await client.indices.create({
  index: "my-index-2",
  mappings: {
    properties: {
      my_vector: {
        type: "dense_vector",
        dims: 3,
        similarity: "dot_product",
      },
    },
  },
});
console.log(response);

PUT my-index-2
{
  "mappings": {
    "properties": {
      "my_vector": {
        "type": "dense_vector",
        "dims": 3,
        "similarity": "dot_product"
      }
    }
  }
}

You can disable indexing by setting the index parameter to false:

resp = client.indices.create(
    index="my-index-2",
    mappings={
        "properties": {
            "my_vector": {
                "type": "dense_vector",
                "dims": 3,
                "index": False
            }
        }
    },
)
print(resp)

response = client.indices.create(
  index: 'my-index-2',
  body: {
    mappings: {
      properties: {
        my_vector: {
          type: 'dense_vector',
          dims: 3,
          index: false
        }
      }
    }
  }
)
puts response

const response = await client.indices.create({
  index: "my-index-2",
  mappings: {
    properties: {
      my_vector: {
        type: "dense_vector",
        dims: 3,
        index: false,
      },
    },
  },
});
console.log(response);

PUT my-index-2
{
  "mappings": {
    "properties": {
      "my_vector": {
        "type": "dense_vector",
        "dims": 3,
        "index": false
      }
    }
  }
}

Elasticsearch uses the HNSW algorithm to support efficient kNN search. Like most kNN algorithms, HNSW is an approximate method that sacrifices result accuracy for improved speed.

The dense_vector type supports quantization to reduce the memory footprint required when searching float vectors. The two following quantization strategies are supported:

+

int8 - Quantizes each dimension of the vector to 1-byte integers. This can reduce the memory footprint by 75% at the cost of some accuracy. int4 - Quantizes each dimension of the vector to half-byte integers. This can reduce the memory footprint by 87% at the cost of some accuracy.

To use a quantized index, you can set your index type to int8_hnsw or int4_hnsw. When indexing float vectors, the current default index type is int8_hnsw.

Here is an example of how to create a byte-quantized index:

resp = client.indices.create(
    index="my-byte-quantized-index",
    mappings={
        "properties": {
            "my_vector": {
                "type": "dense_vector",
                "dims": 3,
                "index": True,
                "index_options": {
                    "type": "int8_hnsw"
                }
            }
        }
    },
)
print(resp)

response = client.indices.create(
  index: 'my-byte-quantized-index',
  body: {
    mappings: {
      properties: {
        my_vector: {
          type: 'dense_vector',
          dims: 3,
          index: true,
          index_options: {
            type: 'int8_hnsw'
          }
        }
      }
    }
  }
)
puts response

const response = await client.indices.create({
  index: "my-byte-quantized-index",
  mappings: {
    properties: {
      my_vector: {
        type: "dense_vector",
        dims: 3,
        index: true,
        index_options: {
          type: "int8_hnsw",
        },
      },
    },
  },
});
console.log(response);

PUT my-byte-quantized-index
{
  "mappings": {
    "properties": {
      "my_vector": {
        "type": "dense_vector",
        "dims": 3,
        "index": true,
        "index_options": {
          "type": "int8_hnsw"
        }
      }
    }
  }
}

Here is an example of how to create a half-byte-quantized index:

resp = client.indices.create(
    index="my-byte-quantized-index",
    mappings={
        "properties": {
            "my_vector": {
                "type": "dense_vector",
                "dims": 4,
                "index": True,
                "index_options": {
                    "type": "int4_hnsw"
                }
            }
        }
    },
)
print(resp)

const response = await client.indices.create({
  index: "my-byte-quantized-index",
  mappings: {
    properties: {
      my_vector: {
        type: "dense_vector",
        dims: 4,
        index: true,
        index_options: {
          type: "int4_hnsw",
        },
      },
    },
  },
});
console.log(response);

PUT my-byte-quantized-index
{
  "mappings": {
    "properties": {
      "my_vector": {
        "type": "dense_vector",
        "dims": 4,
        "index": true,
        "index_options": {
          "type": "int4_hnsw"
        }
      }
    }
  }
}

dense_vector fields support synthetic _source .

When using element_type: bit, this will treat all vectors as bit vectors. Bit vectors utilize only a single bit per dimension and are internally encoded as bytes. This can be useful for very high-dimensional vectors or models.

When using bit, the number of dimensions must be a multiple of 8 and must represent the number of bits. Additionally, with bit vectors, the typical vector similarity values are effectively all scored the same, e.g. with hamming distance.

Let’s compare two byte[] arrays, each representing 40 individual bits.

[-127, 0, 1, 42, 127] in bits 1000000100000000000000010010101001111111 [127, -127, 0, 1, 42] in bits 0111111110000001000000000000000100101010

When comparing these two bit, vectors, we first take the hamming distance.

xor result:

1000000100000000000000010010101001111111
^
0111111110000001000000000000000100101010
=
1111111010000001000000010010101101010101

Then, we gather the count of 1 bits in the xor result: 18. To scale for scoring, we subtract from the total number of bits and divide by the total number of bits: (40 - 18) / 40 = 0.55. This would be the _score betwee these two vectors.

Here is an example of indexing and searching bit vectors:

resp = client.indices.create(
    index="my-bit-vectors",
    mappings={
        "properties": {
            "my_vector": {
                "type": "dense_vector",
                "dims": 40,
                "element_type": "bit"
            }
        }
    },
)
print(resp)

const response = await client.indices.create({
  index: "my-bit-vectors",
  mappings: {
    properties: {
      my_vector: {
        type: "dense_vector",
        dims: 40,
        element_type: "bit",
      },
    },
  },
});
console.log(response);

PUT my-bit-vectors
{
  "mappings": {
    "properties": {
      "my_vector": {
        "type": "dense_vector",
        "dims": 40, 
        "element_type": "bit"
      }
    }
  }
}

resp = client.bulk(
    index="my-bit-vectors",
    refresh=True,
    operations=[
        {
            "index": {
                "_id": "1"
            }
        },
        {
            "my_vector": [
                127,
                -127,
                0,
                1,
                42
            ]
        },
        {
            "index": {
                "_id": "2"
            }
        },
        {
            "my_vector": "8100012a7f"
        }
    ],
)
print(resp)

const response = await client.bulk({
  index: "my-bit-vectors",
  refresh: "true",
  operations: [
    {
      index: {
        _id: "1",
      },
    },
    {
      my_vector: [127, -127, 0, 1, 42],
    },
    {
      index: {
        _id: "2",
      },
    },
    {
      my_vector: "8100012a7f",
    },
  ],
});
console.log(response);

POST /my-bit-vectors/_bulk?refresh
{"index": {"_id" : "1"}}
{"my_vector": [127, -127, 0, 1, 42]} 
{"index": {"_id" : "2"}}
{"my_vector": "8100012a7f"} 

Then, when searching, you can use the knn query to search for similar bit vectors:

resp = client.search(
    index="my-bit-vectors",
    filter_path="hits.hits",
    query={
        "knn": {
            "query_vector": [
                127,
                -127,
                0,
                1,
                42
            ],
            "field": "my_vector"
        }
    },
)
print(resp)

const response = await client.search({
  index: "my-bit-vectors",
  filter_path: "hits.hits",
  query: {
    knn: {
      query_vector: [127, -127, 0, 1, 42],
      field: "my_vector",
    },
  },
});
console.log(response);

POST /my-bit-vectors/_search?filter_path=hits.hits
{
  "query": {
    "knn": {
      "query_vector": [127, -127, 0, 1, 42],
      "field": "my_vector"
    }
  }
}

{
    "hits": {
        "hits": [
            {
                "_index": "my-bit-vectors",
                "_id": "1",
                "_score": 1.0,
                "_source": {
                    "my_vector": [
                        127,
                        -127,
                        0,
                        1,
                        42
                    ]
                }
            },
            {
                "_index": "my-bit-vectors",
                "_id": "2",
                "_score": 0.55,
                "_source": {
                    "my_vector": "8100012a7f"
                }
            }
        ]
    }
}