Showing posts with label Elasticsearch. Show all posts
Showing posts with label Elasticsearch. Show all posts

Elasticsearch part 6 [Nested Queries]

Lets first create the mapping, here in this tutorial we are using the example of book document which will contain he nested author field.

PUT /books/
{
    "mappings": {
        "book":{
            "properties": {
                "id":{
                    "type": "integer"
                },
                "title":{
                    "type": "string"
                },
                "categories":{
                    "type": "integer"
                },
                "tag":{
                    "type": "string"
                },
                "author":{
                    "type": "nested",
                    "properties": {
                        "firstname":{
                            "type": "string"
                        },
                        "lastname":{
                            "type": "string"
                        },
                        "id":{
                            "type": "integer"
                        }
                    }
                }
            }
        }
    }    
}

Here the type of author is nested which means it acts as object.
Note:: "nested" is more advanced form of "object" which allow us to put objects in the arrays.

Lets populate some data 

 PUT /books/book/101
{
        "title" : "An article title",
        "categories" : [1,3,5,7],
        "tag" : ["elasticsearch", "symfony", "Obtao"],
        "author" : [
            {
                "firstname" : "Francois",
                "surname": "francoisg",
                "id" : 18
            },
            {
                "firstname" : "Gregory",
                "surname" : "gregquat",
                "id" : "2"
            }
        ]
}

 PUT /books/book/102
{
        "title" : "Elasticsearch",
        "categories" : [1,3,5,7],
        "tag" : ["elasticsearch", "bigdate", "Text search"],
        "author" : [
            {
                "firstname" : "yubraj",
                "surname": "pokharel",
                "id" : 18
            },
            {
                "firstname" : "dharma",
                "surname" : "kshetri",
                "id" : "2"
            }
        ]
}

Now lets say we need those books whose author name is yubraj pokharel ( thats me ;) ) so what we do now umm lets try some queries


GET /books/book/_search
{
  "query": {
    "filtered": {
      "query": {
        "match_all": {}
      },
      "filter": {
        "nested" : {
          "path" : "author",
          "filter":{
              "bool": {
                  "must": [
                     {
                         "term": {
                            "author.firstname": "yubraj"
                         }
                     },
                     {
                         "term": {
                            "author.surname": "pokharel"
                         }
                     }
                  ]
              }
          }
        }
      }
    }
  }
}


so the output will be 

{
   "took": 29,
   "timed_out": false,
   "_shards": {
      "total": 5,
      "successful": 5,
      "failed": 0
   },
   "hits": {
      "total": 1,
      "max_score": 1,
      "hits": [
         {
            "_index": "books",
            "_type": "book",
            "_id": "102",
            "_score": 1,
            "_source": {
               "title": "Elasticsearch",
               "categories": [
                  1,
                  3,
                  5,
                  7
               ],
               "tag": [
                  "elasticsearch",
                  "bigdate",
                  "Text search"
               ],
               "author": [
                  {
                     "firstname": "yubraj",
                     "surname": "pokharel",
                     "id": 18
                  },
                  {
                     "firstname": "dharma",
                     "surname": "kshetri",
                     "id": "2"
                  }
               ]
            }
         }
      ]
   }
}

 OR you can also use the query like this

 GET /books/book/_search
{
    "query": {
        "bool": {
            "must": [
               {
                "match": {
                   "title": "elasticsearch"
                }
               },
               {
                   "nested": {
                      "path": "author",
                      "query": {
                          "bool": {
                              "must": [
                                 {
                                     "term": {"author.firstname": {"value": "yubraj"}}
                                 },
                                 {
                                     "term": {"author.surname": {"value": "pokharel"}}
                                 }
                              ]
                          }
                      }
                   }
               }
            ]
        }
    }

}

ElasticSearch Part 5 [Selecting Shards Algorithm]

Let me explain this in the natural language. Lets suppose there are 10 shards across all the network, and we need to write a document in the shard. And you start to think about it like how should I select the shards to which we need to write. Don't worry it can ES can select the shards randomly. Yes you think you are smart? think again what will happen when you want to retrieve the data again do you want to use random again fuck no because it may not get the required data in the random shards we pick.

Don't worry there is a algorithm that is used by the partitioner in the ES which determines the shards where we need to write the document. For this the document id is used.

shard = hash(routing) % number_of_primary_shards

here routing value is the arbitrary string which by default is the doc_id. Lets take a example to understand it. Let us suppose we have 10 shards all across the network and we want to put a user detail in one of the shards. Let user doc_id be 100 and after hashing we get some number like 66 then

shard = 66 % 10 = 6

hence the partitioner will write this document in the shard with index 6 i.e. the 7th one. And also use the same algorithm to retrieve it. In the above process the remainder will always be in the range of 0 to number_of_primary_shards -1.

But there is some drawbacks in the scaling of ES. Users sometimes think that having a fixed number of primary shards makes it difficult to scale out an index later. In reality, there are techniques that make it easy to scale out as and when you need. For this we need to re-index all the data and there are some techniques to do it which we will discuss in later post.

ElasticSearch Part 4 [Advanced Query]

lets focus in the Query string what we can achieve and how we can perform advanced queries using the query string

1. +/- Operators: This helps us to filter the words during the search. Let us take an example:
GET /ecommerce/product/_search?q=(name:(+dell)
in the above query the name feild must contain the dell word

GET /ecommerce/product/_search?q=(name:(+dell -hp))
in the above query the name field must contain the dell word and must not contain any hp word

2. Boolean Operators:: lets say we want to search the product with
  - name as "dell"
  - must have status active i.e. 1
  - which description must contain the word "code"

so we will do it by
GET /ecommerce/product/_search?q=(name:(+dell -hp) AND status:1 AND description:+code)

ElasticSearch Part 3 [Categories of Queries]


In elasticsearch there are two main categories in queries which are :

1. Leaf: It looks for particular value in particular fields like "dell" in the product name. This queries can be used by themselves without being part of the compound queries. And the best thing is it can be used as a part of compound query too for the most advanced queries.

2. Compound: This queries wrap the leaf queries and can wrap other compound queries. It combine multiple queries in logical fashion which means as a Boolean logic. Using this we can also alter the behavior of the queries.

3. Full Text: It is used for running full text search query i.e. looking for every fields in the document. Here values are analyzed when adding and updating the document. It is analyzed like by using the stop words like "the".

4. Term Value: Used to match exact matching values. Usually used for the numbers and date rather then the text. E.g. Finding peoples who are born between 2001 and 2010. Here search queries are not analyzed before executing.

5. Joining Queries: As we know that it is very expensive to perform joining in the distributed system so elasticsearch offers two forms of joins that are designed to scale horizontally they are:

I. Nested Queries
Lets go back, where we had defined a propertise called category in the product document which contains the array of the categories. This nested queries are used in such situation where each object can be queried as a nested query as a indepedent query. 

 II. has_child and has_parent queries
has_child returns the parent document which child document match the query similarly has_parent returns the child document which parent document match the query.

6. Geo Queries
I. geo_point:: it is used for latitute/longitude pairs

 II. geo_shape:: it is used for the shapes like triangle, polygons etc.

ElasticSearch Part 2 [Searching]

Lets populate some products first: 

PUT /ecommerce/product/1002
{
    "name": "Dell",
    "price": 1200.00,
    "description": "This product is awesome and I love to code in it",
    "status": 1,
    "quantity": 2,
    "categories": [
        {"name":"dell"},
        {"name":"laptop"}
        ],
    "tags":["dell", "programming", "laptop"]
}

PUT /ecommerce/product/1003
{
    "name": "Dell",
    "price": 1200.00,
    "description": "This product is awesome and I love to design in it",
    "status": 1,
    "quantity": 2,
    "categories": [
        {"name":"dell"},
        {"name":"laptop"}
        ],
    "tags":["dell", "desigining", "laptop"]
}

In the previous post I have explained about how to create, update, read and deleting the documents in ElasticSearch. Now lets talk about Searching, Oh yeah! searching for which ElasticSearch is best about. Basically there are two ways of searching techniques which are: Query String and Query DSL. But lets talk about some basic stuff that we need to know before starting Search.

Relevancy and Scoring: To rank document for query, a score is calculated for each document that matches a query. Which means the higher the search score the document is more relevant to search query.

1. Query String: Its done using the search parameter in the URI through the rest request. It is specilly used for simple queries as well as for ad-hoc. e.g.

GET https://localhost/ecommerce/product/_serarch?q=dell
GET https://localhost/ecommerce/product/_serarch?q=name:dell &desc:"SOME_TEXT"
GET https://localhost/ecommerce/product/_serarch?q=(name:(dell OR hp) AND status:1)

Here all fields are searched for the word "dell" by default.

NOTE :: _search is a API used to perform a search process and q is the query parameter

and the output will be: 

{
   "took": 30,
   "timed_out": false,
   "_shards": {
      "total": 5,
      "successful": 5,
      "failed": 0
   },
   "hits": {
      "total": 1,
      "max_score": 0.3074455,
      "hits": [
         {
            "_index": "ecommerce",
            "_type": "product",
            "_id": "1003",
            "_score": 0.3074455,
            "_source": {
               "name": "Dell",
               "price": 1200,
               "description": "This product is awesome and I love to design in it",
               "status": 1,
               "quantity": 2,
               "categories": [
                  {
                     "name": "dell"
                  },
                  {
                     "name": "laptop"
                  }
               ],
               "tags": [
                  "dell",
                  "desigining",
                  "laptop"
               ]
            }
         }
      ]
   }
}


2. Query DSL :: In this technique the queries are defined within the request body of the JSON. It supports more feature then the query string approach. Its often easier to read and also we can perform more advanced queries. e.g.

GET /ecommerce/product/_search
{
    "query": {
        "filtered": {
           "filter": {
               "range": {
                  "price": {
                     "from": 1000,
                     "to": 2000
                  }
               }
           },
           "query": {
               "query_string": {
                  "fields": ["tags"],
                  "query": "laptop",
                  "query": "hp"
               }
           }
        }
    }
}

 In the above query it will first filter the products having the price in between 1000 to 2000 and then looks up for the words laptop and hp in the tags field.

GET /ecommerce/product/_search
{
    "query": {
        "match": {
           "name": "dell"
        }
    }
}

and the result will be

{
   "took": 664,
   "timed_out": false,
   "_shards": {
      "total": 5,
      "successful": 5,
      "failed": 0
   },
   "hits": {
      "total": 2,
      "max_score": 1.4054651,
      "hits": [
         {
            "_index": "ecommerce",
            "_type": "product",
            "_id": "1003",
            "_score": 1.4054651,
            "_source": {
               "name": "Dell",
               "price": 1200,
               "description": "This product is awesome and I love to design in it",
               "status": 1,
               "quantity": 2,
               "categories": [
                  {
                     "name": "dell"
                  },
                  {
                     "name": "laptop"
                  }
               ],
               "tags": [
                  "dell",
                  "desigining",
                  "laptop"
               ]
            }
         },
         {
            "_index": "ecommerce",
            "_type": "product",
            "_id": "1002",
            "_score": 1.4054651,
            "_source": {
               "name": "Dell",
               "price": 1200,
               "description": "This product is awesome and I love to code in it",
               "status": 1,
               "quantity": 2,
               "categories": [
                  {
                     "name": "dell"
                  },
                  {
                     "name": "laptop"
                  }
               ],
               "tags": [
                  "dell",
                  "programming",
                  "laptop"
               ]
            }
         }
      ]
   }
}

ElasticSearch Part 1 [Mappings]


For this tutorial we are using the "Sense - a Json aware interface to ElasticSearch"

 1. creating a index in elasticsearch
PUT /ecommerce {}


2. creating a mapping named product
PUT /ecommerce/
{
    "mappings": {
        "product":{
            "properties": {
                "name":{
                    "type": "string"
                },
                "price":{
                    "type": "double"
                },
                "description":{
                    "type": "string"
                },
                "quantity":{
                  "type":"string"
                },
                "status":{
                    "type": "integer"
                },
                "categories":{
                    "type": "nested",
                    "properties": {
                        "name":{
                            "type": "string"
                        }
                    }
                },
                "tags":{
                    "type": "string"
                }
            }
        }
    }
}


3. putting data in the schema with id 1001

 PUT /ecommerce/product/1001
{
    "name": "Inception Innovation Center",
    "price": 120.00,
    "description": "Welcome to heaven of code",
    "status": 1,
    "quantity": 2,
    "categories": [
        {"name":"innovation"},
        {"name":"programming"}
        ],
    "tags":["inception", "programming"]
}


4. Getting the item info having ID 1001
GET /ecommerce/product/1001
{
   "_index": "ecommerce",
   "_type": "product",
   "_id": "1001",
   "_version": 1,
   "found": true,
   "_source": {
      "name": "Inception Innovation Center",
      "price": 120,
      "description": "Welcome to heaven of code",
      "status": 1,
      "quantity": 2,
      "categories": [
         {
            "name": "innovation"
         },
         {
            "name": "programming"
         }
      ],
      "tags": [
         "inception",
         "programming"
      ]
   }
}

5. Lets say we want to update the above doc with one more category then what we need to do is 

POST /ecommerce/product/1001/_update
{
  "doc":{
         "categories": [
                {"name":"innovation"},
                {"name":"programming"},
                {"name":"inception"}
            ]
  }
}

here doc represent the the key-value pair of fields that needs to be update. Its a simple way to update and there is also another way to update for that we need to write entire properties like:

PUT /ecommerce/product/1001
{
    "name": "Inception Innovation Center",
    "price": 120.00,
    "description": "Welcome to heaven of code",
    "status": 1,
    "quantity": 2,
    "categories": [
                {"name":"innovation"},
                {"name":"programming"},
                {"name":"inception"}
            ],
    "tags":["inception", "programming"]
}

among this this I prefer the first one. The main advantage of second one is, if there is no product with id 1001 then it will create a new product with the specified id, which means there will be no data loss.
And the following is the output

{
   "_index": "ecommerce",
   "_type": "product",
   "_id": "1001",
   "_version": 2,
   "found": true,
   "_source": {
      "name": "Inception Innovation Center",
      "price": 120,
      "description": "Welcome to heaven of code",
      "status": 1,
      "quantity": 2,
      "categories": [
         {
            "name": "innovation"
         },
         {
            "name": "programming"
         },
         {
            "name": "inception"
         }
      ],
      "tags": [
         "inception",
         "programming"
      ]
   }
}

 .Here the version of the doc has been changed.