Integration tests for Elasticsearch query generation

Add a few tests that exercise the full query parse-generation-execution cycle to
validate the generated queries. Includes some helpers for inserting data into
Elasticsearch in order to have something to search over. The index is re-created
and the data re-indexed at the beginning of the test suite run.

To run these tests, Elasticsearch must be running.

Author: look

Gemfile

@@ -5,3 +5,4 @@ gem 'parslet'
 gem 'minitest'
 gem 'awesome_print'
 gem 'pry'
+gem 'elasticsearch'

Gemfile.lock

@@ -4,8 +4,20 @@ GEM
     awesome_print (1.7.0)
     blankslate (3.1.3)
     coderay (1.1.1)
+    elasticsearch (5.0.4)
+      elasticsearch-api (= 5.0.4)
+      elasticsearch-transport (= 5.0.4)
+    elasticsearch-api (5.0.4)
+      multi_json
+    elasticsearch-transport (5.0.4)
+      faraday
+      multi_json
+    faraday (0.12.0.1)
+      multipart-post (>= 1.2, < 3)
     method_source (0.8.2)
     minitest (5.10.1)
+    multi_json (1.12.1)
+    multipart-post (2.0.0)
     parslet (1.7.1)
       blankslate (>= 2.0, <= 4.0)
     pry (0.10.4)
@@ -19,6 +31,7 @@ PLATFORMS
 
 DEPENDENCIES
   awesome_print
+  elasticsearch
   minitest
   parslet
   pry

boolean_term_parser.rb

@@ -43,15 +43,15 @@ def to_elasticsearch
     }
 
     if should_terms.any?
-      query[:query][:boolean][:should] = should_terms.map { |term| match(term) }
+      query[:query][:bool][:should] = should_terms.map { |term| match(term) }
     end
 
     if must_terms.any?
-      query[:query][:boolean][:must] = must_terms.map { |term| match(term) }
+      query[:query][:bool][:must] = must_terms.map { |term| match(term) }
     end
 
     if must_not_terms.any?
-      query[:query][:boolean][:must_not] = must_not_terms.map { |term| match(term) }
+      query[:query][:bool][:must_not] = must_not_terms.map { |term| match(term) }
     end
 
     query

elasticsearch_helpers.rb

@@ -0,0 +1,89 @@
+require 'elasticsearch'
+
+module ElasticsearchHelpers
+  def self.client
+    @client ||= Elasticsearch::Client.new(:log => true)
+  end
+
+  def self.search(query_dsl)
+    client.search(:index => 'query_parser_test', :type => 'books', :body => query_dsl)
+  end
+
+  def self.create_index!
+    if client.indices.exists?(:index => 'query_parser_test')
+      client.indices.delete(:index => 'query_parser_test')
+    end
+
+    client.indices.create(:index => 'query_parser_test',
+                          :body => {
+                            :settings => {
+                              :index => {
+                                :mapper => {
+                                  :dynamic => false
+                                }
+                              }
+                            },
+                            :mappings => {
+                              :books => {
+                                :_all => { :enabled => false },
+                                :properties => {
+                                  :title => {
+                                    :type => 'text',
+                                    :analyzer => 'standard'
+                                  },
+                                  :author => {
+                                    :type => 'text',
+                                    :analyzer => 'standard'
+                                  },
+                                  :publication_year => {
+                                    :type => 'integer'
+                                  }
+                                }
+                              }
+                            }
+                          })
+  end
+
+  def self.prepare_corpus!
+    create_index!
+    index_documents
+    wait_for_indexing
+  end
+
+  # It takes a moment for documents to become available for search
+  def self.wait_for_indexing
+    retries = 0
+    loop do
+      results = search(:query => {:match_all => {}})
+
+      if results['hits']['hits'].size == corpus.size
+        break
+      end
+
+      if retries > 5
+        raise "Error indexing corpus. Got these results: #{results}"
+      end
+
+      retries += 1
+      sleep(0.5)
+    end
+  end
+
+  def self.corpus
+    [
+      {:title => "The Cat in the Hat", :author => ["Theodor Geisel", "Doctor Seuss"], :publication_year => 1957},
+      {:title => "Cat Sense", :author => "John Bradshaw", :publication_year => 2013},
+      {:title => "How to Tell If Your Cat Is Plotting to Kill You", :author => "Matthew Inman", :publication_year => 2012}
+    ]
+  end
+
+  def self.index_documents
+    corpus.each_with_index do |doc, index|
+      client.index(:index => 'query_parser_test',
+                   :type => 'books',
+                   :id => index,
+                   :body => doc)
+
+    end
+  end
+end

phrase_parser.rb

@@ -44,6 +44,12 @@ def initialize(operator, phrase)
 class PhraseQuery
   attr_accessor :should_clauses, :must_not_clauses, :must_clauses
 
+  def self.elasticsearch_query_for(query_string)
+    tree = PhraseParser.new.parse(query_string)
+    query = PhraseTransformer.new.apply(tree)
+    query.to_elasticsearch
+  end
+
   def initialize(clauses)
     self.should_clauses = clauses.select { |c| c.operator == :should }
     self.must_not_clauses = clauses.select { |c| c.operator == :must_not }
@@ -53,21 +59,21 @@ def initialize(clauses)
   def to_elasticsearch
     query = {
       :query => {
-        :boolean => {
+        :bool => {
         }
       }
     }
 
     if should_clauses.any?
-      query[:query][:boolean][:should] = should_clauses.map { |clause| clause_to_query(clause) }
+      query[:query][:bool][:should] = should_clauses.map { |clause| clause_to_query(clause) }
     end
 
     if must_clauses.any?
-      query[:query][:boolean][:must] = must_clauses.map { |clause| clause_to_query(clause) }
+      query[:query][:bool][:must] = must_clauses.map { |clause| clause_to_query(clause) }
     end
 
     if must_not_clauses.any?
-      query[:query][:boolean][:must_not] = must_not_clauses.map { |clause| clause_to_query(clause) }
+      query[:query][:bool][:must_not] = must_not_clauses.map { |clause| clause_to_query(clause) }
     end
 
     query

tests/integration/search_test.rb

@@ -0,0 +1,37 @@
+require 'minitest/autorun'
+require_relative '../../phrase_parser'
+require_relative '../../elasticsearch_helpers'
+
+class SearchTest < Minitest::Test
+
+  def self.configure_es
+    return if defined?(@configured)
+
+    ElasticsearchHelpers.prepare_corpus!
+
+    @configured = true
+  end
+
+  def setup
+    self.class.configure_es
+  end
+
+  def test_query
+    query_dsl = PhraseQuery.elasticsearch_query_for('kill "cat is plotting"')
+    results = ElasticsearchHelpers.search(query_dsl)
+
+    hits = results['hits']['hits']
+    assert_equal(1, hits.size)
+    assert_equal('How to Tell If Your Cat Is Plotting to Kill You', hits.first['_source']['title'])
+  end
+
+  def test_negation_query
+    query_dsl = PhraseQuery.elasticsearch_query_for('cat -hat')
+    results = ElasticsearchHelpers.search(query_dsl)
+
+    hits = results['hits']['hits']
+    assert_equal(2, hits.size)
+    titles = hits.map { |h| h['_source']['title'] }
+    refute_includes(titles, "The Cat in the Hat")
+  end
+end