Filtering Nested Documents in Elasticsearch Aggregation

Summary

I key misunderstanding for elasticsearch beginners is to expect that query critiera from the query section of your DSL request to be applied within your aggregation automatically.

Basically you need to understand that all documents that meet your query get feed into the aggregation (not just the nested documents).

So you have to apply your filter in the aggregation section of the DSL as well.

Setup

I have a docker-compose in the following github repo with kibana and elasticsearch https://github.com/swarmee/projects/tree/master/elastic-stack-6.3

This docker-compose file includes a short lived container that loads up a nested template and some sample data.

First clone the repo with: https://github.com/swarmee/projects/

Then cd into projects/elastic-stack-6.3.

Finally you should be able to run docker-compose up to bring everything up.

This assumes you have docker and docker-compose installed and your host needs has the following setting.

sudo sysctl -w vm.max_map_count=262144

Aggregation with Filter Only in Query

GET real-estate-sales/_search
{
  "query": {
    "nested": {
      "path": "role.party.name",
      "query": {
        "match": {
          "role.party.name.fullName": "Mangos"
        }
      }
    }
  },
  "aggs": {
    "partyNesting": {
      "nested": {
        "path": "role.party"
      },
      "aggs": {
        "nameNesting": {
          "nested": {
            "path": "role.party.name"
          },
          "aggs": {
            "nameDetails": {
              "terms": {
                "field": "role.party.name.fullName.keyword",
                "size": 10
              }
            }
          }
        },
        "addressNesting": {
          "nested": {
            "path": "role.party.address"
          },
          "aggs": {
            "nameDetails": {
              "terms": {
                "field": "role.party.address.streetAddress.keyword",
                "size": 10
              }
            }
          }
        },
        "accountNesting": {
          "nested": {
            "path": "role.party.account"
          },
          "aggs": {
            "nameDetails": {
              "terms": {
                "field": "role.party.account.number",
                "size": 10
              }
            }
          }
        },
        "identificationNesting": {
          "nested": {
            "path": "role.party.identification"
          },
          "aggs": {
            "nameDetails": {
              "terms": {
                "field": "role.party.identification.identifier.keyword",
                "size": 50
              }
            }
          }
        }
      }
    }
  },
  "size": 0
}

Aggregation with Filter in both Query and Aggregation

GET /real-estate-sales/_search
{
  "query": {
    "nested": {
      "path": "role.party.name",
      "query": {
        "match": {
          "role.party.name.fullName": "Mangos"
        }
      }
    }
  },
  "aggs": {
    "partyNesting": {
      "nested": {
        "path": "role.party"
      },
      "aggs": {
        "partyFilter": {
          "filter": {
            "nested": {
              "path": "role.party.name",
              "query": {
                "match": {
                  "role.party.name.fullName": "Mangos"
                }
              }
            }
          },
          "aggs": {
            "nameNesting": {
              "nested": {
                "path": "role.party.name"
              },
              "aggs": {
                "nameDetails": {
                  "terms": {
                    "field": "role.party.name.fullName.keyword",
                    "size": 10
                  }
                }
              }
            },
            "addressNesting": {
              "nested": {
                "path": "role.party.address"
              },
              "aggs": {
                "nameDetails": {
                  "terms": {
                    "field": "role.party.address.streetAddress.keyword",
                    "size": 10
                  }
                }
              }
            },
            "accountNesting": {
              "nested": {
                "path": "role.party.account"
              },
              "aggs": {
                "nameDetails": {
                  "terms": {
                    "field": "role.party.account.number",
                    "size": 10
                  }
                }
              }
            },
            "identificationNesting": {
              "nested": {
                "path": "role.party.identification"
              },
              "aggs": {
                "nameDetails": {
                  "terms": {
                    "field": "role.party.identification.identifier.keyword",
                    "size": 50
                  }
                }
              }
            }
          }
        }
      }
    }
  },
  "size": 0
}