script_fields issue?

classic Classic list List threaded Threaded
14 messages Options
Reply | Threaded
Open this post in threaded view
|

script_fields issue?

Lukáš Vlček
Hi,

I am experiencing script_fields issue. Below is a shell script with curl examples. In short: I prepare an index, index three objects into it, verify that I can query objects from index and that I can extract values of particular "fields" (though script_fields does not require fields being stored) and then use a script_fields query which fails. Not sure what is wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : { number_of_shards : 1, number_of_replicas : 0 }}'
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" }
  }
}}'
echo 

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d '{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d '{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d '{"date":"2010-01-03","string":"three"}'
echo

echo Search
{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  }
}'
echo

The following is output:

Index setup
{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
  "foo" : {
    "bar" : {
      "_boost" : {
        "name" : "_boost"
      },
      "dynamic" : true,
      "enabled" : true,
      "date_formats" : [ "dateOptionalTime", "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd" ],
      "_source" : {
        "enabled" : true,
        "name" : "_source"
      },
      "_id" : {
        "store" : "no"
      },
      "path" : "full",
      "properties" : {
        "string" : {
          "omit_term_freq_and_positions" : false,
          "index_name" : "string",
          "index" : "not_analyzed",
          "omit_norms" : false,
          "store" : "yes",
          "boost" : 1.0,
          "term_vector" : "no",
          "type" : "string"
        },
        "date" : {
          "omit_term_freq_and_positions" : true,
          "index_name" : "date",
          "index" : "not_analyzed",
          "omit_norms" : true,
          "store" : "yes",
          "boost" : 1.0,
          "format" : "dateOptionalTime",
          "precision_step" : 4,
          "term_vector" : "no",
          "type" : "date"
        }
      },
      "_all" : {
        "enabled" : true,
        "store" : "no",
        "term_vector" : "no"
      },
      "type" : "object"
    }
  }
}
Indexing
{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ {
      "_index" : "foo",
      "_type" : "bar",
      "_id" : "1",
      "_score" : 1.0,
      "fields" : {
        "date" : "2010-01-01T00:00:00.000Z",
        "string" : "one"
      }
    } ]
  }
}
Search with script_fields

{
  "_shards" : {
    "total" : 1,
    "successful" : 0,
    "failed" : 1,
    "failures" : [ {
      "reason" : "CompileException[[Error: null]\n[Near : {... Unknown ....}]\n             ^\n[Line: 1, Column: 0]]; nested: "
    } ]
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ ]
  }
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata         ] [Starshine] [foo] deleting index
[11:45:12,818][INFO ][cluster.metadata         ] [Starshine] [foo] creating index, cause [api], shards [1]/[0], mappings []
[11:45:12,994][INFO ][cluster.metadata         ] [Starshine] [foo] create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type       ] [Starshine] [13] Failed to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
             ^
[Line: 1, Column: 0]
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more


Should I open an issue?

Regards,
Lukas
Reply | Threaded
Open this post in threaded view
|

Re: script_fields issue?

kimchy
Administrator
Hi,

  I think the problem is with curl, and the fact that within the body of the request the ' is used (within the script). It needs to be escaped somehow (not sure how).

-shay.banon

On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček <[hidden email]> wrote:
Hi,

I am experiencing script_fields issue. Below is a shell script with curl examples. In short: I prepare an index, index three objects into it, verify that I can query objects from index and that I can extract values of particular "fields" (though script_fields does not require fields being stored) and then use a script_fields query which fails. Not sure what is wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : { number_of_shards : 1, number_of_replicas : 0 }}'
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" }
  }
}}'
echo 

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d '{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d '{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d '{"date":"2010-01-03","string":"three"}'
echo

echo Search
{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  }
}'
echo

The following is output:

Index setup
{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
  "foo" : {
    "bar" : {
      "_boost" : {
        "name" : "_boost"
      },
      "dynamic" : true,
      "enabled" : true,
      "date_formats" : [ "dateOptionalTime", "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd" ],
      "_source" : {
        "enabled" : true,
        "name" : "_source"
      },
      "_id" : {
        "store" : "no"
      },
      "path" : "full",
      "properties" : {
        "string" : {
          "omit_term_freq_and_positions" : false,
          "index_name" : "string",
          "index" : "not_analyzed",
          "omit_norms" : false,
          "store" : "yes",
          "boost" : 1.0,
          "term_vector" : "no",
          "type" : "string"
        },
        "date" : {
          "omit_term_freq_and_positions" : true,
          "index_name" : "date",
          "index" : "not_analyzed",
          "omit_norms" : true,
          "store" : "yes",
          "boost" : 1.0,
          "format" : "dateOptionalTime",
          "precision_step" : 4,
          "term_vector" : "no",
          "type" : "date"
        }
      },
      "_all" : {
        "enabled" : true,
        "store" : "no",
        "term_vector" : "no"
      },
      "type" : "object"
    }
  }
}
Indexing
{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ {
      "_index" : "foo",
      "_type" : "bar",
      "_id" : "1",
      "_score" : 1.0,
      "fields" : {
        "date" : "2010-01-01T00:00:00.000Z",
        "string" : "one"
      }
    } ]
  }
}
Search with script_fields

{
  "_shards" : {
    "total" : 1,
    "successful" : 0,
    "failed" : 1,
    "failures" : [ {
      "reason" : "CompileException[[Error: null]\n[Near : {... Unknown ....}]\n             ^\n[Line: 1, Column: 0]]; nested: "
    } ]
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ ]
  }
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata         ] [Starshine] [foo] deleting index
[11:45:12,818][INFO ][cluster.metadata         ] [Starshine] [foo] creating index, cause [api], shards [1]/[0], mappings []
[11:45:12,994][INFO ][cluster.metadata         ] [Starshine] [foo] create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type       ] [Starshine] [13] Failed to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
             ^
[Line: 1, Column: 0]
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more


Should I open an issue?

Regards,
Lukas

Reply | Threaded
Open this post in threaded view
|

Re: script_fields issue?

Lukáš Vlček
Oh yeah, that's it.
*BUT*, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d @script_fields.json

I put the following query into script_fields.json file:
{
  "from" : 0, "size" : 3,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  },
  "sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by them (note the "sort" part) it fails:
{
  "error" : "SearchPhaseExecutionException[Failed to execute phase [query], total failure; shardFailures {[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  \"from\" : 0, \"size\" : 3,  \"fields\" : [\"date\",\"string\"],  \"query\" : { \"match_all\" : { } },  \"script_fields\" : {     \"test1\" : { \"script\" : \"doc['date'].date.millis\" },    \"test2\" : { \"script\" : \"doc['string'].stringValue\" }  },  \"sort\" : [\"test1\"]}]]]; nested: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type       ] [Starshine] [foo][0], node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.search.SearchRequest@406039a4]
org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  "from" : 0, "size" : 3,  "fields" : ["date","string"],  "query" : { "match_all" : { } },  "script_fields" : {     "test1" : { "script" : "doc['date'].date.millis" },    "test2" : { "script" : "doc['string'].stringValue" }  },  "sort" : ["test1"]}]]
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]
at org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on scripted field?

Regards,
Lukas


On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon <[hidden email]> wrote:
Hi,

  I think the problem is with curl, and the fact that within the body of the request the ' is used (within the script). It needs to be escaped somehow (not sure how).

-shay.banon


On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček <[hidden email]> wrote:
Hi,

I am experiencing script_fields issue. Below is a shell script with curl examples. In short: I prepare an index, index three objects into it, verify that I can query objects from index and that I can extract values of particular "fields" (though script_fields does not require fields being stored) and then use a script_fields query which fails. Not sure what is wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : { number_of_shards : 1, number_of_replicas : 0 }}'
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" }
  }
}}'
echo 

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d '{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d '{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d '{"date":"2010-01-03","string":"three"}'
echo

echo Search
{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  }
}'
echo

The following is output:

Index setup
{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
  "foo" : {
    "bar" : {
      "_boost" : {
        "name" : "_boost"
      },
      "dynamic" : true,
      "enabled" : true,
      "date_formats" : [ "dateOptionalTime", "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd" ],
      "_source" : {
        "enabled" : true,
        "name" : "_source"
      },
      "_id" : {
        "store" : "no"
      },
      "path" : "full",
      "properties" : {
        "string" : {
          "omit_term_freq_and_positions" : false,
          "index_name" : "string",
          "index" : "not_analyzed",
          "omit_norms" : false,
          "store" : "yes",
          "boost" : 1.0,
          "term_vector" : "no",
          "type" : "string"
        },
        "date" : {
          "omit_term_freq_and_positions" : true,
          "index_name" : "date",
          "index" : "not_analyzed",
          "omit_norms" : true,
          "store" : "yes",
          "boost" : 1.0,
          "format" : "dateOptionalTime",
          "precision_step" : 4,
          "term_vector" : "no",
          "type" : "date"
        }
      },
      "_all" : {
        "enabled" : true,
        "store" : "no",
        "term_vector" : "no"
      },
      "type" : "object"
    }
  }
}
Indexing
{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ {
      "_index" : "foo",
      "_type" : "bar",
      "_id" : "1",
      "_score" : 1.0,
      "fields" : {
        "date" : "2010-01-01T00:00:00.000Z",
        "string" : "one"
      }
    } ]
  }
}
Search with script_fields

{
  "_shards" : {
    "total" : 1,
    "successful" : 0,
    "failed" : 1,
    "failures" : [ {
      "reason" : "CompileException[[Error: null]\n[Near : {... Unknown ....}]\n             ^\n[Line: 1, Column: 0]]; nested: "
    } ]
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ ]
  }
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata         ] [Starshine] [foo] deleting index
[11:45:12,818][INFO ][cluster.metadata         ] [Starshine] [foo] creating index, cause [api], shards [1]/[0], mappings []
[11:45:12,994][INFO ][cluster.metadata         ] [Starshine] [foo] create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type       ] [Starshine] [13] Failed to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
             ^
[Line: 1, Column: 0]
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more


Should I open an issue?

Regards,
Lukas


Reply | Threaded
Open this post in threaded view
|

Re: script_fields issue?

Lukáš Vlček
So, I found that the SortField needs field type here: http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118

That is why it is looking for mappings.

OK, I tried to provide mapping for script_fields:
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" },
    "test1" : { "type" : "long" },
    "test2" : { "type" : "string" }
  }
}}'

Now, "sort" : ["test1"] (test1 is date converted to millis) works (note: I was not able to make it reverse:true or desc) but "sort" : ["test2"] still fails with simple message:
{
  "error" : "ReduceSearchPhaseException[Failed to execute phase [query], [reduce] ]; nested: "
}
(no error in server log).

So, sorting on script_fields does not seem to work without issues to me. Second issue is that I do not want to provide explicit mapping in advance (fixing it to specific type) for script_fields. I think this is little against ES approach because script_fields can be defined per query.

Would it be possible to ask mvel for return type for given script in order to determine type for sort function on the fly?

Or how about if the client can provide the type directly as part of script_fields definiton per query. Something like:
"script_fields" : {
    "test1" : {
        "script" : "_some_fancy_math_",
        "params" : { ... },
        "return_type" : "long"
    }
}

Regards,
Lukas



On Thu, Jul 29, 2010 at 3:33 PM, Lukáš Vlček <[hidden email]> wrote:
Oh yeah, that's it.
*BUT*, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d @script_fields.json

I put the following query into script_fields.json file:
{
  "from" : 0, "size" : 3,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  },
  "sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by them (note the "sort" part) it fails:
{
  "error" : "SearchPhaseExecutionException[Failed to execute phase [query], total failure; shardFailures {[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  \"from\" : 0, \"size\" : 3,  \"fields\" : [\"date\",\"string\"],  \"query\" : { \"match_all\" : { } },  \"script_fields\" : {     \"test1\" : { \"script\" : \"doc['date'].date.millis\" },    \"test2\" : { \"script\" : \"doc['string'].stringValue\" }  },  \"sort\" : [\"test1\"]}]]]; nested: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type       ] [Starshine] [foo][0], node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.search.SearchRequest@406039a4]
org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  "from" : 0, "size" : 3,  "fields" : ["date","string"],  "query" : { "match_all" : { } },  "script_fields" : {     "test1" : { "script" : "doc['date'].date.millis" },    "test2" : { "script" : "doc['string'].stringValue" }  },  "sort" : ["test1"]}]]
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]
at org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on scripted field?

Regards,
Lukas


On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon <[hidden email]> wrote:
Hi,

  I think the problem is with curl, and the fact that within the body of the request the ' is used (within the script). It needs to be escaped somehow (not sure how).

-shay.banon


On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček <[hidden email]> wrote:
Hi,

I am experiencing script_fields issue. Below is a shell script with curl examples. In short: I prepare an index, index three objects into it, verify that I can query objects from index and that I can extract values of particular "fields" (though script_fields does not require fields being stored) and then use a script_fields query which fails. Not sure what is wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : { number_of_shards : 1, number_of_replicas : 0 }}'
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" }
  }
}}'
echo 

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d '{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d '{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d '{"date":"2010-01-03","string":"three"}'
echo

echo Search
{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  }
}'
echo

The following is output:

Index setup
{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
  "foo" : {
    "bar" : {
      "_boost" : {
        "name" : "_boost"
      },
      "dynamic" : true,
      "enabled" : true,
      "date_formats" : [ "dateOptionalTime", "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd" ],
      "_source" : {
        "enabled" : true,
        "name" : "_source"
      },
      "_id" : {
        "store" : "no"
      },
      "path" : "full",
      "properties" : {
        "string" : {
          "omit_term_freq_and_positions" : false,
          "index_name" : "string",
          "index" : "not_analyzed",
          "omit_norms" : false,
          "store" : "yes",
          "boost" : 1.0,
          "term_vector" : "no",
          "type" : "string"
        },
        "date" : {
          "omit_term_freq_and_positions" : true,
          "index_name" : "date",
          "index" : "not_analyzed",
          "omit_norms" : true,
          "store" : "yes",
          "boost" : 1.0,
          "format" : "dateOptionalTime",
          "precision_step" : 4,
          "term_vector" : "no",
          "type" : "date"
        }
      },
      "_all" : {
        "enabled" : true,
        "store" : "no",
        "term_vector" : "no"
      },
      "type" : "object"
    }
  }
}
Indexing
{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ {
      "_index" : "foo",
      "_type" : "bar",
      "_id" : "1",
      "_score" : 1.0,
      "fields" : {
        "date" : "2010-01-01T00:00:00.000Z",
        "string" : "one"
      }
    } ]
  }
}
Search with script_fields

{
  "_shards" : {
    "total" : 1,
    "successful" : 0,
    "failed" : 1,
    "failures" : [ {
      "reason" : "CompileException[[Error: null]\n[Near : {... Unknown ....}]\n             ^\n[Line: 1, Column: 0]]; nested: "
    } ]
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ ]
  }
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata         ] [Starshine] [foo] deleting index
[11:45:12,818][INFO ][cluster.metadata         ] [Starshine] [foo] creating index, cause [api], shards [1]/[0], mappings []
[11:45:12,994][INFO ][cluster.metadata         ] [Starshine] [foo] create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type       ] [Starshine] [13] Failed to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
             ^
[Line: 1, Column: 0]
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more


Should I open an issue?

Regards,
Lukas



Reply | Threaded
Open this post in threaded view
|

Re: script_fields issue?

Lukáš Vlček
Or may be the "return_type" should make more sense to be added in "sort" portion of the query rather then in "script_fields" portion.

Lukas

On Thu, Jul 29, 2010 at 4:52 PM, Lukáš Vlček <[hidden email]> wrote:
So, I found that the SortField needs field type here: http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118

That is why it is looking for mappings.

OK, I tried to provide mapping for script_fields:
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" },
    "test1" : { "type" : "long" },
    "test2" : { "type" : "string" }
  }
}}'

Now, "sort" : ["test1"] (test1 is date converted to millis) works (note: I was not able to make it reverse:true or desc) but "sort" : ["test2"] still fails with simple message:
{
  "error" : "ReduceSearchPhaseException[Failed to execute phase [query], [reduce] ]; nested: "
}
(no error in server log).

So, sorting on script_fields does not seem to work without issues to me. Second issue is that I do not want to provide explicit mapping in advance (fixing it to specific type) for script_fields. I think this is little against ES approach because script_fields can be defined per query.

Would it be possible to ask mvel for return type for given script in order to determine type for sort function on the fly?

Or how about if the client can provide the type directly as part of script_fields definiton per query. Something like:
"script_fields" : {
    "test1" : {
        "script" : "_some_fancy_math_",
        "params" : { ... },
        "return_type" : "long"
    }
}

Regards,
Lukas



On Thu, Jul 29, 2010 at 3:33 PM, Lukáš Vlček <[hidden email]> wrote:
Oh yeah, that's it.
*BUT*, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d @script_fields.json

I put the following query into script_fields.json file:
{
  "from" : 0, "size" : 3,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  },
  "sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by them (note the "sort" part) it fails:
{
  "error" : "SearchPhaseExecutionException[Failed to execute phase [query], total failure; shardFailures {[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  \"from\" : 0, \"size\" : 3,  \"fields\" : [\"date\",\"string\"],  \"query\" : { \"match_all\" : { } },  \"script_fields\" : {     \"test1\" : { \"script\" : \"doc['date'].date.millis\" },    \"test2\" : { \"script\" : \"doc['string'].stringValue\" }  },  \"sort\" : [\"test1\"]}]]]; nested: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type       ] [Starshine] [foo][0], node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.search.SearchRequest@406039a4]
org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  "from" : 0, "size" : 3,  "fields" : ["date","string"],  "query" : { "match_all" : { } },  "script_fields" : {     "test1" : { "script" : "doc['date'].date.millis" },    "test2" : { "script" : "doc['string'].stringValue" }  },  "sort" : ["test1"]}]]
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]
at org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on scripted field?

Regards,
Lukas


On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon <[hidden email]> wrote:
Hi,

  I think the problem is with curl, and the fact that within the body of the request the ' is used (within the script). It needs to be escaped somehow (not sure how).

-shay.banon


On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček <[hidden email]> wrote:
Hi,

I am experiencing script_fields issue. Below is a shell script with curl examples. In short: I prepare an index, index three objects into it, verify that I can query objects from index and that I can extract values of particular "fields" (though script_fields does not require fields being stored) and then use a script_fields query which fails. Not sure what is wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : { number_of_shards : 1, number_of_replicas : 0 }}'
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" }
  }
}}'
echo 

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d '{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d '{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d '{"date":"2010-01-03","string":"three"}'
echo

echo Search
{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  }
}'
echo

The following is output:

Index setup
{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
  "foo" : {
    "bar" : {
      "_boost" : {
        "name" : "_boost"
      },
      "dynamic" : true,
      "enabled" : true,
      "date_formats" : [ "dateOptionalTime", "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd" ],
      "_source" : {
        "enabled" : true,
        "name" : "_source"
      },
      "_id" : {
        "store" : "no"
      },
      "path" : "full",
      "properties" : {
        "string" : {
          "omit_term_freq_and_positions" : false,
          "index_name" : "string",
          "index" : "not_analyzed",
          "omit_norms" : false,
          "store" : "yes",
          "boost" : 1.0,
          "term_vector" : "no",
          "type" : "string"
        },
        "date" : {
          "omit_term_freq_and_positions" : true,
          "index_name" : "date",
          "index" : "not_analyzed",
          "omit_norms" : true,
          "store" : "yes",
          "boost" : 1.0,
          "format" : "dateOptionalTime",
          "precision_step" : 4,
          "term_vector" : "no",
          "type" : "date"
        }
      },
      "_all" : {
        "enabled" : true,
        "store" : "no",
        "term_vector" : "no"
      },
      "type" : "object"
    }
  }
}
Indexing
{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ {
      "_index" : "foo",
      "_type" : "bar",
      "_id" : "1",
      "_score" : 1.0,
      "fields" : {
        "date" : "2010-01-01T00:00:00.000Z",
        "string" : "one"
      }
    } ]
  }
}
Search with script_fields

{
  "_shards" : {
    "total" : 1,
    "successful" : 0,
    "failed" : 1,
    "failures" : [ {
      "reason" : "CompileException[[Error: null]\n[Near : {... Unknown ....}]\n             ^\n[Line: 1, Column: 0]]; nested: "
    } ]
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ ]
  }
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata         ] [Starshine] [foo] deleting index
[11:45:12,818][INFO ][cluster.metadata         ] [Starshine] [foo] creating index, cause [api], shards [1]/[0], mappings []
[11:45:12,994][INFO ][cluster.metadata         ] [Starshine] [foo] create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type       ] [Starshine] [13] Failed to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
             ^
[Line: 1, Column: 0]
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more


Should I open an issue?

Regards,
Lukas




Reply | Threaded
Open this post in threaded view
|

Re: script_fields issue?

kimchy
Administrator
I am not sure that you understood what script fields do. They basically provide a way to fetch custom script evaluation values for hits returned. They are not related to sorting. If you want to control sorting based on scripts, the simplest way is to use the custom score query.

Other than that, I got a bit confused, did you have problems with sorting fields (they don't have to be stored).

-shay.banon

On Thu, Jul 29, 2010 at 6:14 PM, Lukáš Vlček <[hidden email]> wrote:
Or may be the "return_type" should make more sense to be added in "sort" portion of the query rather then in "script_fields" portion.

Lukas


On Thu, Jul 29, 2010 at 4:52 PM, Lukáš Vlček <[hidden email]> wrote:
So, I found that the SortField needs field type here: http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118

That is why it is looking for mappings.

OK, I tried to provide mapping for script_fields:
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" },
    "test1" : { "type" : "long" },
    "test2" : { "type" : "string" }
  }
}}'

Now, "sort" : ["test1"] (test1 is date converted to millis) works (note: I was not able to make it reverse:true or desc) but "sort" : ["test2"] still fails with simple message:
{
  "error" : "ReduceSearchPhaseException[Failed to execute phase [query], [reduce] ]; nested: "
}
(no error in server log).

So, sorting on script_fields does not seem to work without issues to me. Second issue is that I do not want to provide explicit mapping in advance (fixing it to specific type) for script_fields. I think this is little against ES approach because script_fields can be defined per query.

Would it be possible to ask mvel for return type for given script in order to determine type for sort function on the fly?

Or how about if the client can provide the type directly as part of script_fields definiton per query. Something like:
"script_fields" : {
    "test1" : {
        "script" : "_some_fancy_math_",
        "params" : { ... },
        "return_type" : "long"
    }
}

Regards,
Lukas



On Thu, Jul 29, 2010 at 3:33 PM, Lukáš Vlček <[hidden email]> wrote:
Oh yeah, that's it.
*BUT*, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d @script_fields.json

I put the following query into script_fields.json file:
{
  "from" : 0, "size" : 3,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  },
  "sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by them (note the "sort" part) it fails:
{
  "error" : "SearchPhaseExecutionException[Failed to execute phase [query], total failure; shardFailures {[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  \"from\" : 0, \"size\" : 3,  \"fields\" : [\"date\",\"string\"],  \"query\" : { \"match_all\" : { } },  \"script_fields\" : {     \"test1\" : { \"script\" : \"doc['date'].date.millis\" },    \"test2\" : { \"script\" : \"doc['string'].stringValue\" }  },  \"sort\" : [\"test1\"]}]]]; nested: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type       ] [Starshine] [foo][0], node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.search.SearchRequest@406039a4]
org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  "from" : 0, "size" : 3,  "fields" : ["date","string"],  "query" : { "match_all" : { } },  "script_fields" : {     "test1" : { "script" : "doc['date'].date.millis" },    "test2" : { "script" : "doc['string'].stringValue" }  },  "sort" : ["test1"]}]]
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]
at org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on scripted field?

Regards,
Lukas


On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon <[hidden email]> wrote:
Hi,

  I think the problem is with curl, and the fact that within the body of the request the ' is used (within the script). It needs to be escaped somehow (not sure how).

-shay.banon


On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček <[hidden email]> wrote:
Hi,

I am experiencing script_fields issue. Below is a shell script with curl examples. In short: I prepare an index, index three objects into it, verify that I can query objects from index and that I can extract values of particular "fields" (though script_fields does not require fields being stored) and then use a script_fields query which fails. Not sure what is wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : { number_of_shards : 1, number_of_replicas : 0 }}'
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" }
  }
}}'
echo 

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d '{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d '{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d '{"date":"2010-01-03","string":"three"}'
echo

echo Search
{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  }
}'
echo

The following is output:

Index setup
{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
  "foo" : {
    "bar" : {
      "_boost" : {
        "name" : "_boost"
      },
      "dynamic" : true,
      "enabled" : true,
      "date_formats" : [ "dateOptionalTime", "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd" ],
      "_source" : {
        "enabled" : true,
        "name" : "_source"
      },
      "_id" : {
        "store" : "no"
      },
      "path" : "full",
      "properties" : {
        "string" : {
          "omit_term_freq_and_positions" : false,
          "index_name" : "string",
          "index" : "not_analyzed",
          "omit_norms" : false,
          "store" : "yes",
          "boost" : 1.0,
          "term_vector" : "no",
          "type" : "string"
        },
        "date" : {
          "omit_term_freq_and_positions" : true,
          "index_name" : "date",
          "index" : "not_analyzed",
          "omit_norms" : true,
          "store" : "yes",
          "boost" : 1.0,
          "format" : "dateOptionalTime",
          "precision_step" : 4,
          "term_vector" : "no",
          "type" : "date"
        }
      },
      "_all" : {
        "enabled" : true,
        "store" : "no",
        "term_vector" : "no"
      },
      "type" : "object"
    }
  }
}
Indexing
{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ {
      "_index" : "foo",
      "_type" : "bar",
      "_id" : "1",
      "_score" : 1.0,
      "fields" : {
        "date" : "2010-01-01T00:00:00.000Z",
        "string" : "one"
      }
    } ]
  }
}
Search with script_fields

{
  "_shards" : {
    "total" : 1,
    "successful" : 0,
    "failed" : 1,
    "failures" : [ {
      "reason" : "CompileException[[Error: null]\n[Near : {... Unknown ....}]\n             ^\n[Line: 1, Column: 0]]; nested: "
    } ]
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ ]
  }
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata         ] [Starshine] [foo] deleting index
[11:45:12,818][INFO ][cluster.metadata         ] [Starshine] [foo] creating index, cause [api], shards [1]/[0], mappings []
[11:45:12,994][INFO ][cluster.metadata         ] [Starshine] [foo] create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type       ] [Starshine] [13] Failed to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
             ^
[Line: 1, Column: 0]
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more


Should I open an issue?

Regards,
Lukas





Reply | Threaded
Open this post in threaded view
|

Re: script_fields issue?

Lukáš Vlček
I have the following five entities:

A - D | 2
A - B - C | 3
A | 0
A - B | 1
A - E | 4

There the pattern before pipe is some ordered chain of string symbols. The pattern after the pipe is a number. Now I want to get the following order:

A | 0
A - B | 1
A - B - C | 3
A - D | 2
A - E | 4

In fact the part before the pipe is content of multivalued filed (string) and the part after pipe is a numeric filed.

It would be easy to get desired order if we could sort by the string chain first and then by the number value. But because simple string sort would not work on the string chain (because of the variable length and different number of items in it). Then I think it can be easily done using scritpt_values (the number of items in string chain - in this case 3 - is different for each query). In this case I would built four script fileds f1-4 (three for string and one for number). The result would be (for original order of entities, I am simply right-padding the chain of strings):

f1  f2  f3  f4
-------------
A - D - . | 2
A - B - C | 3
A - . - . | 0
A - B - . | 1
A - E - . | 4

Now we can simply "sort":[f1,f2,f3,f4] and get:

A - . - . | 0
A - B - . | 1
A - B - C | 3
A - D - . | 2
A - E - . | 4

I don't think it is that easy to get this by custom scoring. Is it?
Moreover, docs for script_fields http://www.elasticsearch.com/docs/elasticsearch/rest_api/search/script_fields/ made me think that sorting can be used with them.

(Note: In my example it is not really needed to sort by the number value but that is because I simplified it way too much)

Regards,
Lukas

On Thu, Jul 29, 2010 at 6:28 PM, Shay Banon <[hidden email]> wrote:
I am not sure that you understood what script fields do. They basically provide a way to fetch custom script evaluation values for hits returned. They are not related to sorting. If you want to control sorting based on scripts, the simplest way is to use the custom score query.

Other than that, I got a bit confused, did you have problems with sorting fields (they don't have to be stored).

-shay.banon


On Thu, Jul 29, 2010 at 6:14 PM, Lukáš Vlček <[hidden email]> wrote:
Or may be the "return_type" should make more sense to be added in "sort" portion of the query rather then in "script_fields" portion.

Lukas


On Thu, Jul 29, 2010 at 4:52 PM, Lukáš Vlček <[hidden email]> wrote:
So, I found that the SortField needs field type here: http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118

That is why it is looking for mappings.

OK, I tried to provide mapping for script_fields:
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" },
    "test1" : { "type" : "long" },
    "test2" : { "type" : "string" }
  }
}}'

Now, "sort" : ["test1"] (test1 is date converted to millis) works (note: I was not able to make it reverse:true or desc) but "sort" : ["test2"] still fails with simple message:
{
  "error" : "ReduceSearchPhaseException[Failed to execute phase [query], [reduce] ]; nested: "
}
(no error in server log).

So, sorting on script_fields does not seem to work without issues to me. Second issue is that I do not want to provide explicit mapping in advance (fixing it to specific type) for script_fields. I think this is little against ES approach because script_fields can be defined per query.

Would it be possible to ask mvel for return type for given script in order to determine type for sort function on the fly?

Or how about if the client can provide the type directly as part of script_fields definiton per query. Something like:
"script_fields" : {
    "test1" : {
        "script" : "_some_fancy_math_",
        "params" : { ... },
        "return_type" : "long"
    }
}

Regards,
Lukas



On Thu, Jul 29, 2010 at 3:33 PM, Lukáš Vlček <[hidden email]> wrote:
Oh yeah, that's it.
*BUT*, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d @script_fields.json

I put the following query into script_fields.json file:
{
  "from" : 0, "size" : 3,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  },
  "sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by them (note the "sort" part) it fails:
{
  "error" : "SearchPhaseExecutionException[Failed to execute phase [query], total failure; shardFailures {[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  \"from\" : 0, \"size\" : 3,  \"fields\" : [\"date\",\"string\"],  \"query\" : { \"match_all\" : { } },  \"script_fields\" : {     \"test1\" : { \"script\" : \"doc['date'].date.millis\" },    \"test2\" : { \"script\" : \"doc['string'].stringValue\" }  },  \"sort\" : [\"test1\"]}]]]; nested: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type       ] [Starshine] [foo][0], node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.search.SearchRequest@406039a4]
org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  "from" : 0, "size" : 3,  "fields" : ["date","string"],  "query" : { "match_all" : { } },  "script_fields" : {     "test1" : { "script" : "doc['date'].date.millis" },    "test2" : { "script" : "doc['string'].stringValue" }  },  "sort" : ["test1"]}]]
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]
at org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on scripted field?

Regards,
Lukas


On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon <[hidden email]> wrote:
Hi,

  I think the problem is with curl, and the fact that within the body of the request the ' is used (within the script). It needs to be escaped somehow (not sure how).

-shay.banon


On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček <[hidden email]> wrote:
Hi,

I am experiencing script_fields issue. Below is a shell script with curl examples. In short: I prepare an index, index three objects into it, verify that I can query objects from index and that I can extract values of particular "fields" (though script_fields does not require fields being stored) and then use a script_fields query which fails. Not sure what is wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : { number_of_shards : 1, number_of_replicas : 0 }}'
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" }
  }
}}'
echo 

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d '{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d '{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d '{"date":"2010-01-03","string":"three"}'
echo

echo Search
{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  }
}'
echo

The following is output:

Index setup
{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
  "foo" : {
    "bar" : {
      "_boost" : {
        "name" : "_boost"
      },
      "dynamic" : true,
      "enabled" : true,
      "date_formats" : [ "dateOptionalTime", "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd" ],
      "_source" : {
        "enabled" : true,
        "name" : "_source"
      },
      "_id" : {
        "store" : "no"
      },
      "path" : "full",
      "properties" : {
        "string" : {
          "omit_term_freq_and_positions" : false,
          "index_name" : "string",
          "index" : "not_analyzed",
          "omit_norms" : false,
          "store" : "yes",
          "boost" : 1.0,
          "term_vector" : "no",
          "type" : "string"
        },
        "date" : {
          "omit_term_freq_and_positions" : true,
          "index_name" : "date",
          "index" : "not_analyzed",
          "omit_norms" : true,
          "store" : "yes",
          "boost" : 1.0,
          "format" : "dateOptionalTime",
          "precision_step" : 4,
          "term_vector" : "no",
          "type" : "date"
        }
      },
      "_all" : {
        "enabled" : true,
        "store" : "no",
        "term_vector" : "no"
      },
      "type" : "object"
    }
  }
}
Indexing
{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ {
      "_index" : "foo",
      "_type" : "bar",
      "_id" : "1",
      "_score" : 1.0,
      "fields" : {
        "date" : "2010-01-01T00:00:00.000Z",
        "string" : "one"
      }
    } ]
  }
}
Search with script_fields

{
  "_shards" : {
    "total" : 1,
    "successful" : 0,
    "failed" : 1,
    "failures" : [ {
      "reason" : "CompileException[[Error: null]\n[Near : {... Unknown ....}]\n             ^\n[Line: 1, Column: 0]]; nested: "
    } ]
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ ]
  }
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata         ] [Starshine] [foo] deleting index
[11:45:12,818][INFO ][cluster.metadata         ] [Starshine] [foo] creating index, cause [api], shards [1]/[0], mappings []
[11:45:12,994][INFO ][cluster.metadata         ] [Starshine] [foo] create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type       ] [Starshine] [13] Failed to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
             ^
[Line: 1, Column: 0]
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more


Should I open an issue?

Regards,
Lukas






Reply | Threaded
Open this post in threaded view
|

Re: script_fields issue?

Lukáš Vlček


On Thu, Jul 29, 2010 at 7:22 PM, Lukáš Vlček <[hidden email]> wrote:
I have the following five entities:

A - D | 2
A - B - C | 3
A | 0
A - B | 1
A - E | 4

There the pattern before pipe is some ordered chain of string symbols. The pattern after the pipe is a number. Now I want to get the following order:

A | 0
A - B | 1
A - B - C | 3
A - D | 2
A - E | 4

In fact the part before the pipe is content of multivalued filed (string) and the part after pipe is a numeric filed.

It would be easy to get desired order if we could sort by the string chain first and then by the number value. But because simple string sort would not work on the string chain (because of the variable length and different number of items in it). Then I think it can be easily done using scritpt_values (the number of items in string chain - in this case 3 - is different for each query). In this case I would built four script fileds f1-4 (three for string and one for number). The result would be (for original order of entities, I am simply right-padding the chain of strings):

f1  f2  f3  f4
-------------
A - D - . | 2
A - B - C | 3
A - . - . | 0
A - B - . | 1
A - E - . | 4

Now we can simply "sort":[f1,f2,f3,f4] and get:

A - . - . | 0
A - B - . | 1
A - B - C | 3
A - D - . | 2
A - E - . | 4

I don't think it is that easy to get this by custom scoring. Is it?
Moreover, docs for script_fields http://www.elasticsearch.com/docs/elasticsearch/rest_api/search/script_fields/ made me think that sorting can be used with them.

(Note: In my example it is not really needed to sort by the number value but that is because I simplified it way too much)
Just to fix my simplification: replace
A - E - . | 4
with
A - D - . | 4  
and you get the something that needs to be sorted by both string chain and number value.

Regards,
Lukas

On Thu, Jul 29, 2010 at 6:28 PM, Shay Banon <[hidden email]> wrote:
I am not sure that you understood what script fields do. They basically provide a way to fetch custom script evaluation values for hits returned. They are not related to sorting. If you want to control sorting based on scripts, the simplest way is to use the custom score query.

Other than that, I got a bit confused, did you have problems with sorting fields (they don't have to be stored).

-shay.banon


On Thu, Jul 29, 2010 at 6:14 PM, Lukáš Vlček <[hidden email]> wrote:
Or may be the "return_type" should make more sense to be added in "sort" portion of the query rather then in "script_fields" portion.

Lukas


On Thu, Jul 29, 2010 at 4:52 PM, Lukáš Vlček <[hidden email]> wrote:
So, I found that the SortField needs field type here: http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118

That is why it is looking for mappings.

OK, I tried to provide mapping for script_fields:
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" },
    "test1" : { "type" : "long" },
    "test2" : { "type" : "string" }
  }
}}'

Now, "sort" : ["test1"] (test1 is date converted to millis) works (note: I was not able to make it reverse:true or desc) but "sort" : ["test2"] still fails with simple message:
{
  "error" : "ReduceSearchPhaseException[Failed to execute phase [query], [reduce] ]; nested: "
}
(no error in server log).

So, sorting on script_fields does not seem to work without issues to me. Second issue is that I do not want to provide explicit mapping in advance (fixing it to specific type) for script_fields. I think this is little against ES approach because script_fields can be defined per query.

Would it be possible to ask mvel for return type for given script in order to determine type for sort function on the fly?

Or how about if the client can provide the type directly as part of script_fields definiton per query. Something like:
"script_fields" : {
    "test1" : {
        "script" : "_some_fancy_math_",
        "params" : { ... },
        "return_type" : "long"
    }
}

Regards,
Lukas



On Thu, Jul 29, 2010 at 3:33 PM, Lukáš Vlček <[hidden email]> wrote:
Oh yeah, that's it.
*BUT*, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d @script_fields.json

I put the following query into script_fields.json file:
{
  "from" : 0, "size" : 3,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  },
  "sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by them (note the "sort" part) it fails:
{
  "error" : "SearchPhaseExecutionException[Failed to execute phase [query], total failure; shardFailures {[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  \"from\" : 0, \"size\" : 3,  \"fields\" : [\"date\",\"string\"],  \"query\" : { \"match_all\" : { } },  \"script_fields\" : {     \"test1\" : { \"script\" : \"doc['date'].date.millis\" },    \"test2\" : { \"script\" : \"doc['string'].stringValue\" }  },  \"sort\" : [\"test1\"]}]]]; nested: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type       ] [Starshine] [foo][0], node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.search.SearchRequest@406039a4]
org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  "from" : 0, "size" : 3,  "fields" : ["date","string"],  "query" : { "match_all" : { } },  "script_fields" : {     "test1" : { "script" : "doc['date'].date.millis" },    "test2" : { "script" : "doc['string'].stringValue" }  },  "sort" : ["test1"]}]]
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]
at org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on scripted field?

Regards,
Lukas


On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon <[hidden email]> wrote:
Hi,

  I think the problem is with curl, and the fact that within the body of the request the ' is used (within the script). It needs to be escaped somehow (not sure how).

-shay.banon


On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček <[hidden email]> wrote:
Hi,

I am experiencing script_fields issue. Below is a shell script with curl examples. In short: I prepare an index, index three objects into it, verify that I can query objects from index and that I can extract values of particular "fields" (though script_fields does not require fields being stored) and then use a script_fields query which fails. Not sure what is wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : { number_of_shards : 1, number_of_replicas : 0 }}'
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" }
  }
}}'
echo 

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d '{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d '{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d '{"date":"2010-01-03","string":"three"}'
echo

echo Search
{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  }
}'
echo

The following is output:

Index setup
{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
  "foo" : {
    "bar" : {
      "_boost" : {
        "name" : "_boost"
      },
      "dynamic" : true,
      "enabled" : true,
      "date_formats" : [ "dateOptionalTime", "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd" ],
      "_source" : {
        "enabled" : true,
        "name" : "_source"
      },
      "_id" : {
        "store" : "no"
      },
      "path" : "full",
      "properties" : {
        "string" : {
          "omit_term_freq_and_positions" : false,
          "index_name" : "string",
          "index" : "not_analyzed",
          "omit_norms" : false,
          "store" : "yes",
          "boost" : 1.0,
          "term_vector" : "no",
          "type" : "string"
        },
        "date" : {
          "omit_term_freq_and_positions" : true,
          "index_name" : "date",
          "index" : "not_analyzed",
          "omit_norms" : true,
          "store" : "yes",
          "boost" : 1.0,
          "format" : "dateOptionalTime",
          "precision_step" : 4,
          "term_vector" : "no",
          "type" : "date"
        }
      },
      "_all" : {
        "enabled" : true,
        "store" : "no",
        "term_vector" : "no"
      },
      "type" : "object"
    }
  }
}
Indexing
{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ {
      "_index" : "foo",
      "_type" : "bar",
      "_id" : "1",
      "_score" : 1.0,
      "fields" : {
        "date" : "2010-01-01T00:00:00.000Z",
        "string" : "one"
      }
    } ]
  }
}
Search with script_fields

{
  "_shards" : {
    "total" : 1,
    "successful" : 0,
    "failed" : 1,
    "failures" : [ {
      "reason" : "CompileException[[Error: null]\n[Near : {... Unknown ....}]\n             ^\n[Line: 1, Column: 0]]; nested: "
    } ]
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ ]
  }
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata         ] [Starshine] [foo] deleting index
[11:45:12,818][INFO ][cluster.metadata         ] [Starshine] [foo] creating index, cause [api], shards [1]/[0], mappings []
[11:45:12,994][INFO ][cluster.metadata         ] [Starshine] [foo] create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type       ] [Starshine] [13] Failed to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
             ^
[Line: 1, Column: 0]
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more


Should I open an issue?

Regards,
Lukas







Reply | Threaded
Open this post in threaded view
|

Re: script_fields issue?

kimchy
Administrator
In reply to this post by Lukáš Vlček
I would, in this case, add another field to the document you index that does the mentioned logic, index it, and then sort on it.

On Thu, Jul 29, 2010 at 8:22 PM, Lukáš Vlček <[hidden email]> wrote:
I have the following five entities:

A - D | 2
A - B - C | 3
A | 0
A - B | 1
A - E | 4

There the pattern before pipe is some ordered chain of string symbols. The pattern after the pipe is a number. Now I want to get the following order:

A | 0
A - B | 1
A - B - C | 3
A - D | 2
A - E | 4

In fact the part before the pipe is content of multivalued filed (string) and the part after pipe is a numeric filed.

It would be easy to get desired order if we could sort by the string chain first and then by the number value. But because simple string sort would not work on the string chain (because of the variable length and different number of items in it). Then I think it can be easily done using scritpt_values (the number of items in string chain - in this case 3 - is different for each query). In this case I would built four script fileds f1-4 (three for string and one for number). The result would be (for original order of entities, I am simply right-padding the chain of strings):

f1  f2  f3  f4
-------------
A - D - . | 2
A - B - C | 3
A - . - . | 0
A - B - . | 1
A - E - . | 4

Now we can simply "sort":[f1,f2,f3,f4] and get:

A - . - . | 0
A - B - . | 1
A - B - C | 3
A - D - . | 2
A - E - . | 4

I don't think it is that easy to get this by custom scoring. Is it?
Moreover, docs for script_fields http://www.elasticsearch.com/docs/elasticsearch/rest_api/search/script_fields/ made me think that sorting can be used with them.

(Note: In my example it is not really needed to sort by the number value but that is because I simplified it way too much)

Regards,
Lukas

On Thu, Jul 29, 2010 at 6:28 PM, Shay Banon <[hidden email]> wrote:
I am not sure that you understood what script fields do. They basically provide a way to fetch custom script evaluation values for hits returned. They are not related to sorting. If you want to control sorting based on scripts, the simplest way is to use the custom score query.

Other than that, I got a bit confused, did you have problems with sorting fields (they don't have to be stored).

-shay.banon


On Thu, Jul 29, 2010 at 6:14 PM, Lukáš Vlček <[hidden email]> wrote:
Or may be the "return_type" should make more sense to be added in "sort" portion of the query rather then in "script_fields" portion.

Lukas


On Thu, Jul 29, 2010 at 4:52 PM, Lukáš Vlček <[hidden email]> wrote:
So, I found that the SortField needs field type here: http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118

That is why it is looking for mappings.

OK, I tried to provide mapping for script_fields:
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" },
    "test1" : { "type" : "long" },
    "test2" : { "type" : "string" }
  }
}}'

Now, "sort" : ["test1"] (test1 is date converted to millis) works (note: I was not able to make it reverse:true or desc) but "sort" : ["test2"] still fails with simple message:
{
  "error" : "ReduceSearchPhaseException[Failed to execute phase [query], [reduce] ]; nested: "
}
(no error in server log).

So, sorting on script_fields does not seem to work without issues to me. Second issue is that I do not want to provide explicit mapping in advance (fixing it to specific type) for script_fields. I think this is little against ES approach because script_fields can be defined per query.

Would it be possible to ask mvel for return type for given script in order to determine type for sort function on the fly?

Or how about if the client can provide the type directly as part of script_fields definiton per query. Something like:
"script_fields" : {
    "test1" : {
        "script" : "_some_fancy_math_",
        "params" : { ... },
        "return_type" : "long"
    }
}

Regards,
Lukas



On Thu, Jul 29, 2010 at 3:33 PM, Lukáš Vlček <[hidden email]> wrote:
Oh yeah, that's it.
*BUT*, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d @script_fields.json

I put the following query into script_fields.json file:
{
  "from" : 0, "size" : 3,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  },
  "sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by them (note the "sort" part) it fails:
{
  "error" : "SearchPhaseExecutionException[Failed to execute phase [query], total failure; shardFailures {[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  \"from\" : 0, \"size\" : 3,  \"fields\" : [\"date\",\"string\"],  \"query\" : { \"match_all\" : { } },  \"script_fields\" : {     \"test1\" : { \"script\" : \"doc['date'].date.millis\" },    \"test2\" : { \"script\" : \"doc['string'].stringValue\" }  },  \"sort\" : [\"test1\"]}]]]; nested: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type       ] [Starshine] [foo][0], node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.search.SearchRequest@406039a4]
org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  "from" : 0, "size" : 3,  "fields" : ["date","string"],  "query" : { "match_all" : { } },  "script_fields" : {     "test1" : { "script" : "doc['date'].date.millis" },    "test2" : { "script" : "doc['string'].stringValue" }  },  "sort" : ["test1"]}]]
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]
at org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on scripted field?

Regards,
Lukas


On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon <[hidden email]> wrote:
Hi,

  I think the problem is with curl, and the fact that within the body of the request the ' is used (within the script). It needs to be escaped somehow (not sure how).

-shay.banon


On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček <[hidden email]> wrote:
Hi,

I am experiencing script_fields issue. Below is a shell script with curl examples. In short: I prepare an index, index three objects into it, verify that I can query objects from index and that I can extract values of particular "fields" (though script_fields does not require fields being stored) and then use a script_fields query which fails. Not sure what is wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : { number_of_shards : 1, number_of_replicas : 0 }}'
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" }
  }
}}'
echo 

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d '{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d '{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d '{"date":"2010-01-03","string":"three"}'
echo

echo Search
{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  }
}'
echo

The following is output:

Index setup
{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
  "foo" : {
    "bar" : {
      "_boost" : {
        "name" : "_boost"
      },
      "dynamic" : true,
      "enabled" : true,
      "date_formats" : [ "dateOptionalTime", "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd" ],
      "_source" : {
        "enabled" : true,
        "name" : "_source"
      },
      "_id" : {
        "store" : "no"
      },
      "path" : "full",
      "properties" : {
        "string" : {
          "omit_term_freq_and_positions" : false,
          "index_name" : "string",
          "index" : "not_analyzed",
          "omit_norms" : false,
          "store" : "yes",
          "boost" : 1.0,
          "term_vector" : "no",
          "type" : "string"
        },
        "date" : {
          "omit_term_freq_and_positions" : true,
          "index_name" : "date",
          "index" : "not_analyzed",
          "omit_norms" : true,
          "store" : "yes",
          "boost" : 1.0,
          "format" : "dateOptionalTime",
          "precision_step" : 4,
          "term_vector" : "no",
          "type" : "date"
        }
      },
      "_all" : {
        "enabled" : true,
        "store" : "no",
        "term_vector" : "no"
      },
      "type" : "object"
    }
  }
}
Indexing
{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ {
      "_index" : "foo",
      "_type" : "bar",
      "_id" : "1",
      "_score" : 1.0,
      "fields" : {
        "date" : "2010-01-01T00:00:00.000Z",
        "string" : "one"
      }
    } ]
  }
}
Search with script_fields

{
  "_shards" : {
    "total" : 1,
    "successful" : 0,
    "failed" : 1,
    "failures" : [ {
      "reason" : "CompileException[[Error: null]\n[Near : {... Unknown ....}]\n             ^\n[Line: 1, Column: 0]]; nested: "
    } ]
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ ]
  }
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata         ] [Starshine] [foo] deleting index
[11:45:12,818][INFO ][cluster.metadata         ] [Starshine] [foo] creating index, cause [api], shards [1]/[0], mappings []
[11:45:12,994][INFO ][cluster.metadata         ] [Starshine] [foo] create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type       ] [Starshine] [13] Failed to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
             ^
[Line: 1, Column: 0]
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more


Should I open an issue?

Regards,
Lukas







Reply | Threaded
Open this post in threaded view
|

Re: script_fields issue?

Lukáš Vlček
I can not do this at index time because I do not know the length of the string chain in advance.
Take for example document related to A | 0 entity. How can I know that I want to right-padd it to A - . - . | 0 without knowing that there are other related documents having A - B - C ?
As of know it seems that it will be easier to handle that logic on the client side (I will sort document by client, in my case there won't be a lot of related documents, I assume up to 30 max).

However, I am still curious if it is possible (or planned) to allow sort on script_fields. As of now it seems to be possible only if explicit mapping is provided for script_fields (which means it lacks a lot of flexibility considering that script_fields are defined per query while mapping per index).

Regards,
Lukas

2010/7/29 Shay Banon <[hidden email]>
I would, in this case, add another field to the document you index that does the mentioned logic, index it, and then sort on it.


On Thu, Jul 29, 2010 at 8:22 PM, Lukáš Vlček <[hidden email]> wrote:
I have the following five entities:

A - D | 2
A - B - C | 3
A | 0
A - B | 1
A - E | 4

There the pattern before pipe is some ordered chain of string symbols. The pattern after the pipe is a number. Now I want to get the following order:

A | 0
A - B | 1
A - B - C | 3
A - D | 2
A - E | 4

In fact the part before the pipe is content of multivalued filed (string) and the part after pipe is a numeric filed.

It would be easy to get desired order if we could sort by the string chain first and then by the number value. But because simple string sort would not work on the string chain (because of the variable length and different number of items in it). Then I think it can be easily done using scritpt_values (the number of items in string chain - in this case 3 - is different for each query). In this case I would built four script fileds f1-4 (three for string and one for number). The result would be (for original order of entities, I am simply right-padding the chain of strings):

f1  f2  f3  f4
-------------
A - D - . | 2
A - B - C | 3
A - . - . | 0
A - B - . | 1
A - E - . | 4

Now we can simply "sort":[f1,f2,f3,f4] and get:

A - . - . | 0
A - B - . | 1
A - B - C | 3
A - D - . | 2
A - E - . | 4

I don't think it is that easy to get this by custom scoring. Is it?
Moreover, docs for script_fields http://www.elasticsearch.com/docs/elasticsearch/rest_api/search/script_fields/ made me think that sorting can be used with them.

(Note: In my example it is not really needed to sort by the number value but that is because I simplified it way too much)

Regards,
Lukas

On Thu, Jul 29, 2010 at 6:28 PM, Shay Banon <[hidden email]> wrote:
I am not sure that you understood what script fields do. They basically provide a way to fetch custom script evaluation values for hits returned. They are not related to sorting. If you want to control sorting based on scripts, the simplest way is to use the custom score query.

Other than that, I got a bit confused, did you have problems with sorting fields (they don't have to be stored).

-shay.banon


On Thu, Jul 29, 2010 at 6:14 PM, Lukáš Vlček <[hidden email]> wrote:
Or may be the "return_type" should make more sense to be added in "sort" portion of the query rather then in "script_fields" portion.

Lukas


On Thu, Jul 29, 2010 at 4:52 PM, Lukáš Vlček <[hidden email]> wrote:
So, I found that the SortField needs field type here: http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118

That is why it is looking for mappings.

OK, I tried to provide mapping for script_fields:
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" },
    "test1" : { "type" : "long" },
    "test2" : { "type" : "string" }
  }
}}'

Now, "sort" : ["test1"] (test1 is date converted to millis) works (note: I was not able to make it reverse:true or desc) but "sort" : ["test2"] still fails with simple message:
{
  "error" : "ReduceSearchPhaseException[Failed to execute phase [query], [reduce] ]; nested: "
}
(no error in server log).

So, sorting on script_fields does not seem to work without issues to me. Second issue is that I do not want to provide explicit mapping in advance (fixing it to specific type) for script_fields. I think this is little against ES approach because script_fields can be defined per query.

Would it be possible to ask mvel for return type for given script in order to determine type for sort function on the fly?

Or how about if the client can provide the type directly as part of script_fields definiton per query. Something like:
"script_fields" : {
    "test1" : {
        "script" : "_some_fancy_math_",
        "params" : { ... },
        "return_type" : "long"
    }
}

Regards,
Lukas



On Thu, Jul 29, 2010 at 3:33 PM, Lukáš Vlček <[hidden email]> wrote:
Oh yeah, that's it.
*BUT*, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d @script_fields.json

I put the following query into script_fields.json file:
{
  "from" : 0, "size" : 3,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  },
  "sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by them (note the "sort" part) it fails:
{
  "error" : "SearchPhaseExecutionException[Failed to execute phase [query], total failure; shardFailures {[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  \"from\" : 0, \"size\" : 3,  \"fields\" : [\"date\",\"string\"],  \"query\" : { \"match_all\" : { } },  \"script_fields\" : {     \"test1\" : { \"script\" : \"doc['date'].date.millis\" },    \"test2\" : { \"script\" : \"doc['string'].stringValue\" }  },  \"sort\" : [\"test1\"]}]]]; nested: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type       ] [Starshine] [foo][0], node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.search.SearchRequest@406039a4]
org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  "from" : 0, "size" : 3,  "fields" : ["date","string"],  "query" : { "match_all" : { } },  "script_fields" : {     "test1" : { "script" : "doc['date'].date.millis" },    "test2" : { "script" : "doc['string'].stringValue" }  },  "sort" : ["test1"]}]]
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]
at org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on scripted field?

Regards,
Lukas


On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon <[hidden email]> wrote:
Hi,

  I think the problem is with curl, and the fact that within the body of the request the ' is used (within the script). It needs to be escaped somehow (not sure how).

-shay.banon


On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček <[hidden email]> wrote:
Hi,

I am experiencing script_fields issue. Below is a shell script with curl examples. In short: I prepare an index, index three objects into it, verify that I can query objects from index and that I can extract values of particular "fields" (though script_fields does not require fields being stored) and then use a script_fields query which fails. Not sure what is wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : { number_of_shards : 1, number_of_replicas : 0 }}'
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" }
  }
}}'
echo 

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d '{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d '{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d '{"date":"2010-01-03","string":"three"}'
echo

echo Search
{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  }
}'
echo

The following is output:

Index setup
{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
  "foo" : {
    "bar" : {
      "_boost" : {
        "name" : "_boost"
      },
      "dynamic" : true,
      "enabled" : true,
      "date_formats" : [ "dateOptionalTime", "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd" ],
      "_source" : {
        "enabled" : true,
        "name" : "_source"
      },
      "_id" : {
        "store" : "no"
      },
      "path" : "full",
      "properties" : {
        "string" : {
          "omit_term_freq_and_positions" : false,
          "index_name" : "string",
          "index" : "not_analyzed",
          "omit_norms" : false,
          "store" : "yes",
          "boost" : 1.0,
          "term_vector" : "no",
          "type" : "string"
        },
        "date" : {
          "omit_term_freq_and_positions" : true,
          "index_name" : "date",
          "index" : "not_analyzed",
          "omit_norms" : true,
          "store" : "yes",
          "boost" : 1.0,
          "format" : "dateOptionalTime",
          "precision_step" : 4,
          "term_vector" : "no",
          "type" : "date"
        }
      },
      "_all" : {
        "enabled" : true,
        "store" : "no",
        "term_vector" : "no"
      },
      "type" : "object"
    }
  }
}
Indexing
{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ {
      "_index" : "foo",
      "_type" : "bar",
      "_id" : "1",
      "_score" : 1.0,
      "fields" : {
        "date" : "2010-01-01T00:00:00.000Z",
        "string" : "one"
      }
    } ]
  }
}
Search with script_fields

{
  "_shards" : {
    "total" : 1,
    "successful" : 0,
    "failed" : 1,
    "failures" : [ {
      "reason" : "CompileException[[Error: null]\n[Near : {... Unknown ....}]\n             ^\n[Line: 1, Column: 0]]; nested: "
    } ]
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ ]
  }
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata         ] [Starshine] [foo] deleting index
[11:45:12,818][INFO ][cluster.metadata         ] [Starshine] [foo] creating index, cause [api], shards [1]/[0], mappings []
[11:45:12,994][INFO ][cluster.metadata         ] [Starshine] [foo] create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type       ] [Starshine] [13] Failed to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
             ^
[Line: 1, Column: 0]
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more


Should I open an issue?

Regards,
Lukas








Reply | Threaded
Open this post in threaded view
|

Re: script_fields issue?

kimchy
Administrator
Script fields are not using any mappings. It pretty simple what they do, you get 10 hits, for that 10 hits (docs), the script fields are evaluated. What you want is to have the ability to sort based on several scripts, which is possible.

-shay.banon

On Thu, Jul 29, 2010 at 10:09 PM, Lukáš Vlček <[hidden email]> wrote:
I can not do this at index time because I do not know the length of the string chain in advance.
Take for example document related to A | 0 entity. How can I know that I want to right-padd it to A - . - . | 0 without knowing that there are other related documents having A - B - C ?
As of know it seems that it will be easier to handle that logic on the client side (I will sort document by client, in my case there won't be a lot of related documents, I assume up to 30 max).

However, I am still curious if it is possible (or planned) to allow sort on script_fields. As of now it seems to be possible only if explicit mapping is provided for script_fields (which means it lacks a lot of flexibility considering that script_fields are defined per query while mapping per index).

Regards,
Lukas

2010/7/29 Shay Banon <[hidden email]>

I would, in this case, add another field to the document you index that does the mentioned logic, index it, and then sort on it.


On Thu, Jul 29, 2010 at 8:22 PM, Lukáš Vlček <[hidden email]> wrote:
I have the following five entities:

A - D | 2
A - B - C | 3
A | 0
A - B | 1
A - E | 4

There the pattern before pipe is some ordered chain of string symbols. The pattern after the pipe is a number. Now I want to get the following order:

A | 0
A - B | 1
A - B - C | 3
A - D | 2
A - E | 4

In fact the part before the pipe is content of multivalued filed (string) and the part after pipe is a numeric filed.

It would be easy to get desired order if we could sort by the string chain first and then by the number value. But because simple string sort would not work on the string chain (because of the variable length and different number of items in it). Then I think it can be easily done using scritpt_values (the number of items in string chain - in this case 3 - is different for each query). In this case I would built four script fileds f1-4 (three for string and one for number). The result would be (for original order of entities, I am simply right-padding the chain of strings):

f1  f2  f3  f4
-------------
A - D - . | 2
A - B - C | 3
A - . - . | 0
A - B - . | 1
A - E - . | 4

Now we can simply "sort":[f1,f2,f3,f4] and get:

A - . - . | 0
A - B - . | 1
A - B - C | 3
A - D - . | 2
A - E - . | 4

I don't think it is that easy to get this by custom scoring. Is it?
Moreover, docs for script_fields http://www.elasticsearch.com/docs/elasticsearch/rest_api/search/script_fields/ made me think that sorting can be used with them.

(Note: In my example it is not really needed to sort by the number value but that is because I simplified it way too much)

Regards,
Lukas

On Thu, Jul 29, 2010 at 6:28 PM, Shay Banon <[hidden email]> wrote:
I am not sure that you understood what script fields do. They basically provide a way to fetch custom script evaluation values for hits returned. They are not related to sorting. If you want to control sorting based on scripts, the simplest way is to use the custom score query.

Other than that, I got a bit confused, did you have problems with sorting fields (they don't have to be stored).

-shay.banon


On Thu, Jul 29, 2010 at 6:14 PM, Lukáš Vlček <[hidden email]> wrote:
Or may be the "return_type" should make more sense to be added in "sort" portion of the query rather then in "script_fields" portion.

Lukas


On Thu, Jul 29, 2010 at 4:52 PM, Lukáš Vlček <[hidden email]> wrote:
So, I found that the SortField needs field type here: http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118

That is why it is looking for mappings.

OK, I tried to provide mapping for script_fields:
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" },
    "test1" : { "type" : "long" },
    "test2" : { "type" : "string" }
  }
}}'

Now, "sort" : ["test1"] (test1 is date converted to millis) works (note: I was not able to make it reverse:true or desc) but "sort" : ["test2"] still fails with simple message:
{
  "error" : "ReduceSearchPhaseException[Failed to execute phase [query], [reduce] ]; nested: "
}
(no error in server log).

So, sorting on script_fields does not seem to work without issues to me. Second issue is that I do not want to provide explicit mapping in advance (fixing it to specific type) for script_fields. I think this is little against ES approach because script_fields can be defined per query.

Would it be possible to ask mvel for return type for given script in order to determine type for sort function on the fly?

Or how about if the client can provide the type directly as part of script_fields definiton per query. Something like:
"script_fields" : {
    "test1" : {
        "script" : "_some_fancy_math_",
        "params" : { ... },
        "return_type" : "long"
    }
}

Regards,
Lukas



On Thu, Jul 29, 2010 at 3:33 PM, Lukáš Vlček <[hidden email]> wrote:
Oh yeah, that's it.
*BUT*, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d @script_fields.json

I put the following query into script_fields.json file:
{
  "from" : 0, "size" : 3,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  },
  "sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by them (note the "sort" part) it fails:
{
  "error" : "SearchPhaseExecutionException[Failed to execute phase [query], total failure; shardFailures {[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  \"from\" : 0, \"size\" : 3,  \"fields\" : [\"date\",\"string\"],  \"query\" : { \"match_all\" : { } },  \"script_fields\" : {     \"test1\" : { \"script\" : \"doc['date'].date.millis\" },    \"test2\" : { \"script\" : \"doc['string'].stringValue\" }  },  \"sort\" : [\"test1\"]}]]]; nested: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type       ] [Starshine] [foo][0], node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.search.SearchRequest@406039a4]
org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  "from" : 0, "size" : 3,  "fields" : ["date","string"],  "query" : { "match_all" : { } },  "script_fields" : {     "test1" : { "script" : "doc['date'].date.millis" },    "test2" : { "script" : "doc['string'].stringValue" }  },  "sort" : ["test1"]}]]
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]
at org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on scripted field?

Regards,
Lukas


On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon <[hidden email]> wrote:
Hi,

  I think the problem is with curl, and the fact that within the body of the request the ' is used (within the script). It needs to be escaped somehow (not sure how).

-shay.banon


On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček <[hidden email]> wrote:
Hi,

I am experiencing script_fields issue. Below is a shell script with curl examples. In short: I prepare an index, index three objects into it, verify that I can query objects from index and that I can extract values of particular "fields" (though script_fields does not require fields being stored) and then use a script_fields query which fails. Not sure what is wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : { number_of_shards : 1, number_of_replicas : 0 }}'
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" }
  }
}}'
echo 

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d '{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d '{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d '{"date":"2010-01-03","string":"three"}'
echo

echo Search
{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  }
}'
echo

The following is output:

Index setup
{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
  "foo" : {
    "bar" : {
      "_boost" : {
        "name" : "_boost"
      },
      "dynamic" : true,
      "enabled" : true,
      "date_formats" : [ "dateOptionalTime", "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd" ],
      "_source" : {
        "enabled" : true,
        "name" : "_source"
      },
      "_id" : {
        "store" : "no"
      },
      "path" : "full",
      "properties" : {
        "string" : {
          "omit_term_freq_and_positions" : false,
          "index_name" : "string",
          "index" : "not_analyzed",
          "omit_norms" : false,
          "store" : "yes",
          "boost" : 1.0,
          "term_vector" : "no",
          "type" : "string"
        },
        "date" : {
          "omit_term_freq_and_positions" : true,
          "index_name" : "date",
          "index" : "not_analyzed",
          "omit_norms" : true,
          "store" : "yes",
          "boost" : 1.0,
          "format" : "dateOptionalTime",
          "precision_step" : 4,
          "term_vector" : "no",
          "type" : "date"
        }
      },
      "_all" : {
        "enabled" : true,
        "store" : "no",
        "term_vector" : "no"
      },
      "type" : "object"
    }
  }
}
Indexing
{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ {
      "_index" : "foo",
      "_type" : "bar",
      "_id" : "1",
      "_score" : 1.0,
      "fields" : {
        "date" : "2010-01-01T00:00:00.000Z",
        "string" : "one"
      }
    } ]
  }
}
Search with script_fields

{
  "_shards" : {
    "total" : 1,
    "successful" : 0,
    "failed" : 1,
    "failures" : [ {
      "reason" : "CompileException[[Error: null]\n[Near : {... Unknown ....}]\n             ^\n[Line: 1, Column: 0]]; nested: "
    } ]
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ ]
  }
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata         ] [Starshine] [foo] deleting index
[11:45:12,818][INFO ][cluster.metadata         ] [Starshine] [foo] creating index, cause [api], shards [1]/[0], mappings []
[11:45:12,994][INFO ][cluster.metadata         ] [Starshine] [foo] create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type       ] [Starshine] [13] Failed to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
             ^
[Line: 1, Column: 0]
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more


Should I open an issue?

Regards,
Lukas









Reply | Threaded
Open this post in threaded view
|

Re: script_fields issue?

kimchy
Administrator
I meant possible to implement ;).

I had another look at your problem. Maybe one way to solve it, if you know in advance the maximum number of levels you will have in the string field, is to have empty markers for empty levels, that will still perserve sorting? Adding multiple sort scripts feature can be done, but it will be slower. First, because score based sorting is faster (which you get with custom score query), and second because the mentioned scripts will need to be evaluated for every doc the query hits in order to do the sorting correctly. This can be mitigated by having more shards and more machines, basically resulting in distributing the search load across machines (if the perf is really a problem).

-shay.banon

On Fri, Jul 30, 2010 at 12:15 AM, Shay Banon <[hidden email]> wrote:
Script fields are not using any mappings. It pretty simple what they do, you get 10 hits, for that 10 hits (docs), the script fields are evaluated. What you want is to have the ability to sort based on several scripts, which is possible.

-shay.banon


On Thu, Jul 29, 2010 at 10:09 PM, Lukáš Vlček <[hidden email]> wrote:
I can not do this at index time because I do not know the length of the string chain in advance.
Take for example document related to A | 0 entity. How can I know that I want to right-padd it to A - . - . | 0 without knowing that there are other related documents having A - B - C ?
As of know it seems that it will be easier to handle that logic on the client side (I will sort document by client, in my case there won't be a lot of related documents, I assume up to 30 max).

However, I am still curious if it is possible (or planned) to allow sort on script_fields. As of now it seems to be possible only if explicit mapping is provided for script_fields (which means it lacks a lot of flexibility considering that script_fields are defined per query while mapping per index).

Regards,
Lukas

2010/7/29 Shay Banon <[hidden email]>

I would, in this case, add another field to the document you index that does the mentioned logic, index it, and then sort on it.


On Thu, Jul 29, 2010 at 8:22 PM, Lukáš Vlček <[hidden email]> wrote:
I have the following five entities:

A - D | 2
A - B - C | 3
A | 0
A - B | 1
A - E | 4

There the pattern before pipe is some ordered chain of string symbols. The pattern after the pipe is a number. Now I want to get the following order:

A | 0
A - B | 1
A - B - C | 3
A - D | 2
A - E | 4

In fact the part before the pipe is content of multivalued filed (string) and the part after pipe is a numeric filed.

It would be easy to get desired order if we could sort by the string chain first and then by the number value. But because simple string sort would not work on the string chain (because of the variable length and different number of items in it). Then I think it can be easily done using scritpt_values (the number of items in string chain - in this case 3 - is different for each query). In this case I would built four script fileds f1-4 (three for string and one for number). The result would be (for original order of entities, I am simply right-padding the chain of strings):

f1  f2  f3  f4
-------------
A - D - . | 2
A - B - C | 3
A - . - . | 0
A - B - . | 1
A - E - . | 4

Now we can simply "sort":[f1,f2,f3,f4] and get:

A - . - . | 0
A - B - . | 1
A - B - C | 3
A - D - . | 2
A - E - . | 4

I don't think it is that easy to get this by custom scoring. Is it?
Moreover, docs for script_fields http://www.elasticsearch.com/docs/elasticsearch/rest_api/search/script_fields/ made me think that sorting can be used with them.

(Note: In my example it is not really needed to sort by the number value but that is because I simplified it way too much)

Regards,
Lukas

On Thu, Jul 29, 2010 at 6:28 PM, Shay Banon <[hidden email]> wrote:
I am not sure that you understood what script fields do. They basically provide a way to fetch custom script evaluation values for hits returned. They are not related to sorting. If you want to control sorting based on scripts, the simplest way is to use the custom score query.

Other than that, I got a bit confused, did you have problems with sorting fields (they don't have to be stored).

-shay.banon


On Thu, Jul 29, 2010 at 6:14 PM, Lukáš Vlček <[hidden email]> wrote:
Or may be the "return_type" should make more sense to be added in "sort" portion of the query rather then in "script_fields" portion.

Lukas


On Thu, Jul 29, 2010 at 4:52 PM, Lukáš Vlček <[hidden email]> wrote:
So, I found that the SortField needs field type here: http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118

That is why it is looking for mappings.

OK, I tried to provide mapping for script_fields:
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" },
    "test1" : { "type" : "long" },
    "test2" : { "type" : "string" }
  }
}}'

Now, "sort" : ["test1"] (test1 is date converted to millis) works (note: I was not able to make it reverse:true or desc) but "sort" : ["test2"] still fails with simple message:
{
  "error" : "ReduceSearchPhaseException[Failed to execute phase [query], [reduce] ]; nested: "
}
(no error in server log).

So, sorting on script_fields does not seem to work without issues to me. Second issue is that I do not want to provide explicit mapping in advance (fixing it to specific type) for script_fields. I think this is little against ES approach because script_fields can be defined per query.

Would it be possible to ask mvel for return type for given script in order to determine type for sort function on the fly?

Or how about if the client can provide the type directly as part of script_fields definiton per query. Something like:
"script_fields" : {
    "test1" : {
        "script" : "_some_fancy_math_",
        "params" : { ... },
        "return_type" : "long"
    }
}

Regards,
Lukas



On Thu, Jul 29, 2010 at 3:33 PM, Lukáš Vlček <[hidden email]> wrote:
Oh yeah, that's it.
*BUT*, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d @script_fields.json

I put the following query into script_fields.json file:
{
  "from" : 0, "size" : 3,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  },
  "sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by them (note the "sort" part) it fails:
{
  "error" : "SearchPhaseExecutionException[Failed to execute phase [query], total failure; shardFailures {[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  \"from\" : 0, \"size\" : 3,  \"fields\" : [\"date\",\"string\"],  \"query\" : { \"match_all\" : { } },  \"script_fields\" : {     \"test1\" : { \"script\" : \"doc['date'].date.millis\" },    \"test2\" : { \"script\" : \"doc['string'].stringValue\" }  },  \"sort\" : [\"test1\"]}]]]; nested: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type       ] [Starshine] [foo][0], node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.search.SearchRequest@406039a4]
org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  "from" : 0, "size" : 3,  "fields" : ["date","string"],  "query" : { "match_all" : { } },  "script_fields" : {     "test1" : { "script" : "doc['date'].date.millis" },    "test2" : { "script" : "doc['string'].stringValue" }  },  "sort" : ["test1"]}]]
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]
at org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on scripted field?

Regards,
Lukas


On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon <[hidden email]> wrote:
Hi,

  I think the problem is with curl, and the fact that within the body of the request the ' is used (within the script). It needs to be escaped somehow (not sure how).

-shay.banon


On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček <[hidden email]> wrote:
Hi,

I am experiencing script_fields issue. Below is a shell script with curl examples. In short: I prepare an index, index three objects into it, verify that I can query objects from index and that I can extract values of particular "fields" (though script_fields does not require fields being stored) and then use a script_fields query which fails. Not sure what is wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : { number_of_shards : 1, number_of_replicas : 0 }}'
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" }
  }
}}'
echo 

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d '{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d '{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d '{"date":"2010-01-03","string":"three"}'
echo

echo Search
{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  }
}'
echo

The following is output:

Index setup
{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
  "foo" : {
    "bar" : {
      "_boost" : {
        "name" : "_boost"
      },
      "dynamic" : true,
      "enabled" : true,
      "date_formats" : [ "dateOptionalTime", "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd" ],
      "_source" : {
        "enabled" : true,
        "name" : "_source"
      },
      "_id" : {
        "store" : "no"
      },
      "path" : "full",
      "properties" : {
        "string" : {
          "omit_term_freq_and_positions" : false,
          "index_name" : "string",
          "index" : "not_analyzed",
          "omit_norms" : false,
          "store" : "yes",
          "boost" : 1.0,
          "term_vector" : "no",
          "type" : "string"
        },
        "date" : {
          "omit_term_freq_and_positions" : true,
          "index_name" : "date",
          "index" : "not_analyzed",
          "omit_norms" : true,
          "store" : "yes",
          "boost" : 1.0,
          "format" : "dateOptionalTime",
          "precision_step" : 4,
          "term_vector" : "no",
          "type" : "date"
        }
      },
      "_all" : {
        "enabled" : true,
        "store" : "no",
        "term_vector" : "no"
      },
      "type" : "object"
    }
  }
}
Indexing
{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ {
      "_index" : "foo",
      "_type" : "bar",
      "_id" : "1",
      "_score" : 1.0,
      "fields" : {
        "date" : "2010-01-01T00:00:00.000Z",
        "string" : "one"
      }
    } ]
  }
}
Search with script_fields

{
  "_shards" : {
    "total" : 1,
    "successful" : 0,
    "failed" : 1,
    "failures" : [ {
      "reason" : "CompileException[[Error: null]\n[Near : {... Unknown ....}]\n             ^\n[Line: 1, Column: 0]]; nested: "
    } ]
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ ]
  }
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata         ] [Starshine] [foo] deleting index
[11:45:12,818][INFO ][cluster.metadata         ] [Starshine] [foo] creating index, cause [api], shards [1]/[0], mappings []
[11:45:12,994][INFO ][cluster.metadata         ] [Starshine] [foo] create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type       ] [Starshine] [13] Failed to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
             ^
[Line: 1, Column: 0]
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more


Should I open an issue?

Regards,
Lukas










Reply | Threaded
Open this post in threaded view
|

Re: script_fields issue?

Lukáš Vlček
I will do the sorting on the client side because the set of document I need to sort is small (<30 documents in most cases). I just got impression that ES could have done it for me in simple and elegant way (again, the pref would not be a problem due to small number of documents in the set to sort).

However, it seems to me that there is a room for improvement when it comes to script fields and sorting, and may be sorting in general because it could be useful to allow users to provide custom sort function as well (FieldComparatorSource), a nice-to-have feature for some next release :-). 

I red the script_fields docs again and I think I understand now what the note regarding sorting and faceting means (it means that if the document field have been already used for faceting or sorting then there is no additional memory cost when used also in script_fields).

Regards,
Lukas

2010/7/30 Shay Banon <[hidden email]>
I meant possible to implement ;).

I had another look at your problem. Maybe one way to solve it, if you know in advance the maximum number of levels you will have in the string field, is to have empty markers for empty levels, that will still perserve sorting? Adding multiple sort scripts feature can be done, but it will be slower. First, because score based sorting is faster (which you get with custom score query), and second because the mentioned scripts will need to be evaluated for every doc the query hits in order to do the sorting correctly. This can be mitigated by having more shards and more machines, basically resulting in distributing the search load across machines (if the perf is really a problem).

-shay.banon


On Fri, Jul 30, 2010 at 12:15 AM, Shay Banon <[hidden email]> wrote:
Script fields are not using any mappings. It pretty simple what they do, you get 10 hits, for that 10 hits (docs), the script fields are evaluated. What you want is to have the ability to sort based on several scripts, which is possible.

-shay.banon


On Thu, Jul 29, 2010 at 10:09 PM, Lukáš Vlček <[hidden email]> wrote:
I can not do this at index time because I do not know the length of the string chain in advance.
Take for example document related to A | 0 entity. How can I know that I want to right-padd it to A - . - . | 0 without knowing that there are other related documents having A - B - C ?
As of know it seems that it will be easier to handle that logic on the client side (I will sort document by client, in my case there won't be a lot of related documents, I assume up to 30 max).

However, I am still curious if it is possible (or planned) to allow sort on script_fields. As of now it seems to be possible only if explicit mapping is provided for script_fields (which means it lacks a lot of flexibility considering that script_fields are defined per query while mapping per index).

Regards,
Lukas

2010/7/29 Shay Banon <[hidden email]>

I would, in this case, add another field to the document you index that does the mentioned logic, index it, and then sort on it.


On Thu, Jul 29, 2010 at 8:22 PM, Lukáš Vlček <[hidden email]> wrote:
I have the following five entities:

A - D | 2
A - B - C | 3
A | 0
A - B | 1
A - E | 4

There the pattern before pipe is some ordered chain of string symbols. The pattern after the pipe is a number. Now I want to get the following order:

A | 0
A - B | 1
A - B - C | 3
A - D | 2
A - E | 4

In fact the part before the pipe is content of multivalued filed (string) and the part after pipe is a numeric filed.

It would be easy to get desired order if we could sort by the string chain first and then by the number value. But because simple string sort would not work on the string chain (because of the variable length and different number of items in it). Then I think it can be easily done using scritpt_values (the number of items in string chain - in this case 3 - is different for each query). In this case I would built four script fileds f1-4 (three for string and one for number). The result would be (for original order of entities, I am simply right-padding the chain of strings):

f1  f2  f3  f4
-------------
A - D - . | 2
A - B - C | 3
A - . - . | 0
A - B - . | 1
A - E - . | 4

Now we can simply "sort":[f1,f2,f3,f4] and get:

A - . - . | 0
A - B - . | 1
A - B - C | 3
A - D - . | 2
A - E - . | 4

I don't think it is that easy to get this by custom scoring. Is it?
Moreover, docs for script_fields http://www.elasticsearch.com/docs/elasticsearch/rest_api/search/script_fields/ made me think that sorting can be used with them.

(Note: In my example it is not really needed to sort by the number value but that is because I simplified it way too much)

Regards,
Lukas

On Thu, Jul 29, 2010 at 6:28 PM, Shay Banon <[hidden email]> wrote:
I am not sure that you understood what script fields do. They basically provide a way to fetch custom script evaluation values for hits returned. They are not related to sorting. If you want to control sorting based on scripts, the simplest way is to use the custom score query.

Other than that, I got a bit confused, did you have problems with sorting fields (they don't have to be stored).

-shay.banon


On Thu, Jul 29, 2010 at 6:14 PM, Lukáš Vlček <[hidden email]> wrote:
Or may be the "return_type" should make more sense to be added in "sort" portion of the query rather then in "script_fields" portion.

Lukas


On Thu, Jul 29, 2010 at 4:52 PM, Lukáš Vlček <[hidden email]> wrote:
So, I found that the SortField needs field type here: http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118

That is why it is looking for mappings.

OK, I tried to provide mapping for script_fields:
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" },
    "test1" : { "type" : "long" },
    "test2" : { "type" : "string" }
  }
}}'

Now, "sort" : ["test1"] (test1 is date converted to millis) works (note: I was not able to make it reverse:true or desc) but "sort" : ["test2"] still fails with simple message:
{
  "error" : "ReduceSearchPhaseException[Failed to execute phase [query], [reduce] ]; nested: "
}
(no error in server log).

So, sorting on script_fields does not seem to work without issues to me. Second issue is that I do not want to provide explicit mapping in advance (fixing it to specific type) for script_fields. I think this is little against ES approach because script_fields can be defined per query.

Would it be possible to ask mvel for return type for given script in order to determine type for sort function on the fly?

Or how about if the client can provide the type directly as part of script_fields definiton per query. Something like:
"script_fields" : {
    "test1" : {
        "script" : "_some_fancy_math_",
        "params" : { ... },
        "return_type" : "long"
    }
}

Regards,
Lukas



On Thu, Jul 29, 2010 at 3:33 PM, Lukáš Vlček <[hidden email]> wrote:
Oh yeah, that's it.
*BUT*, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d @script_fields.json

I put the following query into script_fields.json file:
{
  "from" : 0, "size" : 3,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  },
  "sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by them (note the "sort" part) it fails:
{
  "error" : "SearchPhaseExecutionException[Failed to execute phase [query], total failure; shardFailures {[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  \"from\" : 0, \"size\" : 3,  \"fields\" : [\"date\",\"string\"],  \"query\" : { \"match_all\" : { } },  \"script_fields\" : {     \"test1\" : { \"script\" : \"doc['date'].date.millis\" },    \"test2\" : { \"script\" : \"doc['string'].stringValue\" }  },  \"sort\" : [\"test1\"]}]]]; nested: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type       ] [Starshine] [foo][0], node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.search.SearchRequest@406039a4]
org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  "from" : 0, "size" : 3,  "fields" : ["date","string"],  "query" : { "match_all" : { } },  "script_fields" : {     "test1" : { "script" : "doc['date'].date.millis" },    "test2" : { "script" : "doc['string'].stringValue" }  },  "sort" : ["test1"]}]]
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]
at org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on scripted field?

Regards,
Lukas


On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon <[hidden email]> wrote:
Hi,

  I think the problem is with curl, and the fact that within the body of the request the ' is used (within the script). It needs to be escaped somehow (not sure how).

-shay.banon


On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček <[hidden email]> wrote:
Hi,

I am experiencing script_fields issue. Below is a shell script with curl examples. In short: I prepare an index, index three objects into it, verify that I can query objects from index and that I can extract values of particular "fields" (though script_fields does not require fields being stored) and then use a script_fields query which fails. Not sure what is wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : { number_of_shards : 1, number_of_replicas : 0 }}'
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" }
  }
}}'
echo 

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d '{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d '{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d '{"date":"2010-01-03","string":"three"}'
echo

echo Search
{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  }
}'
echo

The following is output:

Index setup
{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
  "foo" : {
    "bar" : {
      "_boost" : {
        "name" : "_boost"
      },
      "dynamic" : true,
      "enabled" : true,
      "date_formats" : [ "dateOptionalTime", "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd" ],
      "_source" : {
        "enabled" : true,
        "name" : "_source"
      },
      "_id" : {
        "store" : "no"
      },
      "path" : "full",
      "properties" : {
        "string" : {
          "omit_term_freq_and_positions" : false,
          "index_name" : "string",
          "index" : "not_analyzed",
          "omit_norms" : false,
          "store" : "yes",
          "boost" : 1.0,
          "term_vector" : "no",
          "type" : "string"
        },
        "date" : {
          "omit_term_freq_and_positions" : true,
          "index_name" : "date",
          "index" : "not_analyzed",
          "omit_norms" : true,
          "store" : "yes",
          "boost" : 1.0,
          "format" : "dateOptionalTime",
          "precision_step" : 4,
          "term_vector" : "no",
          "type" : "date"
        }
      },
      "_all" : {
        "enabled" : true,
        "store" : "no",
        "term_vector" : "no"
      },
      "type" : "object"
    }
  }
}
Indexing
{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ {
      "_index" : "foo",
      "_type" : "bar",
      "_id" : "1",
      "_score" : 1.0,
      "fields" : {
        "date" : "2010-01-01T00:00:00.000Z",
        "string" : "one"
      }
    } ]
  }
}
Search with script_fields

{
  "_shards" : {
    "total" : 1,
    "successful" : 0,
    "failed" : 1,
    "failures" : [ {
      "reason" : "CompileException[[Error: null]\n[Near : {... Unknown ....}]\n             ^\n[Line: 1, Column: 0]]; nested: "
    } ]
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ ]
  }
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata         ] [Starshine] [foo] deleting index
[11:45:12,818][INFO ][cluster.metadata         ] [Starshine] [foo] creating index, cause [api], shards [1]/[0], mappings []
[11:45:12,994][INFO ][cluster.metadata         ] [Starshine] [foo] create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type       ] [Starshine] [13] Failed to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
             ^
[Line: 1, Column: 0]
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more


Should I open an issue?

Regards,
Lukas











Reply | Threaded
Open this post in threaded view
|

Re: script_fields issue?

kimchy
Administrator
Agreed. Open an issue, it should be simple to provide complete script based sorting. Note that sorting get much more complicated in distributed search since it need to happen twice, both on each shard, and then on the "reduce" phase.

-shay.banon

On Fri, Jul 30, 2010 at 10:40 AM, Lukáš Vlček <[hidden email]> wrote:
I will do the sorting on the client side because the set of document I need to sort is small (<30 documents in most cases). I just got impression that ES could have done it for me in simple and elegant way (again, the pref would not be a problem due to small number of documents in the set to sort).

However, it seems to me that there is a room for improvement when it comes to script fields and sorting, and may be sorting in general because it could be useful to allow users to provide custom sort function as well (FieldComparatorSource), a nice-to-have feature for some next release :-). 

I red the script_fields docs again and I think I understand now what the note regarding sorting and faceting means (it means that if the document field have been already used for faceting or sorting then there is no additional memory cost when used also in script_fields).

Regards,
Lukas

2010/7/30 Shay Banon <[hidden email]>

I meant possible to implement ;).

I had another look at your problem. Maybe one way to solve it, if you know in advance the maximum number of levels you will have in the string field, is to have empty markers for empty levels, that will still perserve sorting? Adding multiple sort scripts feature can be done, but it will be slower. First, because score based sorting is faster (which you get with custom score query), and second because the mentioned scripts will need to be evaluated for every doc the query hits in order to do the sorting correctly. This can be mitigated by having more shards and more machines, basically resulting in distributing the search load across machines (if the perf is really a problem).

-shay.banon


On Fri, Jul 30, 2010 at 12:15 AM, Shay Banon <[hidden email]> wrote:
Script fields are not using any mappings. It pretty simple what they do, you get 10 hits, for that 10 hits (docs), the script fields are evaluated. What you want is to have the ability to sort based on several scripts, which is possible.

-shay.banon


On Thu, Jul 29, 2010 at 10:09 PM, Lukáš Vlček <[hidden email]> wrote:
I can not do this at index time because I do not know the length of the string chain in advance.
Take for example document related to A | 0 entity. How can I know that I want to right-padd it to A - . - . | 0 without knowing that there are other related documents having A - B - C ?
As of know it seems that it will be easier to handle that logic on the client side (I will sort document by client, in my case there won't be a lot of related documents, I assume up to 30 max).

However, I am still curious if it is possible (or planned) to allow sort on script_fields. As of now it seems to be possible only if explicit mapping is provided for script_fields (which means it lacks a lot of flexibility considering that script_fields are defined per query while mapping per index).

Regards,
Lukas

2010/7/29 Shay Banon <[hidden email]>

I would, in this case, add another field to the document you index that does the mentioned logic, index it, and then sort on it.


On Thu, Jul 29, 2010 at 8:22 PM, Lukáš Vlček <[hidden email]> wrote:
I have the following five entities:

A - D | 2
A - B - C | 3
A | 0
A - B | 1
A - E | 4

There the pattern before pipe is some ordered chain of string symbols. The pattern after the pipe is a number. Now I want to get the following order:

A | 0
A - B | 1
A - B - C | 3
A - D | 2
A - E | 4

In fact the part before the pipe is content of multivalued filed (string) and the part after pipe is a numeric filed.

It would be easy to get desired order if we could sort by the string chain first and then by the number value. But because simple string sort would not work on the string chain (because of the variable length and different number of items in it). Then I think it can be easily done using scritpt_values (the number of items in string chain - in this case 3 - is different for each query). In this case I would built four script fileds f1-4 (three for string and one for number). The result would be (for original order of entities, I am simply right-padding the chain of strings):

f1  f2  f3  f4
-------------
A - D - . | 2
A - B - C | 3
A - . - . | 0
A - B - . | 1
A - E - . | 4

Now we can simply "sort":[f1,f2,f3,f4] and get:

A - . - . | 0
A - B - . | 1
A - B - C | 3
A - D - . | 2
A - E - . | 4

I don't think it is that easy to get this by custom scoring. Is it?
Moreover, docs for script_fields http://www.elasticsearch.com/docs/elasticsearch/rest_api/search/script_fields/ made me think that sorting can be used with them.

(Note: In my example it is not really needed to sort by the number value but that is because I simplified it way too much)

Regards,
Lukas

On Thu, Jul 29, 2010 at 6:28 PM, Shay Banon <[hidden email]> wrote:
I am not sure that you understood what script fields do. They basically provide a way to fetch custom script evaluation values for hits returned. They are not related to sorting. If you want to control sorting based on scripts, the simplest way is to use the custom score query.

Other than that, I got a bit confused, did you have problems with sorting fields (they don't have to be stored).

-shay.banon


On Thu, Jul 29, 2010 at 6:14 PM, Lukáš Vlček <[hidden email]> wrote:
Or may be the "return_type" should make more sense to be added in "sort" portion of the query rather then in "script_fields" portion.

Lukas


On Thu, Jul 29, 2010 at 4:52 PM, Lukáš Vlček <[hidden email]> wrote:
So, I found that the SortField needs field type here: http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118

That is why it is looking for mappings.

OK, I tried to provide mapping for script_fields:
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" },
    "test1" : { "type" : "long" },
    "test2" : { "type" : "string" }
  }
}}'

Now, "sort" : ["test1"] (test1 is date converted to millis) works (note: I was not able to make it reverse:true or desc) but "sort" : ["test2"] still fails with simple message:
{
  "error" : "ReduceSearchPhaseException[Failed to execute phase [query], [reduce] ]; nested: "
}
(no error in server log).

So, sorting on script_fields does not seem to work without issues to me. Second issue is that I do not want to provide explicit mapping in advance (fixing it to specific type) for script_fields. I think this is little against ES approach because script_fields can be defined per query.

Would it be possible to ask mvel for return type for given script in order to determine type for sort function on the fly?

Or how about if the client can provide the type directly as part of script_fields definiton per query. Something like:
"script_fields" : {
    "test1" : {
        "script" : "_some_fancy_math_",
        "params" : { ... },
        "return_type" : "long"
    }
}

Regards,
Lukas



On Thu, Jul 29, 2010 at 3:33 PM, Lukáš Vlček <[hidden email]> wrote:
Oh yeah, that's it.
*BUT*, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d @script_fields.json

I put the following query into script_fields.json file:
{
  "from" : 0, "size" : 3,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  },
  "sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by them (note the "sort" part) it fails:
{
  "error" : "SearchPhaseExecutionException[Failed to execute phase [query], total failure; shardFailures {[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  \"from\" : 0, \"size\" : 3,  \"fields\" : [\"date\",\"string\"],  \"query\" : { \"match_all\" : { } },  \"script_fields\" : {     \"test1\" : { \"script\" : \"doc['date'].date.millis\" },    \"test2\" : { \"script\" : \"doc['string'].stringValue\" }  },  \"sort\" : [\"test1\"]}]]]; nested: SearchParseException[[foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type       ] [Starshine] [foo][0], node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.search.SearchRequest@406039a4]
org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [Failed to parse [{  "from" : 0, "size" : 3,  "fields" : ["date","string"],  "query" : { "match_all" : { } },  "script_fields" : {     "test1" : { "script" : "doc['date'].date.millis" },    "test2" : { "script" : "doc['string'].stringValue" }  },  "sort" : ["test1"]}]]
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException: [foo][0]: query[*:*],from[0],size[3]: Parse Failure [No mapping found for [test1]]
at org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on scripted field?

Regards,
Lukas


On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon <[hidden email]> wrote:
Hi,

  I think the problem is with curl, and the fact that within the body of the request the ' is used (within the script). It needs to be escaped somehow (not sure how).

-shay.banon


On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček <[hidden email]> wrote:
Hi,

I am experiencing script_fields issue. Below is a shell script with curl examples. In short: I prepare an index, index three objects into it, verify that I can query objects from index and that I can extract values of particular "fields" (though script_fields does not require fields being stored) and then use a script_fields query which fails. Not sure what is wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : { number_of_shards : 1, number_of_replicas : 0 }}'
{ "bar" : {
  "properties" : {
    "date" : { "type" : "date", "store" : "yes" },
    "string" : { "type" : "string", "store" : "yes", "index" : "not_analyzed" }
  }
}}'
echo 

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d '{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d '{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d '{"date":"2010-01-03","string":"three"}'
echo

echo Search
{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

{
  "from" : 0, "size" : 1,
  "fields" : ["date","string"],
  "query" : { "match_all" : { } },
  "script_fields" : { 
    "test1" : { "script" : "doc['date'].date.millis" },
    "test2" : { "script" : "doc['string'].stringValue" }
  }
}'
echo

The following is output:

Index setup
{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
  "foo" : {
    "bar" : {
      "_boost" : {
        "name" : "_boost"
      },
      "dynamic" : true,
      "enabled" : true,
      "date_formats" : [ "dateOptionalTime", "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd" ],
      "_source" : {
        "enabled" : true,
        "name" : "_source"
      },
      "_id" : {
        "store" : "no"
      },
      "path" : "full",
      "properties" : {
        "string" : {
          "omit_term_freq_and_positions" : false,
          "index_name" : "string",
          "index" : "not_analyzed",
          "omit_norms" : false,
          "store" : "yes",
          "boost" : 1.0,
          "term_vector" : "no",
          "type" : "string"
        },
        "date" : {
          "omit_term_freq_and_positions" : true,
          "index_name" : "date",
          "index" : "not_analyzed",
          "omit_norms" : true,
          "store" : "yes",
          "boost" : 1.0,
          "format" : "dateOptionalTime",
          "precision_step" : 4,
          "term_vector" : "no",
          "type" : "date"
        }
      },
      "_all" : {
        "enabled" : true,
        "store" : "no",
        "term_vector" : "no"
      },
      "type" : "object"
    }
  }
}
Indexing
{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ {
      "_index" : "foo",
      "_type" : "bar",
      "_id" : "1",
      "_score" : 1.0,
      "fields" : {
        "date" : "2010-01-01T00:00:00.000Z",
        "string" : "one"
      }
    } ]
  }
}
Search with script_fields

{
  "_shards" : {
    "total" : 1,
    "successful" : 0,
    "failed" : 1,
    "failures" : [ {
      "reason" : "CompileException[[Error: null]\n[Near : {... Unknown ....}]\n             ^\n[Line: 1, Column: 0]]; nested: "
    } ]
  },
  "hits" : {
    "total" : 3,
    "max_score" : 1.0,
    "hits" : [ ]
  }
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata         ] [Starshine] [foo] deleting index
[11:45:12,818][INFO ][cluster.metadata         ] [Starshine] [foo] creating index, cause [api], shards [1]/[0], mappings []
[11:45:12,994][INFO ][cluster.metadata         ] [Starshine] [foo] create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type       ] [Starshine] [13] Failed to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
             ^
[Line: 1, Column: 0]
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more


Should I open an issue?

Regards,
Lukas