diff --git a/docs/elasticsearch.txt b/docs/elasticsearch.txt index 771c1797d5..152c1510a6 100644 --- a/docs/elasticsearch.txt +++ b/docs/elasticsearch.txt @@ -11,7 +11,7 @@ Elasticsearch is a flexible and powerful open source, distributed, real-time sea Titan supports http://elasticsearch.org[Elasticsearch] as an index backend. Here are some of the Elasticsearch features supported by Titan: * *Full-Text*: Supports all `Text` predicates to search for text properties that matches a given word, prefix or regular expression. -* *Geo*: Supports the `Geo.WITHIN` condition to search for points that fall within a given circle. Only supports points for indexing and circles for querying. +* *Geo*: Supports all `Geo` predicates to search for geo properties that are intersecting, within, disjoint to or contained in a given query geometry. Supports points, lines and polygons for indexing. Supports circles, boxes and polygons for querying point properties and all shapes for querying non-point properties. * *Numeric Range*: Supports all numeric comparisons in `Compare`. * *Flexible Configuration*: Supports embedded or remote operation, custom transport and discovery, and open-ended settings customization. * *TTL*: Supports automatically expiring indexed elements. @@ -404,7 +404,7 @@ Classpath or Field errors When you see exception referring to lucene implementation details, make sure you don't have a conflicting version of Lucene on the classpath. Exception may look like this: [source, text] -java.lang.NoSuchFieldError: LUCENE_4_10_4 +java.lang.NoSuchFieldError: LUCENE_5_5_2 Optimizing Elasticsearch ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/hadoop.txt b/docs/hadoop.txt index 1973149708..c63d97ecd9 100644 --- a/docs/hadoop.txt +++ b/docs/hadoop.txt @@ -85,6 +85,7 @@ giraph.maxMessagesInMemory=100000 spark.master=local[*] spark.executor.memory=1g spark.serializer=org.apache.spark.serializer.KryoSerializer +spark.kryo.registrator=com.thinkaurelius.titan.hadoop.serialize.TitanKryoRegistrator ---- [source, gremlin] diff --git a/docs/lucene.txt b/docs/lucene.txt index 96c9cc2d85..07e2d81e6e 100644 --- a/docs/lucene.txt +++ b/docs/lucene.txt @@ -27,7 +27,7 @@ Feature Support ~~~~~~~~~~~~~~~ * *Full-Text*: Supports all `Text` predicates to search for text properties that matches a given word, prefix or regular expression. -* *Geo*: Supports the `Geo.WITHIN` condition to search for points that fall within a given geographic shape. Only supports points for indexing and circles and boxes for querying. +* *Geo*: Supports `Geo` predicates to search for geo properties that are intersecting, within, or contained in a given query geometry. Supports points, lines and polygons for indexing. Supports circles and boxes for querying point properties and all shapes for querying non-point properties. * *Numeric Range*: Supports all numeric comparisons in `Compare`. * *Temporal*: Nanosecond granularity temporal indexing. diff --git a/docs/searchpredicates.txt b/docs/searchpredicates.txt index ac83c504af..87baf69c21 100644 --- a/docs/searchpredicates.txt +++ b/docs/searchpredicates.txt @@ -34,8 +34,14 @@ See <> for more information about full-text and string search. Geo Predicate ~~~~~~~~~~~~~ -The `Geo` enum specifies the geo-location predicate `geoWithin` which holds true if one geometric object contains the other. +The `Geo` enum specifies geo-location predicates. +* `geoIntersect` which holds true if the two geometric objects have at least one point in common (opposite of `geoDisjoint`). +* `geoWithin` which holds true if one geometric object contains the other. +* `geoDisjoint` which holds true if the two geometric objects have no points in common (opposite of `geoIntersect`). +* `geoContains` which holds true if one geometric object is contained by the other. + +See <> for more information about geo search. Query Examples ~~~~~~~~~~~~~~ @@ -88,7 +94,7 @@ Additional data types will be supported in the future. Geoshape Data Type ~~~~~~~~~~~~~~~~~~ -The Geoshape data type supports representing a point, circle or box. However all index backends currently only support indexing points. +The Geoshape data type supports representing a point, circle, box, line, polygon, multi-point, multi-line and multi-polygon. Index backends currently support indexing points, lines and polygons. Indexing multi-point, multi-line and multi-polygon properties has not been tested. Geospatial index lookups are only supported via mixed indexes. To construct a Geoshape use the following methods: @@ -100,8 +106,14 @@ Geoshape.point(37.97, 23.72) Geoshape.circle(37.97, 23.72, 50) //SW lat, SW lng, NE lat, NE lng Geoshape.box(37.97, 23.72, 38.97, 24.72) +//lat1,lng1,lat2,lng2,... +Geoshape.line(37.97, 23.72, 37.97, 24.72, 38.97, 24.72, 38.97, 23.72) +Geoshape.polygon(37.97, 23.72, 37.97, 24.72, 38.97, 24.72, 38.97, 23.72, 37.97, 23.72) + +Additional Geoshape constructors for building lines and polygons from a list of coordinate pairs, Spatial4j Shape or JTS Geometry are also available. +Note that, unlike above, the coordinate order is (lon,lat) when providing a list of coordinate pairs. -In addition when importing a graph via GraphSON Point may be represented by: +In addition, when importing a graph via GraphSON the geometry may be represented by GeoJSON: [source, java] //string "37.97, 23.72" @@ -124,7 +136,7 @@ In addition when importing a graph via GraphSON Point may be represented by: "coordinates": [125.6, 10.1] } -link:http://geojson.org/[GeoJSON] may be specified as Point, Circle or Polygon. However polygons must form a box. +link:http://geojson.org/[GeoJSON] may be specified as Point, Circle, LineString or Polygon. Polygons must be closed. Note that unlike the Titan API GeoJSON specifies coordinates as lng lat. Collections diff --git a/docs/solr.txt b/docs/solr.txt index 4fb289cdb8..3b2d484eec 100644 --- a/docs/solr.txt +++ b/docs/solr.txt @@ -9,7 +9,7 @@ Solr is the popular, blazing fast open source enterprise search platform from th Titan supports http://lucene.apache.org/solr/[Solr] as an index backend. Here are some of the Solr features supported by Titan: * *Full-Text*: Supports all `Text` predicates to search for text properties that matches a given word, prefix or regular expression. -* *Geo*: Supports the `Geo.WITHIN` condition to search for points that fall within a given circle. Only supports points for indexing and circles for querying. +* *Geo*: Supports all `Geo` predicates to search for geo properties that are intersecting, within, disjoint to or contained in a given query geometry. Supports points, lines and polygons for indexing. Supports circles, boxes and polygons for querying point properties and all shapes for querying non-point properties. * *Numeric Range*: Supports all numeric comparisons in `Compare`. * *TTL*: Supports automatically expiring indexed elements. * *Temporal*: Millisecond granularity temporal indexing. diff --git a/docs/textsearch.txt b/docs/textsearch.txt index 60e4793d5d..59b3079351 100644 --- a/docs/textsearch.txt +++ b/docs/textsearch.txt @@ -84,8 +84,32 @@ summary = mgmt.makePropertyKey('booksummary').dataType(String.class).make() mgmt.buildIndex('booksBySummary', Vertex.class).addKey(summary, Mapping.TEXTSTRING.asParameter()).buildMixedIndex("search") mgmt.commit() + Note that the data will be stored in the index twice, once for exact matching and once for fuzzy matching. + +[[geo-search]] +Geo Mapping +~~~~~~~~~~~ + +By default, Titan supports indexing geo properties with point type and querying geo properties by circle or box. To index a non-point geo property with support for querying by any geoshape type, specify the mapping as `Mapping.PREFIX_TREE`: + +[source, gremlin] +mgmt = graph.openManagement() +name = mgmt.makePropertyKey('border').dataType(Geoshape.class).make() +mgmt.buildIndex('borderIndex', Vertex.class).addKey(name, Mapping.PREFIX_TREE.asParameter()).buildMixedIndex("search") +mgmt.commit() + +Additional parameters can be specified to tune the configuration of the underlying prefix tree mapping. These optional parameters include the number of levels used in the prefix tree as well as the associated precision. + +[source, gremlin] +mgmt = graph.openManagement() +name = mgmt.makePropertyKey('border').dataType(Geoshape.class).make() +mgmt.buildIndex('borderIndex', Vertex.class).addKey(name, Mapping.PREFIX_TREE.asParameter(), Parameter.of("index-geo-max-levels", 18), Parameter.of("index-geo-dist-error-pct", 0.0125)).buildMixedIndex("search") +mgmt.commit() + +Note that some indexing backends (e.g. Solr) may require additional external schema configuration to support and tune indexing non-point properties. + Field Mapping ~~~~~~~~~~~~~ diff --git a/pom.xml b/pom.xml index 06571ffa40..ec19e78758 100644 --- a/pom.xml +++ b/pom.xml @@ -84,15 +84,15 @@ 1.0.2 ${hbase100.core.version} 1.9.2 - 2.4.4 + 2.6.6 - 4.10.4 - 1.5.1 + 5.5.2 + 2.4.2 1.7.0 - 1.6.2 + 2.8.2 1.3 2.7.7 3.2 @@ -454,6 +454,11 @@ jackson-annotations ${jackson2.version} + + com.fasterxml.jackson.module + jackson-module-scala_2.10 + ${jackson2.version} + joda-time joda-time @@ -798,7 +803,7 @@ commons-cli commons-cli - 1.2 + 1.3.1 org.jboss.netty @@ -808,14 +813,14 @@ io.netty netty - 3.6.6.Final + 3.10.5.Final com.spatial4j spatial4j - 0.4.1 + 0.5 diff --git a/titan-core/pom.xml b/titan-core/pom.xml index b6307fb1cf..9079077a90 100644 --- a/titan-core/pom.xml +++ b/titan-core/pom.xml @@ -63,6 +63,11 @@ com.spatial4j spatial4j + + com.vividsolutions + jts + 1.13 + commons-collections commons-collections @@ -99,6 +104,12 @@ com.google.code.findbugs jsr305 + + + org.noggit + noggit + 0.6 + ${basedir}/target diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/core/attribute/Geo.java b/titan-core/src/main/java/com/thinkaurelius/titan/core/attribute/Geo.java index 53e2122ce9..3947b3e847 100644 --- a/titan-core/src/main/java/com/thinkaurelius/titan/core/attribute/Geo.java +++ b/titan-core/src/main/java/com/thinkaurelius/titan/core/attribute/Geo.java @@ -69,7 +69,7 @@ public TitanPredicate negate() { }, /** - * Whether one geographic region is completely contains within another + * Whether one geographic region is completely within another */ WITHIN { @Override @@ -90,6 +90,34 @@ public boolean hasNegation() { return false; } + @Override + public TitanPredicate negate() { + throw new UnsupportedOperationException(); + } + }, + + /** + * Whether one geographic region completely contains another + */ + CONTAINS { + @Override + public boolean test(Object value, Object condition) { + Preconditions.checkArgument(condition instanceof Geoshape); + if (value == null) return false; + Preconditions.checkArgument(value instanceof Geoshape); + return ((Geoshape) value).contains((Geoshape) condition); + } + + @Override + public String toString() { + return "contains"; + } + + @Override + public boolean hasNegation() { + return false; + } + @Override public TitanPredicate negate() { throw new UnsupportedOperationException(); @@ -123,4 +151,7 @@ public static P geoDisjoint(final V value) { public static P geoWithin(final V value) { return new P(Geo.WITHIN, value); } + public static P geoContains(final V value) { + return new P(Geo.CONTAINS, value); + } } diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/core/attribute/Geoshape.java b/titan-core/src/main/java/com/thinkaurelius/titan/core/attribute/Geoshape.java index 6972d2b676..838707ca0e 100644 --- a/titan-core/src/main/java/com/thinkaurelius/titan/core/attribute/Geoshape.java +++ b/titan-core/src/main/java/com/thinkaurelius/titan/core/attribute/Geoshape.java @@ -2,15 +2,30 @@ import com.google.common.base.Preconditions; import com.google.common.primitives.Doubles; -import com.spatial4j.core.context.SpatialContext; +import com.spatial4j.core.context.SpatialContextFactory; +import com.spatial4j.core.context.jts.DatelineRule; +import com.spatial4j.core.context.jts.JtsSpatialContext; +import com.spatial4j.core.context.jts.JtsSpatialContextFactory; import com.spatial4j.core.distance.DistanceUtils; +import com.spatial4j.core.io.jts.JtsBinaryCodec; +import com.spatial4j.core.io.jts.JtsGeoJSONReader; +import com.spatial4j.core.io.jts.JtsGeoJSONWriter; +import com.spatial4j.core.io.jts.JtsWKTReader; +import com.spatial4j.core.io.jts.JtsWKTWriter; +import com.spatial4j.core.shape.Circle; +import com.spatial4j.core.shape.Rectangle; import com.spatial4j.core.shape.Shape; import com.spatial4j.core.shape.SpatialRelation; +import com.spatial4j.core.shape.jts.JtsGeometry; import com.thinkaurelius.titan.diskstorage.ScanBuffer; import com.thinkaurelius.titan.diskstorage.WriteBuffer; import com.thinkaurelius.titan.graphdb.database.idhandling.VariableLong; -import com.thinkaurelius.titan.graphdb.relations.RelationIdentifier; -import org.apache.commons.lang.builder.HashCodeBuilder; +import com.vividsolutions.jts.geom.Coordinate; +import com.vividsolutions.jts.geom.Geometry; +import com.vividsolutions.jts.geom.GeometryFactory; +import com.vividsolutions.jts.geom.Polygon; + +import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONTokens; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONUtil; @@ -18,6 +33,7 @@ import org.apache.tinkerpop.shaded.jackson.core.JsonParser; import org.apache.tinkerpop.shaded.jackson.core.JsonProcessingException; import org.apache.tinkerpop.shaded.jackson.databind.DeserializationContext; +import org.apache.tinkerpop.shaded.jackson.databind.ObjectMapper; import org.apache.tinkerpop.shaded.jackson.databind.SerializerProvider; import org.apache.tinkerpop.shaded.jackson.databind.deser.std.StdDeserializer; import org.apache.tinkerpop.shaded.jackson.databind.jsontype.TypeSerializer; @@ -27,37 +43,64 @@ import org.apache.tinkerpop.shaded.kryo.io.Input; import org.apache.tinkerpop.shaded.kryo.io.Output; +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.StringReader; import java.lang.reflect.Array; +import java.text.ParseException; import java.util.*; import java.util.stream.Collectors; /** * A generic representation of a geographic shape, which can either be a single point, - * circle, box, or polygon. Use {@link #getType()} to determine the type of shape of a particular Geoshape object. + * circle, box, line or polygon. Use {@link #getType()} to determine the type of shape of a particular Geoshape object. * Use the static constructor methods to create the desired geoshape. * - * Note, polygons are not yet supported. - * * @author Matthias Broecheler (me@matthiasb.com) */ public class Geoshape { + private static String FIELD_LABEL = "geometry"; private static String FIELD_TYPE = "type"; private static String FIELD_COORDINATES = "coordinates"; private static String FIELD_RADIUS = "radius"; - private static final SpatialContext CTX = SpatialContext.GEO; + public static final JtsSpatialContext CTX; + static { + JtsSpatialContextFactory factory = new JtsSpatialContextFactory(); + factory.geo = true; + factory.useJtsPoint = false; + factory.useJtsLineString = true; + // TODO: Use default dateline rule and update to support multiline/polygon to resolve wrapping issues + factory.datelineRule = DatelineRule.none; + CTX = new JtsSpatialContext(factory); + } + + private static JtsWKTReader wktReader = new JtsWKTReader(CTX, new JtsSpatialContextFactory()); + + private static JtsWKTWriter wktWriter = new JtsWKTWriter(CTX, new JtsSpatialContextFactory()); + + private static JtsGeoJSONReader geojsonReader = new JtsGeoJSONReader(CTX, new SpatialContextFactory()); + + private static JtsGeoJSONWriter geojsonWriter = new JtsGeoJSONWriter(CTX, new SpatialContextFactory()); /** - * The Type of a shape: a point, box, circle, or polygon. + * The Type of a shape: a point, box, circle, line or polygon. */ public enum Type { POINT("Point"), BOX("Box"), CIRCLE("Circle"), - POLYGON("Polygon"); + LINE("Line"), + POLYGON("Polygon"), + MULTIPOINT("MultiPoint"), + MULTILINESTRING("MultiLineString"), + MULTIPOLYGON("MultiPolygon"); private final String gsonName; @@ -79,26 +122,16 @@ public String toString() { } } - //coordinates[0] = latitudes, coordinates[1] = longitudes - private final float[][] coordinates; + private final Shape shape; - private Geoshape() { - coordinates = null; - } - - private Geoshape(final float[][] coordinates) { - Preconditions.checkArgument(coordinates!=null && coordinates.length==2); - Preconditions.checkArgument(coordinates[0].length==coordinates[1].length && coordinates[0].length>0); - for (int i=0;i0); - else Preconditions.checkArgument(isValidCoordinate(coordinates[0][i],coordinates[1][i])); - } - this.coordinates=coordinates; + private Geoshape(final Shape shape) { + Preconditions.checkNotNull(shape,"Invalid shape (null)"); + this.shape = shape; } @Override public int hashCode() { - return new HashCodeBuilder().append(coordinates[0]).append(coordinates[1]).toHashCode(); + return shape.hashCode(); } @Override @@ -107,38 +140,32 @@ public boolean equals(Object other) { else if (other==null) return false; else if (!getClass().isInstance(other)) return false; Geoshape oth = (Geoshape)other; - Preconditions.checkArgument(coordinates.length==2 && oth.coordinates.length==2); - for (int i=0;i0) s.append(","); - s.append(getPoint(i)); - } - s.append("]"); - } - return s.toString(); + return wktWriter.toString(shape); + } + + /** + * Returns the GeoJSON representation of the shape. + * @return + */ + public String toGeoJson() { + return GeoshapeGsonSerializer.toGeoJson(this); + } + + /** + * Returns the underlying {@link Shape}. + * @return + */ + public Shape getShape() { + return shape; } /** @@ -147,17 +174,31 @@ public String toString() { * @return */ public Type getType() { - if (coordinates[0].length==1) return Type.POINT; - else if (coordinates[0].length>2) return Type.POLYGON; - else { //coordinates[0].length==2 - if (Float.isNaN(coordinates[0][1])) return Type.CIRCLE; - else return Type.BOX; + final Type type; + if (com.spatial4j.core.shape.Point.class.isAssignableFrom(shape.getClass())) { + type = Type.POINT; + } else if (Circle.class.isAssignableFrom(shape.getClass())) { + type = Type.CIRCLE; + } else if (Rectangle.class.isAssignableFrom(shape.getClass())) { + type = Type.BOX; + } else if (JtsGeometry.class.isAssignableFrom(shape.getClass()) + && "LineString".equals(((JtsGeometry) shape).getGeom().getGeometryType())) { + type = Type.LINE; + } else if (JtsGeometry.class.isAssignableFrom(shape.getClass())) { + try { + type = Type.fromGson((((JtsGeometry) shape).getGeom().getGeometryType())); + } catch (IllegalArgumentException e) { + throw new IllegalStateException("Unrecognized shape type"); + } + } else { + throw new IllegalStateException("Unrecognized shape type"); } + return type; } /** * Returns the number of points comprising this geoshape. A point and circle have only one point (center of cricle), - * a box has two points (the south-west and north-east corners) and a polygon has a variable number of points (>=3). + * a box has two points (the south-west and north-east corners). Lines and polygons have a variable number of points. * * @return */ @@ -166,8 +207,9 @@ public int size() { case POINT: return 1; case CIRCLE: return 1; case BOX: return 2; - case POLYGON: return coordinates[0].length; - default: throw new IllegalStateException("Unrecognized type: " + getType()); + case LINE: + case POLYGON: return ((JtsGeometry) shape).getGeom().getCoordinates().length; + default: throw new IllegalStateException("size() not supported for type: " + getType()); } } @@ -179,7 +221,22 @@ public int size() { */ public Point getPoint(int position) { if (position<0 || position>=size()) throw new ArrayIndexOutOfBoundsException("Invalid position: " + position); - return new Point(coordinates[0][position],coordinates[1][position]); + switch(getType()) { + case POINT: + case CIRCLE: + return getPoint(); + case BOX: + if (position == 0) + return new Point(shape.getBoundingBox().getMinY(), shape.getBoundingBox().getMinX()); + else + return new Point(shape.getBoundingBox().getMaxY(), shape.getBoundingBox().getMaxX()); + case LINE: + case POLYGON: + Coordinate coordinate = ((JtsGeometry) shape).getGeom().getCoordinates()[position]; + return new Point(coordinate.y, coordinate.x); + default: + throw new IllegalStateException("getPoint(int) not supported for type: " + getType()); + } } /** @@ -188,63 +245,63 @@ public Point getPoint(int position) { * @return */ public Point getPoint() { - Preconditions.checkArgument(size()==1,"Shape does not have a single point"); - return getPoint(0); + Preconditions.checkArgument(getType()==Type.POINT || getType()==Type.CIRCLE,"Shape does not have a single point"); + return new Point(shape.getCenter().getY(), shape.getCenter().getX()); } /** * Returns the radius in kilometers of this circle. Only applicable to circle shapes. * @return */ - public float getRadius() { + public double getRadius() { Preconditions.checkArgument(getType()==Type.CIRCLE,"This shape is not a circle"); - return coordinates[1][1]; + double radiusInDeg = ((Circle) shape).getRadius(); + return DistanceUtils.degrees2Dist(radiusInDeg, DistanceUtils.EARTH_MEAN_RADIUS_KM); } private SpatialRelation getSpatialRelation(Geoshape other) { Preconditions.checkNotNull(other); - return convert2Spatial4j().relate(other.convert2Spatial4j()); + return shape.relate(other.shape); } + /** + * Whether this geometry has any points in common with the given geometry. + * @param other + * @return + */ public boolean intersect(Geoshape other) { SpatialRelation r = getSpatialRelation(other); return r==SpatialRelation.INTERSECTS || r==SpatialRelation.CONTAINS || r==SpatialRelation.WITHIN; } + /** + * Whether this geometry is within the given geometry. + * @param outer + * @return + */ public boolean within(Geoshape outer) { return getSpatialRelation(outer)==SpatialRelation.WITHIN; } - public boolean disjoint(Geoshape other) { - return getSpatialRelation(other)==SpatialRelation.DISJOINT; - } - /** - * Converts this shape into its equivalent Spatial4j {@link Shape}. + * Whether this geometry contains the given geometry. + * @param outer * @return */ - public Shape convert2Spatial4j() { - switch(getType()) { - case POINT: return getPoint().getSpatial4jPoint(); - case CIRCLE: return CTX.makeCircle(getPoint(0).getSpatial4jPoint(), DistanceUtils.dist2Degrees(getRadius(), DistanceUtils.EARTH_MEAN_RADIUS_KM)); - case BOX: return CTX.makeRectangle(getPoint(0).getSpatial4jPoint(),getPoint(1).getSpatial4jPoint()); - case POLYGON: throw new UnsupportedOperationException("Not yet supported"); - default: throw new IllegalStateException("Unrecognized type: " + getType()); - } + public boolean contains(Geoshape outer) { + return getSpatialRelation(outer)==SpatialRelation.CONTAINS; } - /** - * Constructs a point from its latitude and longitude information - * @param latitude - * @param longitude + * Whether this geometry has no points in common with the given geometry. + * @param other * @return */ - public static final Geoshape point(final float latitude, final float longitude) { - Preconditions.checkArgument(isValidCoordinate(latitude,longitude),"Invalid coordinate provided"); - return new Geoshape(new float[][]{ new float[]{latitude}, new float[]{longitude}}); + public boolean disjoint(Geoshape other) { + return getSpatialRelation(other)==SpatialRelation.DISJOINT; } + /** * Constructs a point from its latitude and longitude information * @param latitude @@ -252,20 +309,8 @@ public static final Geoshape point(final float latitude, final float longitude) * @return */ public static final Geoshape point(final double latitude, final double longitude) { - return point((float)latitude,(float)longitude); - } - - /** - * Constructs a circle from a given center point and a radius in kilometer - * @param latitude - * @param longitude - * @param radiusInKM - * @return - */ - public static final Geoshape circle(final float latitude, final float longitude, final float radiusInKM) { Preconditions.checkArgument(isValidCoordinate(latitude,longitude),"Invalid coordinate provided"); - Preconditions.checkArgument(radiusInKM>0,"Invalid radius provided [%s]",radiusInKM); - return new Geoshape(new float[][]{ new float[]{latitude, Float.NaN}, new float[]{longitude, radiusInKM}}); + return new Geoshape(CTX.makePoint(longitude, latitude)); } /** @@ -276,7 +321,10 @@ public static final Geoshape circle(final float latitude, final float longitude, * @return */ public static final Geoshape circle(final double latitude, final double longitude, final double radiusInKM) { - return circle((float)latitude,(float)longitude,(float)radiusInKM); + Preconditions.checkArgument(isValidCoordinate(latitude,longitude),"Invalid coordinate provided"); + Preconditions.checkArgument(radiusInKM>0,"Invalid radius provided [%s]",radiusInKM); + double radius = DistanceUtils.dist2Degrees(radiusInKM, DistanceUtils.EARTH_MEAN_RADIUS_KM); + return new Geoshape(CTX.makeCircle(longitude, latitude, radius)); } /** @@ -287,24 +335,102 @@ public static final Geoshape circle(final double latitude, final double longitud * @param northEastLongitude * @return */ - public static final Geoshape box(final float southWestLatitude, final float southWestLongitude, - final float northEastLatitude, final float northEastLongitude) { + public static final Geoshape box(final double southWestLatitude, final double southWestLongitude, + final double northEastLatitude, final double northEastLongitude) { Preconditions.checkArgument(isValidCoordinate(southWestLatitude,southWestLongitude),"Invalid south-west coordinate provided"); Preconditions.checkArgument(isValidCoordinate(northEastLatitude,northEastLongitude),"Invalid north-east coordinate provided"); - return new Geoshape(new float[][]{ new float[]{southWestLatitude, northEastLatitude}, new float[]{southWestLongitude, northEastLongitude}}); + return new Geoshape(CTX.makeRectangle(southWestLongitude, northEastLongitude, southWestLatitude, northEastLatitude)); } /** - * Constructs a new box shape which is identified by its south-west and north-east corner points - * @param southWestLatitude - * @param southWestLongitude - * @param northEastLatitude - * @param northEastLongitude + * Constructs a new line shape which is identified by its coordinates + * @param coordinates Sequence of coordinates (lat1,lon1,...,latN,lonN) * @return */ - public static final Geoshape box(final double southWestLatitude, final double southWestLongitude, - final double northEastLatitude, final double northEastLongitude) { - return box((float)southWestLatitude,(float)southWestLongitude,(float)northEastLatitude,(float)northEastLongitude); + public static final Geoshape line(final double ... coordinates) { + Preconditions.checkArgument(coordinates.length % 2 == 0, "Odd number of coordinates provided"); + Preconditions.checkArgument(coordinates.length >= 4, "Too few coordinate pairs provided"); + List points = new ArrayList<>(); + for (int i = 0; i < coordinates.length; i+=2) { + points.add(new double[] {coordinates[i+1], coordinates[i]}); + } + return line(points); + } + + /** + * Constructs a line from list of coordinates + * @param coordinates Coordinate (lon,lat) pairs + * @return + */ + public static final Geoshape line(List coordinates) { + Preconditions.checkArgument(coordinates.size() >= 2, "Too few coordinate pairs provided"); + List points = new ArrayList<>(); + for (double[] coordinate : coordinates) { + Preconditions.checkArgument(isValidCoordinate(coordinate[1],coordinate[0]),"Invalid coordinate provided"); + points.add(CTX.makePoint(coordinate[0], coordinate[1])); + } + return new Geoshape(CTX.makeLineString(points)); + } + + /** + * Constructs a new polygon shape which is identified by its coordinates + * @param coordinates Sequence of coordinates (lat1,lon1,...,latN,lonN) + * @return + */ + public static final Geoshape polygon(final double ... coordinates) { + Preconditions.checkArgument(coordinates.length % 2 == 0, "Odd number of coordinates provided"); + Preconditions.checkArgument(coordinates.length >= 6, "Too few coordinate pairs provided"); + List points = new ArrayList<>(); + for (int i = 0; i < coordinates.length; i+=2) { + points.add(new double[] {coordinates[i+1], coordinates[i]}); + } + return polygon(points); + } + + /** + * Constructs a polygon from list of coordinates + * @param coordinates Coordinate (lon,lat) pairs + * @return + */ + public static final Geoshape polygon(List coordinates) { + Preconditions.checkArgument(coordinates.size() >= 3, "Too few coordinate pairs provided"); + Coordinate[] points = new Coordinate[coordinates.size()]; + for (int i=0; i= 2, "Too few coordinates in pair"); + Preconditions.checkArgument(isValidCoordinate(coordinates.get(i)[1],coordinates.get(i)[0]),"Invalid coordinate provided"); + points[i] = new Coordinate(coordinates.get(i)[0], coordinates.get(i)[1]); + } + + Polygon polygon = new GeometryFactory().createPolygon(points); + return new Geoshape(CTX.makeShape(polygon)); + } + + /** + * Constructs a Geoshape from a JTS {@link Geometry}. + * @param geometry + * @return + */ + public static final Geoshape geoshape(Geometry geometry) { + return new Geoshape(CTX.makeShape(geometry)); + } + + /** + * Constructs a Geoshape from a spatial4j {@link Shape}. + * @param shape + * @return + */ + public static final Geoshape geoshape(Shape shape) { + return new Geoshape(shape); + } + + /** + * Create Geoshape from WKT representation. + * @param wkt + * @return + * @throws ParseException + */ + public static final Geoshape fromWkt(String wkt) throws ParseException { + return new Geoshape(wktReader.parse(wkt)); } /** @@ -313,7 +439,7 @@ public static final Geoshape box(final double southWestLatitude, final double so * @param longitude * @return */ - public static final boolean isValidCoordinate(final float latitude, final float longitude) { + public static final boolean isValidCoordinate(final double latitude, final double longitude) { return latitude>=-90.0 && latitude<=90.0 && longitude>=-180.0 && longitude<=180.0; } @@ -323,15 +449,15 @@ public static final boolean isValidCoordinate(final float latitude, final float */ public static final class Point { - private final float longitude; - private final float latitude; + private final double longitude; + private final double latitude; /** * Constructs a point with the given latitude and longitude * @param latitude Between -90 and 90 degrees * @param longitude Between -180 and 180 degrees */ - Point(float latitude, float longitude) { + Point(double latitude, double longitude) { this.longitude = longitude; this.latitude = latitude; } @@ -340,7 +466,7 @@ public static final class Point { * Longitude of this point * @return */ - public float getLongitude() { + public double getLongitude() { return longitude; } @@ -348,7 +474,7 @@ public float getLongitude() { * Latitude of this point * @return */ - public float getLatitude() { + public double getLatitude() { return latitude; } @@ -366,28 +492,10 @@ public double distance(Point other) { return DistanceUtils.degrees2Dist(CTX.getDistCalc().distance(getSpatial4jPoint(),other.getSpatial4jPoint()),DistanceUtils.EARTH_MEAN_RADIUS_KM); } - @Override - public String toString() { - return "["+latitude+","+longitude+"]"; - } - - @Override - public int hashCode() { - return new HashCodeBuilder().append(latitude).append(longitude).toHashCode(); - } - - @Override - public boolean equals(Object other) { - if (this==other) return true; - else if (other==null) return false; - else if (!getClass().isInstance(other)) return false; - Point oth = (Point)other; - return latitude==oth.latitude && longitude==oth.longitude; - } - } /** + * Geoshape attribute serializer for Titan. * @author Matthias Broecheler (me@matthiasb.com) */ public static class GeoshapeSerializer implements AttributeSerializer { @@ -456,24 +564,20 @@ private Geoshape convertGeoJson(Object value) { try { Map map = (Map) value; String type = (String) map.get("type"); - if("Point".equals(type) || "Circle".equals(type) || "Polygon".equals(type)) { - return convertGeometry(map); - } - else if("Feature".equals(type)) { + if("Feature".equals(type)) { Map geometry = (Map) map.get("geometry"); return convertGeometry(geometry); + } else { + return convertGeometry(map); } - throw new IllegalArgumentException("Only Point, Circle, Polygon or feature types are supported"); - } catch (ClassCastException e) { + } catch (ClassCastException | IOException | ParseException e) { throw new IllegalArgumentException("GeoJSON was unparsable"); } - } - private Geoshape convertGeometry(Map geometry) { + private Geoshape convertGeometry(Map geometry) throws IOException, ParseException { String type = (String) geometry.get("type"); List coordinates = (List) geometry.get("coordinates"); - //Either this is a single point or a collection of points if ("Point".equals(type)) { double[] parsedCoordinates = convertCollection(coordinates); @@ -486,26 +590,23 @@ private Geoshape convertGeometry(Map geometry) { double[] parsedCoordinates = convertCollection(coordinates); return circle(parsedCoordinates[1], parsedCoordinates[0], radius.doubleValue()); } else if ("Polygon".equals(type)) { - if (coordinates.size() != 4) { - throw new IllegalArgumentException("GeoJSON polygons are only supported if they form a box"); - } - List polygon = (List) coordinates.stream().map(o -> convertCollection((Collection) o)).collect(Collectors.toList()); - - double[] p0 = polygon.get(0); - double[] p1 = polygon.get(1); - double[] p2 = polygon.get(2); - double[] p3 = polygon.get(3); - - //This may be a clockwise or counterclockwise polygon, we have to verify that it is a box - if ((p0[0] == p1[0] && p1[1] == p2[1] && p2[0] == p3[0] && p3[1] == p0[1]) || - (p0[1] == p1[1] && p1[0] == p2[0] && p2[1] == p3[1] && p3[0] == p0[0])) { - return box(min(p0[1], p1[1], p2[1], p3[1]), min(p0[0], p1[0], p2[0], p3[0]), max(p0[1], p1[1], p2[1], p3[1]), max(p0[0], p1[0], p2[0], p3[0])); + // check whether this is a box + if (coordinates.size() == 4) { + double[] p0 = convertCollection((Collection) coordinates.get(0)); + double[] p1 = convertCollection((Collection) coordinates.get(1)); + double[] p2 = convertCollection((Collection) coordinates.get(2)); + double[] p3 = convertCollection((Collection) coordinates.get(3)); + + //This may be a clockwise or counterclockwise polygon, we have to verify that it is a box + if ((p0[0] == p1[0] && p1[1] == p2[1] && p2[0] == p3[0] && p3[1] == p0[1] && p3[0] != p0[0]) || + (p0[1] == p1[1] && p1[0] == p2[0] && p2[1] == p3[1] && p3[0] == p0[0] && p3[1] != p0[1])) { + return box(min(p0[1], p1[1], p2[1], p3[1]), min(p0[0], p1[0], p2[0], p3[0]), max(p0[1], p1[1], p2[1], p3[1]), max(p0[0], p1[0], p2[0], p3[0])); + } } - - throw new IllegalArgumentException("GeoJSON polygons are only supported if they form a box"); - } else { - throw new IllegalArgumentException("GeoJSON support is restricted to Point, Circle or Polygon."); } + + String json = new ObjectMapper().writeValueAsString(geometry); + return new Geoshape(geojsonReader.read(new StringReader(json))); } private double min(double... numbers) { @@ -522,63 +623,60 @@ public Geoshape read(ScanBuffer buffer) { long l = VariableLong.readPositive(buffer); assert l>0 && l0); - int length = coordinates[0].length; - VariableLong.writePositive(buffer,length); - for (int i = 0; i < 2; i++) { - for (int j = 0; j < length; j++) { - buffer.putFloat(coordinates[i][j]); - } + try { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + GeoshapeBinarySerializer.write(outputStream, attribute); + byte[] bytes = outputStream.toByteArray(); + VariableLong.writePositive(buffer,bytes.length); + buffer.putBytes(bytes); + } catch (IOException e) { + throw new RuntimeException("I/O exception writing geoshape"); } } } /** - * Serializer for TinkerPop's Gryo. + * Geoshape serializer for TinkerPop's Gryo. */ public static class GeoShapeGryoSerializer extends Serializer { @Override public void write(Kryo kryo, Output output, Geoshape geoshape) { - float[][] coordinates = geoshape.coordinates; - assert (coordinates.length==2); - assert (coordinates[0].length==coordinates[1].length && coordinates[0].length>0); - int length = coordinates[0].length; - output.writeLong(length); - for (int i = 0; i < 2; i++) { - for (int j = 0; j < length; j++) { - output.writeFloat(coordinates[i][j]); - } + try { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + GeoshapeBinarySerializer.write(outputStream, geoshape); + byte[] bytes = outputStream.toByteArray(); + output.write(bytes.length); + output.write(bytes); + } catch (IOException e) { + throw new RuntimeException("I/O exception writing geoshape"); } } @Override public Geoshape read(Kryo kryo, Input input, Class aClass) { - long l = input.readLong(); - assert l>0 && l0; + InputStream inputStream = new ByteArrayInputStream(input.readBytes(length)); + try { + return GeoshapeBinarySerializer.read(inputStream); + } catch (IOException e) { + throw new RuntimeException("I/O exception reding geoshape"); } - return new Geoshape(coordinates); } } /** - * Serialization of Geoshape for JSON purposes uses the standard GeoJSON(http://geojson.org/) format. - * - * @author Bryn Cooke + * Geoshape serializer supports writing GeoJSON (http://geojson.org/). */ public static class GeoshapeGsonSerializer extends StdSerializer { @@ -588,84 +686,121 @@ public GeoshapeGsonSerializer() { @Override public void serialize(Geoshape value, JsonGenerator jgen, SerializerProvider provider) throws IOException, JsonProcessingException { - jgen.writeStartObject(); - jgen.writeFieldName(FIELD_TYPE); - switch(value.getType()) { - case POLYGON: - throw new UnsupportedOperationException("Polygons are not supported"); - case BOX: - jgen.writeString(Type.BOX.toString()); - jgen.writeFieldName(FIELD_COORDINATES); - jgen.writeStartArray(); - - jgen.writeStartArray(); - jgen.writeNumber(value.coordinates[1][0]); - jgen.writeNumber(value.coordinates[0][0]); - jgen.writeEndArray(); - - jgen.writeStartArray(); - jgen.writeNumber(value.coordinates[1][1]); - jgen.writeNumber(value.coordinates[0][0]); - jgen.writeEndArray(); - - jgen.writeStartArray(); - jgen.writeNumber(value.coordinates[1][1]); - jgen.writeNumber(value.coordinates[0][1]); - jgen.writeEndArray(); - - jgen.writeStartArray(); - jgen.writeNumber(value.coordinates[1][0]); - jgen.writeNumber(value.coordinates[0][1]); - jgen.writeEndArray(); - - jgen.writeEndArray(); - break; - case CIRCLE: - jgen.writeString(Type.CIRCLE.toString()); - jgen.writeFieldName(FIELD_RADIUS); - jgen.writeNumber(value.getRadius()); - jgen.writeFieldName(FIELD_COORDINATES); - jgen.writeStartArray(); - jgen.writeNumber(value.coordinates[1][0]); - jgen.writeNumber(value.coordinates[0][0]); - jgen.writeEndArray(); - break; case POINT: + jgen.writeStartObject(); + jgen.writeFieldName(FIELD_TYPE); jgen.writeString(Type.POINT.toString()); jgen.writeFieldName(FIELD_COORDINATES); jgen.writeStartArray(); - jgen.writeNumber(value.coordinates[1][0]); - jgen.writeNumber(value.coordinates[0][0]); + jgen.writeNumber(value.getPoint().getLongitude()); + jgen.writeNumber(value.getPoint().getLatitude()); jgen.writeEndArray(); + jgen.writeEndObject(); + break; + default: + jgen.writeRawValue(toGeoJson(value)); break; } - jgen.writeEndObject(); } @Override public void serializeWithType(Geoshape geoshape, JsonGenerator jgen, SerializerProvider serializerProvider, TypeSerializer typeSerializer) throws IOException, JsonProcessingException { + jgen.writeStartObject(); if (typeSerializer != null) jgen.writeStringField(GraphSONTokens.CLASS, Geoshape.class.getName()); - GraphSONUtil.writeWithType(FIELD_COORDINATES, geoshape.coordinates, jgen, serializerProvider, typeSerializer); + String geojson = toGeoJson(geoshape); + Map json = new ObjectMapper().readValue(geojson, LinkedHashMap.class); + if (geoshape.getType() == Type.POINT) { + double[] coords = ((List) json.get("coordinates")).stream().map(i -> i.doubleValue()).mapToDouble(i -> i).toArray(); + GraphSONUtil.writeWithType(FIELD_COORDINATES, coords, jgen, serializerProvider, typeSerializer); + } else { + GraphSONUtil.writeWithType(FIELD_LABEL, json, jgen, serializerProvider, typeSerializer); + } jgen.writeEndObject(); } + + public static String toGeoJson(Geoshape geoshape) { + return geojsonWriter.toString(geoshape.shape); + } + } + /** + * Geoshape JSON deserializer supporting reading from GeoJSON (http://geojson.org/). + */ public static class GeoshapeGsonDeserializer extends StdDeserializer { + public GeoshapeGsonDeserializer() { super(Geoshape.class); } @Override public Geoshape deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JsonProcessingException { - // move the parser forward jsonParser.nextToken(); + if (jsonParser.getCurrentName().equals("coordinates")) { + double[] f = jsonParser.readValueAs(double[].class); + jsonParser.nextToken(); + return Geoshape.point(f[1], f[0]); + } else { + try { + HashMap map = jsonParser.readValueAs(LinkedHashMap.class); + jsonParser.nextToken(); + String json = new ObjectMapper().writeValueAsString(map); + Geoshape shape = new Geoshape(geojsonReader.read(new StringReader(json))); + return shape; + } catch (ParseException e) { + throw new IOException("Unable to read and parse geojson", e); + } + } + } + } - float[][] f = jsonParser.readValueAs(float[][].class); - jsonParser.nextToken(); - return new Geoshape(f); + /** + * Geoshape binary serializer using spatial4j's {@link JtsBinaryCodec}. + * + */ + public static class GeoshapeBinarySerializer { + + private static JtsBinaryCodec binaryCodec = new JtsBinaryCodec(CTX, new JtsSpatialContextFactory()); + + /** + * Serialize a geoshape. + * @param outputStream + * @param attribute + * @return + * @throws IOException + */ + public static void write(OutputStream outputStream, Geoshape attribute) throws IOException { + outputStream.write(attribute.shape instanceof JtsGeometry ? 0 : 1); + try (DataOutputStream dataOutput = new DataOutputStream(outputStream)) { + if (attribute.shape instanceof JtsGeometry) { + binaryCodec.writeJtsGeom(dataOutput, attribute.shape); + } else { + binaryCodec.writeShape(dataOutput, attribute.shape); + } + dataOutput.flush(); + } + outputStream.flush(); + } + + /** + * Deserialize a geoshape. + * @param inputStream + * @return + * @throws IOException + */ + public static Geoshape read(InputStream inputStream) throws IOException { + boolean isJts = inputStream.read()==0; + try (DataInputStream dataInput = new DataInputStream(inputStream)) { + if (isJts) { + return new Geoshape(binaryCodec.readJtsGeom(dataInput)); + } else { + return new Geoshape(binaryCodec.readShape(dataInput)); + } + } } } + } diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/core/schema/Mapping.java b/titan-core/src/main/java/com/thinkaurelius/titan/core/schema/Mapping.java index 6ca3fface6..42ed6b0118 100644 --- a/titan-core/src/main/java/com/thinkaurelius/titan/core/schema/Mapping.java +++ b/titan-core/src/main/java/com/thinkaurelius/titan/core/schema/Mapping.java @@ -18,7 +18,8 @@ public enum Mapping { DEFAULT, TEXT, STRING, - TEXTSTRING; + TEXTSTRING, + PREFIX_TREE; /** * Returns the mapping as a parameter so that it can be passed to {@link TitanManagement#addIndexKey(TitanGraphIndex, com.thinkaurelius.titan.core.PropertyKey, Parameter[])} diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/database/serialize/AttributeUtil.java b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/database/serialize/AttributeUtil.java index 296a2b39d1..61fbfa9e34 100644 --- a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/database/serialize/AttributeUtil.java +++ b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/database/serialize/AttributeUtil.java @@ -1,6 +1,8 @@ package com.thinkaurelius.titan.graphdb.database.serialize; import com.thinkaurelius.titan.core.PropertyKey; +import com.thinkaurelius.titan.core.attribute.Geoshape; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,6 +40,10 @@ public static final boolean isString(Class clazz) { return clazz.equals(String.class); } + public static final boolean isGeo(Class clazz) { + return clazz.equals(Geoshape.class); + } + /** * Compares the two elements like {@link java.util.Comparator#compare(Object, Object)} but returns * null in case the two elements are not comparable. diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/types/ParameterType.java b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/types/ParameterType.java index 424eb50378..4698aaaf4a 100644 --- a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/types/ParameterType.java +++ b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/types/ParameterType.java @@ -9,7 +9,7 @@ */ public enum ParameterType { - MAPPING("mapping"), INDEX_POSITION("index-pos"), MAPPED_NAME("mapped-name"), STATUS("status"); + MAPPING("mapping"), INDEX_POSITION("index-pos"), MAPPED_NAME("mapped-name"), STATUS("status"), INDEX_GEO_MAX_LEVELS("index-geo-max-levels"), INDEX_GEO_DIST_ERROR_PCT("index-geo-dist-error-pct"); private final String name; diff --git a/titan-es/pom.xml b/titan-es/pom.xml index f91d02d46f..fc9219d59b 100644 --- a/titan-es/pom.xml +++ b/titan-es/pom.xml @@ -55,19 +55,22 @@ org.elasticsearch elasticsearch ${elasticsearch.version} + + + + org.elasticsearch.module + lang-groovy + ${elasticsearch.version} + zip + test - org.antlr - antlr-runtime + org.codehaus.groovy + groovy - - org.antlr - antlr-runtime - ${antlr.version} - - @@ -94,6 +97,21 @@ test-jar + + + pre-test-jar + process-test-classes + + jar + + + es_jarhell + + **/JarHell.class + + + @@ -178,6 +196,23 @@ ${project.build.directory}/es_classpath.txt + + + unpack-plugins + + unpack-dependencies + + generate-test-resources + + ${java.io.tmpdir}/plugins/lang-groovy + lang-groovy + org.elasticsearch.module + true + pom + groovy*.jarexcludes> + test + + diff --git a/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchIndex.java b/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchIndex.java index 0a8d525b33..f4c9be429a 100644 --- a/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchIndex.java +++ b/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchIndex.java @@ -24,7 +24,9 @@ import com.thinkaurelius.titan.graphdb.internal.Order; import com.thinkaurelius.titan.graphdb.query.TitanPredicate; import com.thinkaurelius.titan.graphdb.query.condition.*; +import com.thinkaurelius.titan.graphdb.types.ParameterType; import com.thinkaurelius.titan.util.system.IOUtils; + import org.apache.commons.lang.StringUtils; import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; @@ -42,16 +44,21 @@ import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.geo.builders.LineStringBuilder; +import org.elasticsearch.common.geo.builders.PolygonBuilder; +import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.*; -import org.elasticsearch.indices.IndexMissingException; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeBuilder; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -60,14 +67,19 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.time.Instant; import java.util.*; +import java.util.AbstractMap.SimpleEntry; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * @author Matthias Broecheler (me@matthiasb.com) @@ -173,6 +185,18 @@ public class ElasticSearchIndex implements IndexProvider { public static final int HOST_PORT_DEFAULT = 9300; + /** + * Default tree_levels used when creating geo_shape mappings. + */ + public static final int DEFAULT_GEO_MAX_LEVELS = 20; + + /** + * Default distance_error_pct used when creating geo_shape mappings. + */ + public static final double DEFAULT_GEO_DIST_ERROR_PCT = 0.025; + + private static final Map SPATIAL_PREDICATES = spatialPredicates(); + private final Node node; private final Client client; private final String indexName; @@ -221,7 +245,7 @@ private void checkForOrCreateIndex(Configuration config) { IndicesExistsResponse response = client.admin().indices().exists(new IndicesExistsRequest(indexName)).actionGet(); if (!response.isExists()) { - ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder(); + Settings.Builder settings = Settings.settingsBuilder(); ElasticSearchSetup.applySettingsFromTitanConf(settings, config, ES_CREATE_EXTRAS_NS); @@ -294,7 +318,8 @@ private ElasticSearchSetup.Connection legacyConfiguration(Configuration config) "Must either configure configuration file or base directory"); if (config.has(INDEX_CONF_FILE)) { String configFile = config.get(INDEX_CONF_FILE); - ImmutableSettings.Builder sb = ImmutableSettings.settingsBuilder(); + Settings.Builder sb = Settings.settingsBuilder(); + sb.put("path.home", System.getProperty("java.io.tmpdir")); log.debug("Configuring ES from YML file [{}]", configFile); FileInputStream fis = null; try { @@ -311,15 +336,16 @@ private ElasticSearchSetup.Connection legacyConfiguration(Configuration config) log.debug("Configuring ES with data directory [{}]", dataDirectory); File f = new File(dataDirectory); if (!f.exists()) f.mkdirs(); - ImmutableSettings.Builder b = ImmutableSettings.settingsBuilder(); + Settings.Builder b = Settings.settingsBuilder(); for (String sub : DATA_SUBDIRS) { String subdir = dataDirectory + File.separator + sub; f = new File(subdir); if (!f.exists()) f.mkdirs(); b.put("path." + sub, subdir); } - b.put("script.disable_dynamic", false); + b.put("script.inline", "on"); b.put("indices.ttl.interval", "5s"); + b.put("path.home", System.getProperty("java.io.tmpdir")); builder.settings(b.build()); @@ -328,12 +354,14 @@ private ElasticSearchSetup.Connection legacyConfiguration(Configuration config) builder.clusterName(clustername); } + builder.getSettings().put("index.max_result_window", Integer.MAX_VALUE); + node = builder.client(clientOnly).data(!clientOnly).local(local).node(); client = node.client(); } else { log.debug("Configuring ES for network transport"); - ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder(); + Settings.Builder settings = Settings.settingsBuilder(); if (config.has(CLUSTER_NAME)) { String clustername = config.get(CLUSTER_NAME); Preconditions.checkArgument(StringUtils.isNotBlank(clustername), "Invalid cluster name: %s", clustername); @@ -343,8 +371,9 @@ private ElasticSearchSetup.Connection legacyConfiguration(Configuration config) } log.debug("Transport sniffing enabled: {}", config.get(CLIENT_SNIFF)); settings.put("client.transport.sniff", config.get(CLIENT_SNIFF)); - settings.put("script.disable_dynamic", false); - TransportClient tc = new TransportClient(settings.build()); + settings.put("script.inline", "on"); + settings.put("index.max_result_window", Integer.MAX_VALUE); + TransportClient tc = TransportClient.builder().settings(settings.build()).build(); int defaultPort = config.has(INDEX_PORT)?config.get(INDEX_PORT):HOST_PORT_DEFAULT; for (String host : config.get(INDEX_HOSTS)) { String[] hostparts = host.split(":"); @@ -352,7 +381,12 @@ private ElasticSearchSetup.Connection legacyConfiguration(Configuration config) int hostport = defaultPort; if (hostparts.length == 2) hostport = Integer.parseInt(hostparts[1]); log.info("Configured remote host: {} : {}", hostname, hostport); - tc.addTransportAddress(new InetSocketTransportAddress(hostname, hostport)); + try { + tc.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(hostname), hostport)); + } catch (UnknownHostException e) { + log.error("unknown host", e); + throw new RuntimeException(e); + } } client = tc; node = null; @@ -373,12 +407,22 @@ private static String getDualMappingName(String key) { return key + STRING_MAPPING_SUFFIX; } + private static Map spatialPredicates() { + return Collections.unmodifiableMap(Stream.of( + new SimpleEntry<>(Geo.WITHIN, ShapeRelation.WITHIN), + new SimpleEntry<>(Geo.CONTAINS, ShapeRelation.CONTAINS), + new SimpleEntry<>(Geo.INTERSECT, ShapeRelation.INTERSECTS), + new SimpleEntry<>(Geo.DISJOINT, ShapeRelation.DISJOINT)) + .collect(Collectors.toMap((e) -> e.getKey(), (e) -> e.getValue()))); + } + @Override public void register(String store, String key, KeyInformation information, BaseTransaction tx) throws BackendException { XContentBuilder mapping; Class dataType = information.getDataType(); Mapping map = Mapping.getMapping(information); - Preconditions.checkArgument(map==Mapping.DEFAULT || AttributeUtil.isString(dataType), + Preconditions.checkArgument(map==Mapping.DEFAULT || AttributeUtil.isString(dataType) || + (map==Mapping.PREFIX_TREE && AttributeUtil.isGeo(dataType)), "Specified illegal mapping [%s] for data type [%s]",map,dataType); try { @@ -433,8 +477,20 @@ public void register(String store, String key, KeyInformation information, BaseT log.debug("Registering boolean type for {}", key); mapping.field("type", "boolean"); } else if (dataType == Geoshape.class) { - log.debug("Registering geo_point type for {}", key); - mapping.field("type", "geo_point"); + switch (map) { + case PREFIX_TREE: + int maxLevels = (int) ParameterType.INDEX_GEO_MAX_LEVELS.findParameter(information.getParameters(), DEFAULT_GEO_MAX_LEVELS); + double distErrorPct = (double) ParameterType.INDEX_GEO_DIST_ERROR_PCT.findParameter(information.getParameters(), DEFAULT_GEO_DIST_ERROR_PCT); + log.debug("Registering geo_shape type for {} with tree_levels={} and distance_error_pct={}", key, maxLevels, distErrorPct); + mapping.field("type", "geo_shape"); + mapping.field("tree", "quadtree"); + mapping.field("tree_levels", maxLevels); + mapping.field("distance_error_pct", distErrorPct); + break; + default: + log.debug("Registering geo_point type for {}", key); + mapping.field("type", "geo_point"); + } } else if (dataType == Date.class || dataType == Instant.class) { log.debug("Registering date type for {}", key); mapping.field("type", "date"); @@ -455,7 +511,7 @@ public void register(String store, String key, KeyInformation information, BaseT try { PutMappingResponse response = client.admin().indices().preparePutMapping(indexName). - setIgnoreConflicts(false).setType(store).setSource(mapping).execute().actionGet(); + setType(store).setSource(mapping).execute().actionGet(); } catch (Exception e) { throw convert(e); } @@ -472,8 +528,7 @@ private static boolean hasDualStringMapping(KeyInformation information) { return AttributeUtil.isString(information.getDataType()) && getStringMapping(information)==Mapping.TEXTSTRING; } - public XContentBuilder getNewDocument(final List additions, KeyInformation.StoreRetriever informations, int ttl) throws BackendException { - Preconditions.checkArgument(ttl >= 0); + public XContentBuilder getNewDocument(final List additions, KeyInformation.StoreRetriever informations) throws BackendException { try { XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); @@ -495,19 +550,26 @@ public XContentBuilder getNewDocument(final List additions, KeyInfor break; case SET: case LIST: - value = add.getValue().stream().map(v -> convertToEsType(v.value)).collect(Collectors.toList()).toArray(); + value = add.getValue().stream().map(v -> convertToEsType(v.value)) + .filter(v -> { + Preconditions.checkArgument(!(v instanceof byte[]), "Collections not supported for " + add.getKey()); + return true; + }) + .collect(Collectors.toList()).toArray(); break; } - - builder.field(add.getKey(), value); + if (value instanceof byte[]) { + builder.rawField(add.getKey(), new ByteArrayInputStream((byte[]) value)); + } else { + builder.field(add.getKey(), value); + } if (hasDualStringMapping(informations.get(add.getKey())) && keyInformation.getDataType() == String.class) { builder.field(getDualMappingName(add.getKey()), value); } } - if (ttl>0) builder.field(TTL_FIELD, TimeUnit.MILLISECONDS.convert(ttl,TimeUnit.SECONDS)); builder.endObject(); @@ -527,12 +589,7 @@ private static Object convertToEsType(Object value) { } else if (AttributeUtil.isString(value)) { return value; } else if (value instanceof Geoshape) { - Geoshape shape = (Geoshape) value; - if (shape.getType() == Geoshape.Type.POINT) { - Geoshape.Point p = shape.getPoint(); - return new double[]{p.getLongitude(), p.getLatitude()}; - } else throw new UnsupportedOperationException("Geo type is not supported: " + shape.getType()); - + return convertgeo((Geoshape) value); } else if (value instanceof Date || value instanceof Instant) { return value; } else if (value instanceof Boolean) { @@ -542,6 +599,17 @@ private static Object convertToEsType(Object value) { } else throw new IllegalArgumentException("Unsupported type: " + value.getClass() + " (value: " + value + ")"); } + private static Object convertgeo(Geoshape geoshape) { + if (geoshape.getType() == Geoshape.Type.POINT) { + Geoshape.Point p = geoshape.getPoint(); + return new double[]{p.getLongitude(), p.getLatitude()}; + } else if (geoshape.getType() != Geoshape.Type.BOX && geoshape.getType() != Geoshape.Type.CIRCLE) { + return geoshape.toGeoJson().getBytes(); + } else { + throw new IllegalArgumentException("Unsupported or invalid shape type for indexing: " + geoshape.getType()); + } + } + @Override public void mutate(Map> mutations, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException { BulkRequestBuilder brb = client.prepareBulk(); @@ -565,28 +633,35 @@ public void mutate(Map> mutations, KeyInforma brb.add(new DeleteRequest(indexName, storename, docid)); } else { String script = getDeletionScript(informations, storename, mutation); - brb.add(client.prepareUpdate(indexName, storename, docid).setScript(script, ScriptService.ScriptType.INLINE)); + brb.add(client.prepareUpdate(indexName, storename, docid).setScript(new Script(script, ScriptService.ScriptType.INLINE, null, null))); log.trace("Adding script {}", script); } bulkrequests++; } if (mutation.hasAdditions()) { - int ttl = mutation.determineTTL(); + long ttl = mutation.determineTTL() * 1000l; if (mutation.isNew()) { //Index log.trace("Adding entire document {}", docid); - brb.add(new IndexRequest(indexName, storename, docid) - .source(getNewDocument(mutation.getAdditions(), informations.get(storename), ttl))); + Preconditions.checkArgument(ttl >= 0); + IndexRequest request = new IndexRequest(indexName, storename, docid) + .source(getNewDocument(mutation.getAdditions(), informations.get(storename))); + if (ttl > 0) { + request.ttl(ttl); + } + brb.add(request); } else { Preconditions.checkArgument(ttl == 0, "Elasticsearch only supports TTL on new documents [%s]", docid); boolean needUpsert = !mutation.hasDeletions(); String script = getAdditionScript(informations, storename, mutation); - UpdateRequestBuilder update = client.prepareUpdate(indexName, storename, docid).setScript(script, ScriptService.ScriptType.INLINE); + UpdateRequestBuilder update = client.prepareUpdate(indexName, storename, docid).setScript( + new Script(script, ScriptService.ScriptType.INLINE, null, null)); if (needUpsert) { - XContentBuilder doc = getNewDocument(mutation.getAdditions(), informations.get(storename), ttl); + XContentBuilder doc = getNewDocument(mutation.getAdditions(), informations.get(storename)); + update.setUpsert(doc); } @@ -678,7 +753,12 @@ private static String convertToJsType(Object value) throws PermanentBackendExcep try { XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); - builder.field("value", convertToEsType(value)); + Object esValue = convertToEsType(value); + if (esValue instanceof byte[]) { + builder.rawField("value", new ByteArrayInputStream((byte[]) esValue)); + } else { + builder.field("value", esValue); + } String s = builder.string(); int prefixLength = "{\"value\":".length(); @@ -694,6 +774,7 @@ private static String convertToJsType(Object value) throws PermanentBackendExcep } + @Override public void restore(Map>> documents, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException { BulkRequestBuilder bulk = client.prepareBulk(); int requests = 0; @@ -716,7 +797,13 @@ public void restore(Map>> documents, KeyInfo // Add if (log.isTraceEnabled()) log.trace("Adding entire document {}", docID); - bulk.add(new IndexRequest(indexName, store, docID).source(getNewDocument(content, informations.get(store), IndexMutation.determineTTL(content)))); + long ttl = IndexMutation.determineTTL(content) * 1000l; + Preconditions.checkArgument(ttl >= 0); + IndexRequest request = new IndexRequest(indexName, store, docID).source(getNewDocument(content, informations.get(store))); + if (ttl > 0) { + request.ttl(ttl); + } + bulk.add(request); requests++; } } @@ -729,7 +816,7 @@ public void restore(Map>> documents, KeyInfo } } - public FilterBuilder getFilter(Condition condition, KeyInformation.StoreRetriever informations) { + public QueryBuilder getFilter(Condition condition, KeyInformation.StoreRetriever informations) { if (condition instanceof PredicateCondition) { PredicateCondition atom = (PredicateCondition) condition; Object value = atom.getValue(); @@ -742,17 +829,17 @@ public FilterBuilder getFilter(Condition condition, KeyInformation.StoreRetri switch (numRel) { case EQUAL: - return FilterBuilders.inFilter(key, value); + return QueryBuilders.matchQuery(key, value); case NOT_EQUAL: - return FilterBuilders.notFilter(FilterBuilders.inFilter(key, value)); + return QueryBuilders.notQuery(QueryBuilders.matchQuery(key, value)); case LESS_THAN: - return FilterBuilders.rangeFilter(key).lt(value); + return QueryBuilders.rangeQuery(key).lt(value); case LESS_THAN_EQUAL: - return FilterBuilders.rangeFilter(key).lte(value); + return QueryBuilders.rangeQuery(key).lte(value); case GREATER_THAN: - return FilterBuilders.rangeFilter(key).gt(value); + return QueryBuilders.rangeQuery(key).gt(value); case GREATER_THAN_EQUAL: - return FilterBuilders.rangeFilter(key).gte(value); + return QueryBuilders.rangeQuery(key).gte(value); default: throw new IllegalArgumentException("Unexpected relation: " + numRel); } @@ -768,56 +855,105 @@ public FilterBuilder getFilter(Condition condition, KeyInformation.StoreRetri if (titanPredicate == Text.CONTAINS) { value = ((String) value).toLowerCase(); - AndFilterBuilder b = FilterBuilders.andFilter(); + BoolQueryBuilder b = QueryBuilders.boolQuery(); for (String term : Text.tokenize((String)value)) { - b.add(FilterBuilders.termFilter(fieldName, term)); + b.must(QueryBuilders.termQuery(fieldName, term)); } return b; } else if (titanPredicate == Text.CONTAINS_PREFIX) { value = ((String) value).toLowerCase(); - return FilterBuilders.prefixFilter(fieldName, (String) value); + return QueryBuilders.prefixQuery(fieldName, (String) value); } else if (titanPredicate == Text.CONTAINS_REGEX) { value = ((String) value).toLowerCase(); - return FilterBuilders.regexpFilter(fieldName, (String) value); + return QueryBuilders.regexpQuery(fieldName, (String) value); } else if (titanPredicate == Text.PREFIX) { - return FilterBuilders.prefixFilter(fieldName, (String) value); + return QueryBuilders.prefixQuery(fieldName, (String) value); } else if (titanPredicate == Text.REGEX) { - return FilterBuilders.regexpFilter(fieldName, (String) value); + return QueryBuilders.regexpQuery(fieldName, (String) value); } else if (titanPredicate == Cmp.EQUAL) { - return FilterBuilders.termFilter(fieldName, (String) value); + return QueryBuilders.termQuery(fieldName, (String) value); } else if (titanPredicate == Cmp.NOT_EQUAL) { - return FilterBuilders.notFilter(FilterBuilders.termFilter(fieldName, (String) value)); + return QueryBuilders.notQuery(QueryBuilders.termQuery(fieldName, (String) value)); } else throw new IllegalArgumentException("Predicate is not supported for string value: " + titanPredicate); - } else if (value instanceof Geoshape) { - Preconditions.checkArgument(titanPredicate == Geo.WITHIN, "Relation is not supported for geo value: " + titanPredicate); + } else if (value instanceof Geoshape && Mapping.getMapping(informations.get(key)) == Mapping.DEFAULT) { + // geopoint Geoshape shape = (Geoshape) value; + Preconditions.checkArgument(titanPredicate instanceof Geo && titanPredicate != Geo.CONTAINS, "Relation not supported on geopoint types: " + titanPredicate); + + final QueryBuilder queryBuilder; if (shape.getType() == Geoshape.Type.CIRCLE) { Geoshape.Point center = shape.getPoint(); - return FilterBuilders.geoDistanceFilter(key).lat(center.getLatitude()).lon(center.getLongitude()).distance(shape.getRadius(), DistanceUnit.KILOMETERS); + queryBuilder = QueryBuilders.geoDistanceQuery(key).lat(center.getLatitude()).lon(center.getLongitude()).distance(shape.getRadius(), DistanceUnit.KILOMETERS); } else if (shape.getType() == Geoshape.Type.BOX) { Geoshape.Point southwest = shape.getPoint(0); Geoshape.Point northeast = shape.getPoint(1); - return FilterBuilders.geoBoundingBoxFilter(key).bottomRight(southwest.getLatitude(), northeast.getLongitude()).topLeft(northeast.getLatitude(), southwest.getLongitude()); - } else + queryBuilder = QueryBuilders.geoBoundingBoxQuery(key).bottomRight(southwest.getLatitude(), northeast.getLongitude()).topLeft(northeast.getLatitude(), southwest.getLongitude()); + } else if (shape.getType() == Geoshape.Type.POLYGON) { + queryBuilder = QueryBuilders.geoPolygonQuery(key); + for (int i = 0; i < shape.size(); i++) { + Geoshape.Point point = shape.getPoint(i); + ((GeoPolygonQueryBuilder) queryBuilder).addPoint(point.getLatitude(), point.getLongitude()); + } + } else { + throw new IllegalArgumentException("Unsupported or invalid search shape type for geopoint: " + shape.getType()); + } + + return titanPredicate == Geo.DISJOINT ? QueryBuilders.notQuery(queryBuilder) : queryBuilder; + } else if (value instanceof Geoshape) { + // geoshape + Preconditions.checkArgument(titanPredicate instanceof Geo, "Relation not supported on geoshape types: " + titanPredicate); + Geoshape shape = (Geoshape) value; + final ShapeBuilder sb; + switch (shape.getType()) { + case CIRCLE: + Geoshape.Point center = shape.getPoint(); + sb = ShapeBuilder.newCircleBuilder().center(center.getLongitude(), center.getLatitude()).radius(shape.getRadius(), DistanceUnit.KILOMETERS); + break; + case BOX: + Geoshape.Point southwest = shape.getPoint(0); + Geoshape.Point northeast = shape.getPoint(1); + sb = ShapeBuilder.newEnvelope().bottomRight(northeast.getLongitude(),southwest.getLatitude()).topLeft(southwest.getLongitude(),northeast.getLatitude()); + break; + case POLYGON: + sb = ShapeBuilder.newPolygon(); + for (int i = 0; i < shape.size(); i++) { + Geoshape.Point point = shape.getPoint(i); + ((PolygonBuilder) sb).point(point.getLongitude(), point.getLatitude()); + } + break; + case LINE: + sb = ShapeBuilder.newLineString(); + for (int i = 0; i < shape.size(); i++) { + Geoshape.Point point = shape.getPoint(i); + ((LineStringBuilder) sb).point(point.getLongitude(), point.getLatitude()); + } + break; + case POINT: + sb = ShapeBuilder.newPoint(shape.getPoint().getLongitude(),shape.getPoint().getLatitude()); + break; + default: throw new IllegalArgumentException("Unsupported or invalid search shape type: " + shape.getType()); + } + + return QueryBuilders.geoShapeQuery(key, sb, SPATIAL_PREDICATES.get((Geo) titanPredicate)); } else if (value instanceof Date || value instanceof Instant) { Preconditions.checkArgument(titanPredicate instanceof Cmp, "Relation not supported on date types: " + titanPredicate); Cmp numRel = (Cmp) titanPredicate; switch (numRel) { case EQUAL: - return FilterBuilders.inFilter(key, value); + return QueryBuilders.matchQuery(key, value); case NOT_EQUAL: - return FilterBuilders.notFilter(FilterBuilders.inFilter(key, value)); + return QueryBuilders.notQuery(QueryBuilders.matchQuery(key, value)); case LESS_THAN: - return FilterBuilders.rangeFilter(key).lt(value); + return QueryBuilders.rangeQuery(key).lt(value); case LESS_THAN_EQUAL: - return FilterBuilders.rangeFilter(key).lte(value); + return QueryBuilders.rangeQuery(key).lte(value); case GREATER_THAN: - return FilterBuilders.rangeFilter(key).gt(value); + return QueryBuilders.rangeQuery(key).gt(value); case GREATER_THAN_EQUAL: - return FilterBuilders.rangeFilter(key).gte(value); + return QueryBuilders.rangeQuery(key).gte(value); default: throw new IllegalArgumentException("Unexpected relation: " + numRel); } @@ -825,34 +961,35 @@ public FilterBuilder getFilter(Condition condition, KeyInformation.StoreRetri Cmp numRel = (Cmp) titanPredicate; switch (numRel) { case EQUAL: - return FilterBuilders.inFilter(key, value); + return QueryBuilders.matchQuery(key, value); case NOT_EQUAL: - return FilterBuilders.notFilter(FilterBuilders.inFilter(key, value)); + return QueryBuilders.notQuery(QueryBuilders.matchQuery(key, value)); default: throw new IllegalArgumentException("Boolean types only support EQUAL or NOT_EQUAL"); } } else if (value instanceof UUID) { if (titanPredicate == Cmp.EQUAL) { - return FilterBuilders.termFilter(key, value); + return QueryBuilders.termQuery(key, value); } else if (titanPredicate == Cmp.NOT_EQUAL) { - return FilterBuilders.notFilter(FilterBuilders.termFilter(key, value)); + return QueryBuilders.notQuery(QueryBuilders.termQuery(key, value)); } else { throw new IllegalArgumentException("Only equal or not equal is supported for UUIDs: " + titanPredicate); } } else throw new IllegalArgumentException("Unsupported type: " + value); } else if (condition instanceof Not) { - return FilterBuilders.notFilter(getFilter(((Not) condition).getChild(),informations)); + return QueryBuilders.notQuery(getFilter(((Not) condition).getChild(),informations)); } else if (condition instanceof And) { - AndFilterBuilder b = FilterBuilders.andFilter(); + BoolQueryBuilder b = QueryBuilders.boolQuery(); for (Condition c : condition.getChildren()) { - b.add(getFilter(c,informations)); + b.must(getFilter(c,informations)); } return b; } else if (condition instanceof Or) { - OrFilterBuilder b = FilterBuilders.orFilter(); + BoolQueryBuilder b = QueryBuilders.boolQuery(); + b.minimumNumberShouldMatch(1); for (Condition c : condition.getChildren()) { - b.add(getFilter(c,informations)); + b.should(getFilter(c,informations)); } return b; } else throw new IllegalArgumentException("Invalid condition: " + condition); @@ -873,8 +1010,10 @@ public List query(IndexQuery query, KeyInformation.IndexRetriever inform if (useDeprecatedIgnoreUnmapped) { fsb.ignoreUnmapped(true); } else { + KeyInformation information = informations.get(query.getStore()).get(orders.get(i).getKey()); + Mapping mapping = Mapping.getMapping(information); Class datatype = orderEntry.getDatatype(); - fsb.unmappedType(convertToEsDataType(datatype)); + fsb.unmappedType(convertToEsDataType(datatype, mapping)); } srb.addSort(fsb); } @@ -897,7 +1036,7 @@ public List query(IndexQuery query, KeyInformation.IndexRetriever inform return result; } - private String convertToEsDataType(Class datatype) { + private String convertToEsDataType(Class datatype, Mapping mapping) { if(String.class.isAssignableFrom(datatype)) { return "string"; } @@ -923,7 +1062,7 @@ else if (Instant.class.isAssignableFrom(datatype)) { return "date"; } else if (Geoshape.class.isAssignableFrom(datatype)) { - return "geo_point"; + return mapping == Mapping.DEFAULT ? "geo_point" : "geo_shape"; } return null; @@ -957,12 +1096,18 @@ public Iterable> query(RawQuery query, KeyInformation.In public boolean supports(KeyInformation information, TitanPredicate titanPredicate) { Class dataType = information.getDataType(); Mapping mapping = Mapping.getMapping(information); - if (mapping!=Mapping.DEFAULT && !AttributeUtil.isString(dataType)) return false; + if (mapping!=Mapping.DEFAULT && !AttributeUtil.isString(dataType) && + !(mapping==Mapping.PREFIX_TREE && AttributeUtil.isGeo(dataType))) return false; if (Number.class.isAssignableFrom(dataType)) { if (titanPredicate instanceof Cmp) return true; } else if (dataType == Geoshape.class) { - return titanPredicate == Geo.WITHIN; + switch(mapping) { + case DEFAULT: + return titanPredicate instanceof Geo && titanPredicate != Geo.CONTAINS; + case PREFIX_TREE: + return titanPredicate instanceof Geo; + } } else if (AttributeUtil.isString(dataType)) { switch(mapping) { case DEFAULT: @@ -988,11 +1133,13 @@ public boolean supports(KeyInformation information, TitanPredicate titanPredicat public boolean supports(KeyInformation information) { Class dataType = information.getDataType(); Mapping mapping = Mapping.getMapping(information); - if (Number.class.isAssignableFrom(dataType) || dataType == Geoshape.class || dataType == Date.class || dataType== Instant.class || dataType == Boolean.class || dataType == UUID.class) { + if (Number.class.isAssignableFrom(dataType) || dataType == Date.class || dataType== Instant.class || dataType == Boolean.class || dataType == UUID.class) { if (mapping==Mapping.DEFAULT) return true; } else if (AttributeUtil.isString(dataType)) { if (mapping==Mapping.DEFAULT || mapping==Mapping.STRING || mapping==Mapping.TEXT || mapping==Mapping.TEXTSTRING) return true; + } else if (AttributeUtil.isGeo(dataType)) { + if (mapping==Mapping.DEFAULT || mapping==Mapping.PREFIX_TREE) return true; } return false; } @@ -1031,7 +1178,7 @@ public void clearStorage() throws BackendException { .delete(new DeleteIndexRequest(indexName)).actionGet(); // We wait for one second to let ES delete the river Thread.sleep(1000); - } catch (IndexMissingException e) { + } catch (IndexNotFoundException e) { // Index does not exist... Fine } } catch (Exception e) { diff --git a/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchSetup.java b/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchSetup.java index 0bdf47bba5..307db932e0 100644 --- a/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchSetup.java +++ b/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchSetup.java @@ -6,10 +6,11 @@ import com.thinkaurelius.titan.diskstorage.configuration.ConfigOption; import com.thinkaurelius.titan.diskstorage.configuration.Configuration; import com.thinkaurelius.titan.util.system.IOUtils; + import org.apache.commons.lang.StringUtils; import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeBuilder; @@ -18,6 +19,7 @@ import java.io.*; import java.lang.reflect.Array; +import java.net.InetAddress; import java.util.List; import java.util.Map; @@ -68,7 +70,7 @@ public enum ElasticSearchSetup { public Connection connect(Configuration config) throws IOException { log.debug("Configuring TransportClient"); - ImmutableSettings.Builder settingsBuilder = settingsBuilder(config); + Settings.Builder settingsBuilder = settingsBuilder(config); if (config.has(ElasticSearchIndex.CLIENT_SNIFF)) { String k = "client.transport.sniff"; @@ -76,7 +78,10 @@ public Connection connect(Configuration config) throws IOException { log.debug("Set {}: {}", k, config.get(ElasticSearchIndex.CLIENT_SNIFF)); } - TransportClient tc = new TransportClient(settingsBuilder.build()); + settingsBuilder.put("index.max_result_window", Integer.MAX_VALUE); + makeLocalDirsIfNecessary(settingsBuilder, config); + + TransportClient tc = TransportClient.builder().settings(settingsBuilder.build()).build(); int defaultPort = config.has(INDEX_PORT) ? config.get(INDEX_PORT) : ElasticSearchIndex.HOST_PORT_DEFAULT; for (String host : config.get(INDEX_HOSTS)) { String[] hostparts = host.split(":"); @@ -84,7 +89,7 @@ public Connection connect(Configuration config) throws IOException { int hostport = defaultPort; if (hostparts.length == 2) hostport = Integer.parseInt(hostparts[1]); log.info("Configured remote host: {} : {}", hostname, hostport); - tc.addTransportAddress(new InetSocketTransportAddress(hostname, hostport)); + tc.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(hostname), hostport)); } return new Connection(null, tc); } @@ -99,7 +104,7 @@ public Connection connect(Configuration config) throws IOException { log.debug("Configuring Node Client"); - ImmutableSettings.Builder settingsBuilder = settingsBuilder(config); + Settings.Builder settingsBuilder = settingsBuilder(config); if (config.has(ElasticSearchIndex.TTL_INTERVAL)) { String k = "indices.ttl.interval"; @@ -109,7 +114,7 @@ public Connection connect(Configuration config) throws IOException { makeLocalDirsIfNecessary(settingsBuilder, config); - NodeBuilder nodeBuilder = NodeBuilder.nodeBuilder().settings(settingsBuilder.build()); + NodeBuilder nodeBuilder = NodeBuilder.nodeBuilder(); // Apply explicit Titan properties file overrides (otherwise conf-file or ES defaults apply) if (config.has(ElasticSearchIndex.CLIENT_ONLY)) { @@ -120,9 +125,14 @@ public Connection connect(Configuration config) throws IOException { if (config.has(ElasticSearchIndex.LOCAL_MODE)) nodeBuilder.local(config.get(ElasticSearchIndex.LOCAL_MODE)); - if (config.has(ElasticSearchIndex.LOAD_DEFAULT_NODE_SETTINGS)) - nodeBuilder.loadConfigSettings(config.get(ElasticSearchIndex.LOAD_DEFAULT_NODE_SETTINGS)); + if (config.has(ElasticSearchIndex.LOAD_DEFAULT_NODE_SETTINGS)) { + // Elasticsearch >2.3 always loads default settings + String k = "config.ignore_system_properties"; + settingsBuilder.put(k, !config.get(ElasticSearchIndex.LOAD_DEFAULT_NODE_SETTINGS)); + } + settingsBuilder.put("index.max_result_window", Integer.MAX_VALUE); + nodeBuilder.settings(settingsBuilder.build()); Node node = nodeBuilder.node(); Client client = node.client(); return new Connection(node, client); @@ -145,7 +155,7 @@ public Connection connect(Configuration config) throws IOException { *
  • If ignore-cluster-name is set, copy that value to client.transport.ignore_cluster_name in the settings builder
    • *
  • If client-sniff is set, copy that value to client.transport.sniff in the settings builder
    • *
  • If ttl-interval is set, copy that volue to indices.ttl.interval in the settings builder
    • - *
  • Unconditionally set script.disable_dynamic to false (i.e. enable dynamic scripting)
    • + *
  • Unconditionally set script.inline to on (i.e. enable inline scripting)
    • * * * This method then returns the builder. @@ -154,9 +164,9 @@ public Connection connect(Configuration config) throws IOException { * @return ES settings builder configured according to the {@code config} parameter * @throws java.io.IOException if conf-file was set but could not be read */ - private static ImmutableSettings.Builder settingsBuilder(Configuration config) throws IOException { + private static Settings.Builder settingsBuilder(Configuration config) throws IOException { - ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder(); + Settings.Builder settings = Settings.settingsBuilder(); // Set Titan defaults settings.put("client.transport.ignore_cluster_name", true); @@ -185,20 +195,21 @@ private static ImmutableSettings.Builder settingsBuilder(Configuration config) t } // Force-enable dynamic scripting. This is probably only useful in Node mode. - String disableScriptsKey = "script.disable_dynamic"; + String disableScriptsKey = "script.inline"; String disableScriptsVal = settings.get(disableScriptsKey); - if (null != disableScriptsVal && !"false".equals(disableScriptsVal)) { - log.warn("Titan requires Elasticsearch dynamic scripting. Setting {} to false. " + + if (null != disableScriptsVal && !"on".equals(disableScriptsVal)) { + log.warn("Titan requires Elasticsearch dynamic scripting. Setting {} to 'on'. " + "Dynamic scripting must be allowed in the Elasticsearch cluster configuration.", disableScriptsKey); } - settings.put(disableScriptsKey, false); - log.debug("Set {}: {}", disableScriptsKey, false); + settings.put(disableScriptsKey, "on"); + settings.put("path.home", System.getProperty("java.io.tmpdir")); + log.debug("Set {}: {}", disableScriptsKey, "on"); return settings; } - static void applySettingsFromFile(ImmutableSettings.Builder settings, + static void applySettingsFromFile(Settings.Builder settings, Configuration config, ConfigOption confFileOption) throws FileNotFoundException { if (config.has(confFileOption)) { @@ -216,7 +227,7 @@ static void applySettingsFromFile(ImmutableSettings.Builder settings, } } - static void applySettingsFromTitanConf(ImmutableSettings.Builder settings, + static void applySettingsFromTitanConf(Settings.Builder settings, Configuration config, ConfigNamespace rootNS) { int keysLoaded = 0; @@ -248,7 +259,7 @@ static void applySettingsFromTitanConf(ImmutableSettings.Builder settings, } - private static void makeLocalDirsIfNecessary(ImmutableSettings.Builder settingsBuilder, Configuration config) { + private static void makeLocalDirsIfNecessary(Settings.Builder settingsBuilder, Configuration config) { if (config.has(INDEX_DIRECTORY)) { String dataDirectory = config.get(INDEX_DIRECTORY); File f = new File(dataDirectory); diff --git a/titan-es/src/main/java/org/elasticsearch/bootstrap/JarHell.java b/titan-es/src/main/java/org/elasticsearch/bootstrap/JarHell.java new file mode 100644 index 0000000000..7060d85ffc --- /dev/null +++ b/titan-es/src/main/java/org/elasticsearch/bootstrap/JarHell.java @@ -0,0 +1,70 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.bootstrap; + +import org.elasticsearch.common.io.PathUtils; + +import java.net.MalformedURLException; +import java.net.URL; + +/** + * This class masks the Elasticsearch class of the same name. + * Clients are responsible for ensuring their classpath is sane. + */ +public class JarHell { + + public static void checkJarHell() { } + + public static URL[] parseClassPath() { + return parseClassPath(System.getProperty("java.class.path")); + } + + static URL[] parseClassPath(String classPath) { + String pathSeparator = System.getProperty("path.separator"); + String fileSeparator = System.getProperty("file.separator"); + String elements[] = classPath.split(pathSeparator); + URL urlElements[] = new URL[elements.length]; + for (int i = 0; i < elements.length; i++) { + String element = elements[i]; + if (element.isEmpty()) { + throw new IllegalStateException("Classpath should not contain empty elements! (outdated shell script from a previous version?) classpath='" + classPath + "'"); + } + if (element.startsWith("/") && "\\".equals(fileSeparator)) { + element = element.replace("/", "\\"); + if (element.length() >= 3 && element.charAt(2) == ':') { + element = element.substring(1); + } + } + try { + urlElements[i] = PathUtils.get(element).toUri().toURL(); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + return urlElements; + } + + public static void checkJarHell(URL urls[]) { } + + public static void checkVersionFormat(String targetVersion) { } + + public static void checkJavaVersion(String resource, String targetVersion) { } + +} diff --git a/titan-es/src/test/bin/elasticsearch b/titan-es/src/test/bin/elasticsearch index 518ec37ff4..334d4c3f8e 100755 --- a/titan-es/src/test/bin/elasticsearch +++ b/titan-es/src/test/bin/elasticsearch @@ -1,9 +1,5 @@ #!/bin/sh -# OPTIONS: -# -d: daemonize, start in the background -# -p : log the pid to a file (useful to kill it later) - # CONTROLLING STARTUP: # # This script relies on few environment variables to determine startup @@ -46,16 +42,16 @@ # Be aware that you will be entirely responsible for populating the needed # environment variables. - # Maven will replace the project.name with elasticsearch below. If that # hasn't been done, we assume that this is not a packaged version and the # user has forgotten to run Maven to create a package. -IS_PACKAGED_VERSION='elasticsearch' -if [ "$IS_PACKAGED_VERSION" != "elasticsearch" ]; then + +IS_PACKAGED_VERSION='distributions' +if [ "$IS_PACKAGED_VERSION" != "distributions" ]; then cat >&2 << EOF Error: You must build the project with Maven or download a pre-built package before you can run Elasticsearch. See 'Building from Source' in README.textile -or visit http://www.elasticsearch.org/download to get a pre-built package. +or visit https://www.elastic.co/download to get a pre-built package. EOF exit 1 fi @@ -83,7 +79,7 @@ ES_HOME=`cd "$ES_HOME"; pwd` #### Start Titan-specific edit ES_INCLUDE="$ES_HOME/bin/elasticsearch.in.sh" -ES_CLASSPATH="`cat $ES_HOME/target/es_classpath.txt`" +ES_CLASSPATH="$ES_HOME/target/es_jarhell.jar:`cat $ES_HOME/target/es_classpath.txt`" #### End Titan-specific edit # If an include wasn't specified in the environment, then search for one... @@ -93,6 +89,7 @@ if [ "x$ES_INCLUDE" = "x" ]; then /usr/local/share/elasticsearch/elasticsearch.in.sh \ /opt/elasticsearch/elasticsearch.in.sh \ ~/.elasticsearch.in.sh \ + "$ES_HOME/bin/elasticsearch.in.sh" \ "`dirname "$0"`"/elasticsearch.in.sh; do if [ -r "$include" ]; then . "$include" @@ -120,6 +117,14 @@ if [ -z "$ES_CLASSPATH" ]; then exit 1 fi +# don't let JAVA_TOOL_OPTIONS slip in (e.g. crazy agents in ubuntu) +# works around https://bugs.launchpad.net/ubuntu/+source/jayatana/+bug/1441487 +if [ "x$JAVA_TOOL_OPTIONS" != "x" ]; then + echo "Warning: Ignoring JAVA_TOOL_OPTIONS=$JAVA_TOOL_OPTIONS" + echo "Please pass JVM parameters via JAVA_OPTS instead" + unset JAVA_TOOL_OPTIONS +fi + # Special-case path variables. case `uname` in CYGWIN*) @@ -128,91 +133,29 @@ case `uname` in ;; esac -launch_service() -{ - pidpath=$1 - daemonized=$2 - props=$3 - es_parms="-Delasticsearch" +# full hostname passed through cut for portability on systems that do not support hostname -s +# export on separate line for shells that do not support combining definition and export +HOSTNAME=`hostname | cut -d. -f1` +export HOSTNAME - if [ "x$pidpath" != "x" ]; then - es_parms="$es_parms -Des.pidfile=$pidpath" +# manual parsing to find out, if process should be detached +daemonized=`echo $* | egrep -- '(^-d |-d$| -d |--daemonize$|--daemonize )'` +if [ -z "$daemonized" ] ; then + exec "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS -Des.path.home="$ES_HOME" -cp "$ES_CLASSPATH" \ + org.elasticsearch.bootstrap.Elasticsearch start "$@" +else + exec "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS -Des.path.home="$ES_HOME" -cp "$ES_CLASSPATH" \ + org.elasticsearch.bootstrap.Elasticsearch start "$@" <&- & + retval=$? + pid=$! + [ $retval -eq 0 ] || exit $retval + if [ ! -z "$ES_STARTUP_SLEEP_TIME" ]; then + sleep $ES_STARTUP_SLEEP_TIME fi - - # The es-foreground option will tell Elasticsearch not to close stdout/stderr, but it's up to us not to daemonize. - if [ "x$daemonized" = "x" ]; then - es_parms="$es_parms -Des.foreground=yes" - exec "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS $es_parms -Des.path.home="$ES_HOME" -cp "$ES_CLASSPATH" $props \ - org.elasticsearch.bootstrap.Elasticsearch - # exec without running it in the background, makes it replace this shell, we'll never get here... - # no need to return something - else - # Startup Elasticsearch, background it, and write the pid. - exec "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS $es_parms -Des.path.home="$ES_HOME" -cp "$ES_CLASSPATH" $props \ - org.elasticsearch.bootstrap.Elasticsearch <&- & - return $? + if ! ps -p $pid > /dev/null ; then + exit 1 fi -} - -# Parse any long getopt options and put them into properties before calling getopt below -# Be dash compatible to make sure running under ubuntu works -ARGV="" -while [ $# -gt 0 ] -do - case $1 in - --*=*) properties="$properties -Des.${1#--}" - shift 1 - ;; - --*) properties="$properties -Des.${1#--}=$2" - shift 2 - ;; - *) ARGV="$ARGV $1" ; shift - esac -done - -# Parse any command line options. -args=`getopt vdhp:D:X: $ARGV` -eval set -- "$args" - -while true; do - case $1 in - -v) - "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS $es_parms -Des.path.home="$ES_HOME" -cp "$ES_CLASSPATH" $props \ - org.elasticsearch.Version - exit 0 - ;; - -p) - pidfile="$2" - shift 2 - ;; - -d) - daemonized="yes" - shift - ;; - -h) - echo "Usage: $0 [-d] [-h] [-p pidfile]" - exit 0 - ;; - -D) - properties="$properties -D$2" - shift 2 - ;; - -X) - properties="$properties -X$2" - shift 2 - ;; - --) - shift - break - ;; - *) - echo "Error parsing argument $1!" >&2 - exit 1 - ;; - esac -done - -# Start up the service -launch_service "$pidfile" "$daemonized" "$properties" + exit 0 +fi exit $? diff --git a/titan-es/src/test/bin/elasticsearch.in.sh b/titan-es/src/test/bin/elasticsearch.in.sh index 4263fdfc0a..543d5a86cb 100755 --- a/titan-es/src/test/bin/elasticsearch.in.sh +++ b/titan-es/src/test/bin/elasticsearch.in.sh @@ -1,6 +1,18 @@ #!/bin/sh -ES_CLASSPATH=$ES_CLASSPATH:$ES_HOME/lib/elasticsearch-1.2.1.jar:$ES_HOME/lib/*:$ES_HOME/lib/sigar/* +# check in case a user was using this mechanism +#### Start Titan-specific edit (commenting out) +#if [ "x$ES_CLASSPATH" != "x" ]; then +# cat >&2 << EOF +#Error: Don't modify the classpath with ES_CLASSPATH. Best is to add +#additional elements via the plugin mechanism, or if code must really be +#added to the main classpath, add jars to lib/ (unsupported). +#EOF +# exit 1 +#fi +# +#ES_CLASSPATH="$ES_HOME/lib/elasticsearch-2.3.3.jar:$ES_HOME/lib/*" +#### End Titan-specific edit if [ "x$ES_MIN_MEM" = "x" ]; then ES_MIN_MEM=256m @@ -30,9 +42,6 @@ if [ "x$ES_DIRECT_SIZE" != "x" ]; then JAVA_OPTS="$JAVA_OPTS -XX:MaxDirectMemorySize=${ES_DIRECT_SIZE}" fi -# reduce the per-thread stack size -JAVA_OPTS="$JAVA_OPTS -Xss256k" - # set to headless, just in case JAVA_OPTS="$JAVA_OPTS -Djava.awt.headless=true" @@ -41,20 +50,28 @@ if [ "x$ES_USE_IPV4" != "x" ]; then JAVA_OPTS="$JAVA_OPTS -Djava.net.preferIPv4Stack=true" fi -JAVA_OPTS="$JAVA_OPTS -XX:+UseParNewGC" -JAVA_OPTS="$JAVA_OPTS -XX:+UseConcMarkSweepGC" +# Add gc options. ES_GC_OPTS is unsupported, for internal testing +if [ "x$ES_GC_OPTS" = "x" ]; then + ES_GC_OPTS="$ES_GC_OPTS -XX:+UseParNewGC" + ES_GC_OPTS="$ES_GC_OPTS -XX:+UseConcMarkSweepGC" + ES_GC_OPTS="$ES_GC_OPTS -XX:CMSInitiatingOccupancyFraction=75" + ES_GC_OPTS="$ES_GC_OPTS -XX:+UseCMSInitiatingOccupancyOnly" +fi -JAVA_OPTS="$JAVA_OPTS -XX:CMSInitiatingOccupancyFraction=75" -JAVA_OPTS="$JAVA_OPTS -XX:+UseCMSInitiatingOccupancyOnly" +JAVA_OPTS="$JAVA_OPTS $ES_GC_OPTS" # GC logging options -if [ "x$ES_USE_GC_LOGGING" != "x" ]; then +if [ -n "$ES_GC_LOG_FILE" ]; then JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCDetails" JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCTimeStamps" + JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCDateStamps" JAVA_OPTS="$JAVA_OPTS -XX:+PrintClassHistogram" JAVA_OPTS="$JAVA_OPTS -XX:+PrintTenuringDistribution" JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCApplicationStoppedTime" - JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/elasticsearch/gc.log" + JAVA_OPTS="$JAVA_OPTS -Xloggc:$ES_GC_LOG_FILE" + + # Ensure that the directory for the log file exists: the JVM will not create it. + mkdir -p "`dirname \"$ES_GC_LOG_FILE\"`" fi # Causes the JVM to dump its heap on OutOfMemory. @@ -62,3 +79,12 @@ JAVA_OPTS="$JAVA_OPTS -XX:+HeapDumpOnOutOfMemoryError" # The path to the heap dump location, note directory must exists and have enough # space for a full heap dump. #JAVA_OPTS="$JAVA_OPTS -XX:HeapDumpPath=$ES_HOME/logs/heapdump.hprof" + +# Disables explicit GC +JAVA_OPTS="$JAVA_OPTS -XX:+DisableExplicitGC" + +# Ensure UTF-8 encoding by default (e.g. filenames) +JAVA_OPTS="$JAVA_OPTS -Dfile.encoding=UTF-8" + +# Use our provided JNA always versus the system one +JAVA_OPTS="$JAVA_OPTS -Djna.nosys=true" diff --git a/titan-es/src/test/config/elasticsearch.yml b/titan-es/src/test/config/elasticsearch.yml index 89b9ef291c..0cc9d632d8 100644 --- a/titan-es/src/test/config/elasticsearch.yml +++ b/titan-es/src/test/config/elasticsearch.yml @@ -1,378 +1,97 @@ -##################### Elasticsearch Configuration Example ##################### - -# This file contains an overview of various configuration settings, -# targeted at operations staff. Application developers should -# consult the guide at . -# -# The installation procedure is covered at -# . +# ======================== Elasticsearch Configuration ========================= # -# Elasticsearch comes with reasonable defaults for most settings, -# so you can try it out without bothering with configuration. -# -# Most of the time, these defaults are just fine for running a production -# cluster. If you're fine-tuning your cluster, or wondering about the -# effect of certain configuration option, please _do ask_ on the -# mailing list or IRC channel [http://elasticsearch.org/community]. - -# Any element in the configuration can be replaced with environment variables -# by placing them in ${...} notation. For example: +# NOTE: Elasticsearch comes with reasonable defaults for most settings. +# Before you set out to tweak and tune the configuration, make sure you +# understand what are you trying to accomplish and the consequences. # -#node.rack: ${RACK_ENV_VAR} - -# For information on supported formats and syntax for the config file, see -# - - -################################### Cluster ################################### - -# Cluster name identifies your cluster for auto-discovery. If you're running -# multiple clusters on the same network, make sure you're using unique names. +# The primary way of configuring a node is via this file. This template lists +# the most important settings you may want to configure for a production cluster. # -#cluster.name: elasticsearch - - -#################################### Node ##################################### - -# Node names are generated dynamically on startup, so you're relieved -# from configuring them manually. You can tie this node to a specific name: +# Please see the documentation for further information on configuration options: +# # -#node.name: "Franz Kafka" - -# Every node can be configured to allow or deny being eligible as the master, -# and to allow or deny to store the data. +# ---------------------------------- Cluster ----------------------------------- # -# Allow this node to be eligible as a master node (enabled by default): +# Use a descriptive name for your cluster: # -#node.master: true +# cluster.name: my-application # -# Allow this node to store data (enabled by default): +# ------------------------------------ Node ------------------------------------ # -#node.data: true - -# You can exploit these settings to design advanced cluster topologies. +# Use a descriptive name for the node: # -# 1. You want this node to never become a master node, only to hold data. -# This will be the "workhorse" of your cluster. +# node.name: node-1 # -#node.master: false -#node.data: true +# Add custom attributes to the node: # -# 2. You want this node to only serve as a master: to not store any data and -# to have free resources. This will be the "coordinator" of your cluster. +# node.rack: r1 # -#node.master: true -#node.data: false +# ----------------------------------- Paths ------------------------------------ # -# 3. You want this node to be neither master nor data node, but -# to act as a "search load balancer" (fetching data from nodes, -# aggregating results, etc.) -# -#node.master: false -#node.data: false - -# Use the Cluster Health API [http://localhost:9200/_cluster/health], the -# Node Info API [http://localhost:9200/_nodes] or GUI tools -# such as , -# , -# and -# to inspect the cluster state. - -# A node can have generic attributes associated with it, which can later be used -# for customized shard allocation filtering, or allocation awareness. An attribute -# is a simple key value pair, similar to node.key: value, here is an example: -# -#node.rack: rack314 - -# By default, multiple nodes are allowed to start from the same installation location -# to disable it, set the following: -#node.max_local_storage_nodes: 1 - - -#################################### Index #################################### - -# You can set a number of options (such as shard/replica options, mapping -# or analyzer definitions, translog settings, ...) for indices globally, -# in this file. -# -# Note, that it makes more sense to configure index settings specifically for -# a certain index, either when creating it or by using the index templates API. -# -# See and -# -# for more information. - -# Set the number of shards (splits) of an index (5 by default): -# -#index.number_of_shards: 5 - -# Set the number of replicas (additional copies) of an index (1 by default): -# -#index.number_of_replicas: 1 - -# Note, that for development on a local machine, with small indices, it usually -# makes sense to "disable" the distributed features: -# -#index.number_of_shards: 1 -#index.number_of_replicas: 0 - -# These settings directly affect the performance of index and search operations -# in your cluster. Assuming you have enough machines to hold shards and -# replicas, the rule of thumb is: -# -# 1. Having more *shards* enhances the _indexing_ performance and allows to -# _distribute_ a big index across machines. -# 2. Having more *replicas* enhances the _search_ performance and improves the -# cluster _availability_. -# -# The "number_of_shards" is a one-time setting for an index. -# -# The "number_of_replicas" can be increased or decreased anytime, -# by using the Index Update Settings API. -# -# Elasticsearch takes care about load balancing, relocating, gathering the -# results from nodes, etc. Experiment with different settings to fine-tune -# your setup. - -# Use the Index Status API () to inspect -# the index status. - - -#################################### Paths #################################### - -# Path to directory containing configuration (this file and logging.yml): -# -path.conf: $MAVEN{project.basedir}/config - -# Path to directory where to store index data allocated for this node. +# Path to directory where to store the data (separate multiple locations by comma): # path.data: $MAVEN{project.build.directory}/es-data - -# -# Can optionally include more than one location, causing data to be striped across -# the locations (a la RAID 0) on a file level, favouring locations with most free -# space on creation. For example: -# -#path.data: /path/to/data1,/path/to/data2 - -# Path to temporary files: # -path.work: $MAVEN{project.build.directory}/es-work - # Path to log files: # path.logs: $MAVEN{project.build.directory}/es-logs - -# Path to where plugins are installed: # -#path.plugins: /path/to/plugins - - -#################################### Plugin ################################### - -# If a plugin listed here is not installed for current node, the node will not start. +# ----------------------------------- Memory ----------------------------------- # -#plugin.mandatory: mapper-attachments,lang-groovy - - -################################### Memory #################################### - -# Elasticsearch performs poorly when JVM starts swapping: you should ensure that -# it _never_ swaps. +# Lock the memory on startup: # -# Set this property to true to lock the memory: +# bootstrap.mlockall: true # -#bootstrap.mlockall: true - -# Make sure that the ES_MIN_MEM and ES_MAX_MEM environment variables are set -# to the same value, and that the machine has enough memory to allocate -# for Elasticsearch, leaving enough memory for the operating system itself. +# Make sure that the `ES_HEAP_SIZE` environment variable is set to about half the memory +# available on the system and that the owner of the process is allowed to use this limit. # -# You should also make sure that the Elasticsearch process is allowed to lock -# the memory, eg. by using `ulimit -l unlimited`. - - -############################## Network And HTTP ############################### - -# Elasticsearch, by default, binds itself to the 0.0.0.0 address, and listens -# on port [9200-9300] for HTTP traffic and on port [9300-9400] for node-to-node -# communication. (the range means that if the port is busy, it will automatically -# try the next port). - -# Set the bind address specifically (IPv4 or IPv6): +# Elasticsearch performs poorly when the system is swapping the memory. # -network.bind_host: 127.0.0.1 - -# Set the address other nodes will use to communicate with this node. If not -# set, it is automatically derived. It must point to an actual IP address. +# ---------------------------------- Network ----------------------------------- # -network.publish_host: 127.0.0.1 - -# Set both 'bind_host' and 'publish_host': +# Set the bind address to a specific IP (IPv4 or IPv6): # network.host: 127.0.0.1 - -# Set a custom port for the node to node communication (9300 by default): -# -#transport.tcp.port: 9300 - -# Enable compression for all communication between nodes (disabled by default): -# -#transport.tcp.compress: true - -# Set a custom port to listen for HTTP traffic: -# -#http.port: 9200 - -# Set a custom allowed content length: -# -#http.max_content_length: 100mb - -# Disable HTTP completely: -# -#http.enabled: false - - -################################### Gateway ################################### - -# The gateway allows for persisting the cluster state between full cluster -# restarts. Every change to the state (such as adding an index) will be stored -# in the gateway, and when the cluster starts up for the first time, -# it will read its state from the gateway. - -# There are several types of gateway implementations. For more information, see -# . - -# The default gateway type is the "local" gateway (recommended): -# -#gateway.type: local - -# Settings below control how and when to start the initial recovery process on -# a full cluster restart (to reuse as much local data as possible when using shared -# gateway). - -# Allow recovery process after N nodes in a cluster are up: -# -#gateway.recover_after_nodes: 1 - -# Set the timeout to initiate the recovery process, once the N nodes -# from previous setting are up (accepts time value): # -#gateway.recover_after_time: 5m - -# Set how many nodes are expected in this cluster. Once these N nodes -# are up (and recover_after_nodes is met), begin recovery process immediately -# (without waiting for recover_after_time to expire): +# Set a custom port for HTTP: # -#gateway.expected_nodes: 2 - - -############################# Recovery Throttling ############################# - -# These settings allow to control the process of shards allocation between -# nodes during initial recovery, replica allocation, rebalancing, -# or when adding and removing nodes. - -# Set the number of concurrent recoveries happening on a node: -# -# 1. During the initial recovery +# http.port: 9200 # -#cluster.routing.allocation.node_initial_primaries_recoveries: 4 +# For more information, see the documentation at: +# # -# 2. During adding/removing nodes, rebalancing, etc +# --------------------------------- Discovery ---------------------------------- # -#cluster.routing.allocation.node_concurrent_recoveries: 2 - -# Set to throttle throughput when recovering (eg. 100mb, by default 20mb): -# -#indices.recovery.max_bytes_per_sec: 20mb - -# Set to limit the number of open concurrent streams when -# recovering a shard from a peer: -# -#indices.recovery.concurrent_streams: 5 - - -################################## Discovery ################################## - -# Discovery infrastructure ensures nodes can be found within a cluster -# and master node is elected. Multicast discovery is the default. - -# Set to ensure a node sees N other master eligible nodes to be considered -# operational within the cluster. Its recommended to set it to a higher value -# than 1 when running more than 2 nodes in the cluster. +# Pass an initial list of hosts to perform discovery when new node is started: +# The default list of hosts is ["127.0.0.1", "[::1]"] # -#discovery.zen.minimum_master_nodes: 1 - -# Set the time to wait for ping responses from other nodes when discovering. -# Set this option to a higher value on a slow or congested network -# to minimize discovery failures: +# discovery.zen.ping.unicast.hosts: ["host1", "host2"] # -#discovery.zen.ping.timeout: 3s - -# For more information, see -# - -# Unicast discovery allows to explicitly control which nodes will be used -# to discover the cluster. It can be used when multicast is not present, -# or to restrict the cluster communication-wise. +# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1): # -# 1. Disable multicast discovery (enabled by default): +# discovery.zen.minimum_master_nodes: 3 # -#discovery.zen.ping.multicast.enabled: false +# For more information, see the documentation at: +# # -# 2. Configure an initial list of master nodes in the cluster -# to perform discovery when new nodes (master or data) are started: +# ---------------------------------- Gateway ----------------------------------- # -#discovery.zen.ping.unicast.hosts: ["host1", "host2:port"] - -# EC2 discovery allows to use AWS EC2 API in order to perform discovery. +# Block initial recovery after a full cluster restart until N nodes are started: # -# You have to install the cloud-aws plugin for enabling the EC2 discovery. +# gateway.recover_after_nodes: 3 # -# For more information, see -# +# For more information, see the documentation at: +# # -# See -# for a step-by-step tutorial. - -# GCE discovery allows to use Google Compute Engine API in order to perform discovery. +# ---------------------------------- Various ----------------------------------- # -# You have to install the cloud-gce plugin for enabling the GCE discovery. +# Disable starting multiple nodes on a single system: # -# For more information, see . - -# Azure discovery allows to use Azure API in order to perform discovery. +# node.max_local_storage_nodes: 1 # -# You have to install the cloud-azure plugin for enabling the Azure discovery. +# Require explicit names when deleting indices: # -# For more information, see . - -################################## Slow Log ################################## - -# Shard level query and fetch threshold logging. - -#index.search.slowlog.threshold.query.warn: 10s -#index.search.slowlog.threshold.query.info: 5s -#index.search.slowlog.threshold.query.debug: 2s -#index.search.slowlog.threshold.query.trace: 500ms - -#index.search.slowlog.threshold.fetch.warn: 1s -#index.search.slowlog.threshold.fetch.info: 800ms -#index.search.slowlog.threshold.fetch.debug: 500ms -#index.search.slowlog.threshold.fetch.trace: 200ms - -#index.indexing.slowlog.threshold.index.warn: 10s -#index.indexing.slowlog.threshold.index.info: 5s -#index.indexing.slowlog.threshold.index.debug: 2s -#index.indexing.slowlog.threshold.index.trace: 500ms - -################################## GC Logging ################################ +# action.destructive_requires_name: true -#monitor.jvm.gc.young.warn: 1000ms -#monitor.jvm.gc.young.info: 700ms -#monitor.jvm.gc.young.debug: 400ms +index.max_result_window: 10000000 -#monitor.jvm.gc.old.warn: 10s -#monitor.jvm.gc.old.info: 5s -#monitor.jvm.gc.old.debug: 2s diff --git a/titan-es/src/test/config/indexCreationOptions.yml b/titan-es/src/test/config/indexCreationOptions.yml deleted file mode 100644 index d91c8867a1..0000000000 --- a/titan-es/src/test/config/indexCreationOptions.yml +++ /dev/null @@ -1,378 +0,0 @@ -##################### Elasticsearch Configuration Example ##################### - -# This file contains an overview of various configuration settings, -# targeted at operations staff. Application developers should -# consult the guide at . -# -# The installation procedure is covered at -# . -# -# Elasticsearch comes with reasonable defaults for most settings, -# so you can try it out without bothering with configuration. -# -# Most of the time, these defaults are just fine for running a production -# cluster. If you're fine-tuning your cluster, or wondering about the -# effect of certain configuration option, please _do ask_ on the -# mailing list or IRC channel [http://elasticsearch.org/community]. - -# Any element in the configuration can be replaced with environment variables -# by placing them in ${...} notation. For example: -# -#node.rack: ${RACK_ENV_VAR} - -# For information on supported formats and syntax for the config file, see -# - - -################################### Cluster ################################### - -# Cluster name identifies your cluster for auto-discovery. If you're running -# multiple clusters on the same network, make sure you're using unique names. -# -cluster.name: indexCreationOptions - - -#################################### Node ##################################### - -# Node names are generated dynamically on startup, so you're relieved -# from configuring them manually. You can tie this node to a specific name: -# -#node.name: "Franz Kafka" - -# Every node can be configured to allow or deny being eligible as the master, -# and to allow or deny to store the data. -# -# Allow this node to be eligible as a master node (enabled by default): -# -#node.master: true -# -# Allow this node to store data (enabled by default): -# -#node.data: true - -# You can exploit these settings to design advanced cluster topologies. -# -# 1. You want this node to never become a master node, only to hold data. -# This will be the "workhorse" of your cluster. -# -#node.master: false -#node.data: true -# -# 2. You want this node to only serve as a master: to not store any data and -# to have free resources. This will be the "coordinator" of your cluster. -# -#node.master: true -#node.data: false -# -# 3. You want this node to be neither master nor data node, but -# to act as a "search load balancer" (fetching data from nodes, -# aggregating results, etc.) -# -#node.master: false -#node.data: false - -# Use the Cluster Health API [http://localhost:9200/_cluster/health], the -# Node Info API [http://localhost:9200/_nodes] or GUI tools -# such as , -# , -# and -# to inspect the cluster state. - -# A node can have generic attributes associated with it, which can later be used -# for customized shard allocation filtering, or allocation awareness. An attribute -# is a simple key value pair, similar to node.key: value, here is an example: -# -#node.rack: rack314 - -# By default, multiple nodes are allowed to start from the same installation location -# to disable it, set the following: -#node.max_local_storage_nodes: 1 - - -#################################### Index #################################### - -# You can set a number of options (such as shard/replica options, mapping -# or analyzer definitions, translog settings, ...) for indices globally, -# in this file. -# -# Note, that it makes more sense to configure index settings specifically for -# a certain index, either when creating it or by using the index templates API. -# -# See and -# -# for more information. - -# Set the number of shards (splits) of an index (5 by default): -# -#index.number_of_shards: 5 - -# Set the number of replicas (additional copies) of an index (1 by default): -# -#index.number_of_replicas: 1 - -# Note, that for development on a local machine, with small indices, it usually -# makes sense to "disable" the distributed features: -# -#index.number_of_shards: 1 -#index.number_of_replicas: 0 - -# These settings directly affect the performance of index and search operations -# in your cluster. Assuming you have enough machines to hold shards and -# replicas, the rule of thumb is: -# -# 1. Having more *shards* enhances the _indexing_ performance and allows to -# _distribute_ a big index across machines. -# 2. Having more *replicas* enhances the _search_ performance and improves the -# cluster _availability_. -# -# The "number_of_shards" is a one-time setting for an index. -# -# The "number_of_replicas" can be increased or decreased anytime, -# by using the Index Update Settings API. -# -# Elasticsearch takes care about load balancing, relocating, gathering the -# results from nodes, etc. Experiment with different settings to fine-tune -# your setup. - -# Use the Index Status API () to inspect -# the index status. - - -#################################### Paths #################################### - -# Path to directory containing configuration (this file and logging.yml): -# -path.conf: $MAVEN{project.basedir}/config - -# Path to directory where to store index data allocated for this node. -# -path.data: $MAVEN{project.build.directory}/es-data - -# -# Can optionally include more than one location, causing data to be striped across -# the locations (a la RAID 0) on a file level, favouring locations with most free -# space on creation. For example: -# -#path.data: /path/to/data1,/path/to/data2 - -# Path to temporary files: -# -path.work: $MAVEN{project.build.directory}/es-work - -# Path to log files: -# -path.logs: $MAVEN{project.build.directory}/es-logs - -# Path to where plugins are installed: -# -#path.plugins: /path/to/plugins - - -#################################### Plugin ################################### - -# If a plugin listed here is not installed for current node, the node will not start. -# -#plugin.mandatory: mapper-attachments,lang-groovy - - -################################### Memory #################################### - -# Elasticsearch performs poorly when JVM starts swapping: you should ensure that -# it _never_ swaps. -# -# Set this property to true to lock the memory: -# -#bootstrap.mlockall: true - -# Make sure that the ES_MIN_MEM and ES_MAX_MEM environment variables are set -# to the same value, and that the machine has enough memory to allocate -# for Elasticsearch, leaving enough memory for the operating system itself. -# -# You should also make sure that the Elasticsearch process is allowed to lock -# the memory, eg. by using `ulimit -l unlimited`. - - -############################## Network And HTTP ############################### - -# Elasticsearch, by default, binds itself to the 0.0.0.0 address, and listens -# on port [9200-9300] for HTTP traffic and on port [9300-9400] for node-to-node -# communication. (the range means that if the port is busy, it will automatically -# try the next port). - -# Set the bind address specifically (IPv4 or IPv6): -# -network.bind_host: 127.0.0.1 - -# Set the address other nodes will use to communicate with this node. If not -# set, it is automatically derived. It must point to an actual IP address. -# -network.publish_host: 127.0.0.1 - -# Set both 'bind_host' and 'publish_host': -# -network.host: 127.0.0.1 - -# Set a custom port for the node to node communication (9300 by default): -# -#transport.tcp.port: 9300 - -# Enable compression for all communication between nodes (disabled by default): -# -#transport.tcp.compress: true - -# Set a custom port to listen for HTTP traffic: -# -#http.port: 9200 - -# Set a custom allowed content length: -# -#http.max_content_length: 100mb - -# Disable HTTP completely: -# -#http.enabled: false - - -################################### Gateway ################################### - -# The gateway allows for persisting the cluster state between full cluster -# restarts. Every change to the state (such as adding an index) will be stored -# in the gateway, and when the cluster starts up for the first time, -# it will read its state from the gateway. - -# There are several types of gateway implementations. For more information, see -# . - -# The default gateway type is the "local" gateway (recommended): -# -#gateway.type: local - -# Settings below control how and when to start the initial recovery process on -# a full cluster restart (to reuse as much local data as possible when using shared -# gateway). - -# Allow recovery process after N nodes in a cluster are up: -# -#gateway.recover_after_nodes: 1 - -# Set the timeout to initiate the recovery process, once the N nodes -# from previous setting are up (accepts time value): -# -#gateway.recover_after_time: 5m - -# Set how many nodes are expected in this cluster. Once these N nodes -# are up (and recover_after_nodes is met), begin recovery process immediately -# (without waiting for recover_after_time to expire): -# -#gateway.expected_nodes: 2 - - -############################# Recovery Throttling ############################# - -# These settings allow to control the process of shards allocation between -# nodes during initial recovery, replica allocation, rebalancing, -# or when adding and removing nodes. - -# Set the number of concurrent recoveries happening on a node: -# -# 1. During the initial recovery -# -#cluster.routing.allocation.node_initial_primaries_recoveries: 4 -# -# 2. During adding/removing nodes, rebalancing, etc -# -#cluster.routing.allocation.node_concurrent_recoveries: 2 - -# Set to throttle throughput when recovering (eg. 100mb, by default 20mb): -# -#indices.recovery.max_bytes_per_sec: 20mb - -# Set to limit the number of open concurrent streams when -# recovering a shard from a peer: -# -#indices.recovery.concurrent_streams: 5 - - -################################## Discovery ################################## - -# Discovery infrastructure ensures nodes can be found within a cluster -# and master node is elected. Multicast discovery is the default. - -# Set to ensure a node sees N other master eligible nodes to be considered -# operational within the cluster. Its recommended to set it to a higher value -# than 1 when running more than 2 nodes in the cluster. -# -#discovery.zen.minimum_master_nodes: 1 - -# Set the time to wait for ping responses from other nodes when discovering. -# Set this option to a higher value on a slow or congested network -# to minimize discovery failures: -# -#discovery.zen.ping.timeout: 3s - -# For more information, see -# - -# Unicast discovery allows to explicitly control which nodes will be used -# to discover the cluster. It can be used when multicast is not present, -# or to restrict the cluster communication-wise. -# -# 1. Disable multicast discovery (enabled by default): -# -#discovery.zen.ping.multicast.enabled: false -# -# 2. Configure an initial list of master nodes in the cluster -# to perform discovery when new nodes (master or data) are started: -# -#discovery.zen.ping.unicast.hosts: ["host1", "host2:port"] - -# EC2 discovery allows to use AWS EC2 API in order to perform discovery. -# -# You have to install the cloud-aws plugin for enabling the EC2 discovery. -# -# For more information, see -# -# -# See -# for a step-by-step tutorial. - -# GCE discovery allows to use Google Compute Engine API in order to perform discovery. -# -# You have to install the cloud-gce plugin for enabling the GCE discovery. -# -# For more information, see . - -# Azure discovery allows to use Azure API in order to perform discovery. -# -# You have to install the cloud-azure plugin for enabling the Azure discovery. -# -# For more information, see . - -################################## Slow Log ################################## - -# Shard level query and fetch threshold logging. - -#index.search.slowlog.threshold.query.warn: 10s -#index.search.slowlog.threshold.query.info: 5s -#index.search.slowlog.threshold.query.debug: 2s -#index.search.slowlog.threshold.query.trace: 500ms - -#index.search.slowlog.threshold.fetch.warn: 1s -#index.search.slowlog.threshold.fetch.info: 800ms -#index.search.slowlog.threshold.fetch.debug: 500ms -#index.search.slowlog.threshold.fetch.trace: 200ms - -#index.indexing.slowlog.threshold.index.warn: 10s -#index.indexing.slowlog.threshold.index.info: 5s -#index.indexing.slowlog.threshold.index.debug: 2s -#index.indexing.slowlog.threshold.index.trace: 500ms - -################################## GC Logging ################################ - -#monitor.jvm.gc.young.warn: 1000ms -#monitor.jvm.gc.young.info: 700ms -#monitor.jvm.gc.young.debug: 400ms - -#monitor.jvm.gc.old.warn: 10s -#monitor.jvm.gc.old.info: 5s -#monitor.jvm.gc.old.debug: 2s diff --git a/titan-es/src/test/config/networkNodeUsingExt.yml b/titan-es/src/test/config/networkNodeUsingExt.yml deleted file mode 100644 index 7d0e2ac206..0000000000 --- a/titan-es/src/test/config/networkNodeUsingExt.yml +++ /dev/null @@ -1,378 +0,0 @@ -##################### Elasticsearch Configuration Example ##################### - -# This file contains an overview of various configuration settings, -# targeted at operations staff. Application developers should -# consult the guide at . -# -# The installation procedure is covered at -# . -# -# Elasticsearch comes with reasonable defaults for most settings, -# so you can try it out without bothering with configuration. -# -# Most of the time, these defaults are just fine for running a production -# cluster. If you're fine-tuning your cluster, or wondering about the -# effect of certain configuration option, please _do ask_ on the -# mailing list or IRC channel [http://elasticsearch.org/community]. - -# Any element in the configuration can be replaced with environment variables -# by placing them in ${...} notation. For example: -# -#node.rack: ${RACK_ENV_VAR} - -# For information on supported formats and syntax for the config file, see -# - - -################################### Cluster ################################### - -# Cluster name identifies your cluster for auto-discovery. If you're running -# multiple clusters on the same network, make sure you're using unique names. -# -cluster.name: networkNodeUsingExt - - -#################################### Node ##################################### - -# Node names are generated dynamically on startup, so you're relieved -# from configuring them manually. You can tie this node to a specific name: -# -#node.name: "Franz Kafka" - -# Every node can be configured to allow or deny being eligible as the master, -# and to allow or deny to store the data. -# -# Allow this node to be eligible as a master node (enabled by default): -# -#node.master: true -# -# Allow this node to store data (enabled by default): -# -#node.data: true - -# You can exploit these settings to design advanced cluster topologies. -# -# 1. You want this node to never become a master node, only to hold data. -# This will be the "workhorse" of your cluster. -# -#node.master: false -#node.data: true -# -# 2. You want this node to only serve as a master: to not store any data and -# to have free resources. This will be the "coordinator" of your cluster. -# -#node.master: true -#node.data: false -# -# 3. You want this node to be neither master nor data node, but -# to act as a "search load balancer" (fetching data from nodes, -# aggregating results, etc.) -# -#node.master: false -#node.data: false - -# Use the Cluster Health API [http://localhost:9200/_cluster/health], the -# Node Info API [http://localhost:9200/_nodes] or GUI tools -# such as , -# , -# and -# to inspect the cluster state. - -# A node can have generic attributes associated with it, which can later be used -# for customized shard allocation filtering, or allocation awareness. An attribute -# is a simple key value pair, similar to node.key: value, here is an example: -# -#node.rack: rack314 - -# By default, multiple nodes are allowed to start from the same installation location -# to disable it, set the following: -#node.max_local_storage_nodes: 1 - - -#################################### Index #################################### - -# You can set a number of options (such as shard/replica options, mapping -# or analyzer definitions, translog settings, ...) for indices globally, -# in this file. -# -# Note, that it makes more sense to configure index settings specifically for -# a certain index, either when creating it or by using the index templates API. -# -# See and -# -# for more information. - -# Set the number of shards (splits) of an index (5 by default): -# -#index.number_of_shards: 5 - -# Set the number of replicas (additional copies) of an index (1 by default): -# -#index.number_of_replicas: 1 - -# Note, that for development on a local machine, with small indices, it usually -# makes sense to "disable" the distributed features: -# -#index.number_of_shards: 1 -#index.number_of_replicas: 0 - -# These settings directly affect the performance of index and search operations -# in your cluster. Assuming you have enough machines to hold shards and -# replicas, the rule of thumb is: -# -# 1. Having more *shards* enhances the _indexing_ performance and allows to -# _distribute_ a big index across machines. -# 2. Having more *replicas* enhances the _search_ performance and improves the -# cluster _availability_. -# -# The "number_of_shards" is a one-time setting for an index. -# -# The "number_of_replicas" can be increased or decreased anytime, -# by using the Index Update Settings API. -# -# Elasticsearch takes care about load balancing, relocating, gathering the -# results from nodes, etc. Experiment with different settings to fine-tune -# your setup. - -# Use the Index Status API () to inspect -# the index status. - - -#################################### Paths #################################### - -# Path to directory containing configuration (this file and logging.yml): -# -path.conf: $MAVEN{project.basedir}/config - -# Path to directory where to store index data allocated for this node. -# -path.data: $MAVEN{project.build.directory}/es-data - -# -# Can optionally include more than one location, causing data to be striped across -# the locations (a la RAID 0) on a file level, favouring locations with most free -# space on creation. For example: -# -#path.data: /path/to/data1,/path/to/data2 - -# Path to temporary files: -# -path.work: $MAVEN{project.build.directory}/es-work - -# Path to log files: -# -path.logs: $MAVEN{project.build.directory}/es-logs - -# Path to where plugins are installed: -# -#path.plugins: /path/to/plugins - - -#################################### Plugin ################################### - -# If a plugin listed here is not installed for current node, the node will not start. -# -#plugin.mandatory: mapper-attachments,lang-groovy - - -################################### Memory #################################### - -# Elasticsearch performs poorly when JVM starts swapping: you should ensure that -# it _never_ swaps. -# -# Set this property to true to lock the memory: -# -#bootstrap.mlockall: true - -# Make sure that the ES_MIN_MEM and ES_MAX_MEM environment variables are set -# to the same value, and that the machine has enough memory to allocate -# for Elasticsearch, leaving enough memory for the operating system itself. -# -# You should also make sure that the Elasticsearch process is allowed to lock -# the memory, eg. by using `ulimit -l unlimited`. - - -############################## Network And HTTP ############################### - -# Elasticsearch, by default, binds itself to the 0.0.0.0 address, and listens -# on port [9200-9300] for HTTP traffic and on port [9300-9400] for node-to-node -# communication. (the range means that if the port is busy, it will automatically -# try the next port). - -# Set the bind address specifically (IPv4 or IPv6): -# -network.bind_host: 127.0.0.1 - -# Set the address other nodes will use to communicate with this node. If not -# set, it is automatically derived. It must point to an actual IP address. -# -network.publish_host: 127.0.0.1 - -# Set both 'bind_host' and 'publish_host': -# -network.host: 127.0.0.1 - -# Set a custom port for the node to node communication (9300 by default): -# -#transport.tcp.port: 9300 - -# Enable compression for all communication between nodes (disabled by default): -# -#transport.tcp.compress: true - -# Set a custom port to listen for HTTP traffic: -# -#http.port: 9200 - -# Set a custom allowed content length: -# -#http.max_content_length: 100mb - -# Disable HTTP completely: -# -#http.enabled: false - - -################################### Gateway ################################### - -# The gateway allows for persisting the cluster state between full cluster -# restarts. Every change to the state (such as adding an index) will be stored -# in the gateway, and when the cluster starts up for the first time, -# it will read its state from the gateway. - -# There are several types of gateway implementations. For more information, see -# . - -# The default gateway type is the "local" gateway (recommended): -# -#gateway.type: local - -# Settings below control how and when to start the initial recovery process on -# a full cluster restart (to reuse as much local data as possible when using shared -# gateway). - -# Allow recovery process after N nodes in a cluster are up: -# -#gateway.recover_after_nodes: 1 - -# Set the timeout to initiate the recovery process, once the N nodes -# from previous setting are up (accepts time value): -# -#gateway.recover_after_time: 5m - -# Set how many nodes are expected in this cluster. Once these N nodes -# are up (and recover_after_nodes is met), begin recovery process immediately -# (without waiting for recover_after_time to expire): -# -#gateway.expected_nodes: 2 - - -############################# Recovery Throttling ############################# - -# These settings allow to control the process of shards allocation between -# nodes during initial recovery, replica allocation, rebalancing, -# or when adding and removing nodes. - -# Set the number of concurrent recoveries happening on a node: -# -# 1. During the initial recovery -# -#cluster.routing.allocation.node_initial_primaries_recoveries: 4 -# -# 2. During adding/removing nodes, rebalancing, etc -# -#cluster.routing.allocation.node_concurrent_recoveries: 2 - -# Set to throttle throughput when recovering (eg. 100mb, by default 20mb): -# -#indices.recovery.max_bytes_per_sec: 20mb - -# Set to limit the number of open concurrent streams when -# recovering a shard from a peer: -# -#indices.recovery.concurrent_streams: 5 - - -################################## Discovery ################################## - -# Discovery infrastructure ensures nodes can be found within a cluster -# and master node is elected. Multicast discovery is the default. - -# Set to ensure a node sees N other master eligible nodes to be considered -# operational within the cluster. Its recommended to set it to a higher value -# than 1 when running more than 2 nodes in the cluster. -# -#discovery.zen.minimum_master_nodes: 1 - -# Set the time to wait for ping responses from other nodes when discovering. -# Set this option to a higher value on a slow or congested network -# to minimize discovery failures: -# -#discovery.zen.ping.timeout: 3s - -# For more information, see -# - -# Unicast discovery allows to explicitly control which nodes will be used -# to discover the cluster. It can be used when multicast is not present, -# or to restrict the cluster communication-wise. -# -# 1. Disable multicast discovery (enabled by default): -# -#discovery.zen.ping.multicast.enabled: false -# -# 2. Configure an initial list of master nodes in the cluster -# to perform discovery when new nodes (master or data) are started: -# -#discovery.zen.ping.unicast.hosts: ["host1", "host2:port"] - -# EC2 discovery allows to use AWS EC2 API in order to perform discovery. -# -# You have to install the cloud-aws plugin for enabling the EC2 discovery. -# -# For more information, see -# -# -# See -# for a step-by-step tutorial. - -# GCE discovery allows to use Google Compute Engine API in order to perform discovery. -# -# You have to install the cloud-gce plugin for enabling the GCE discovery. -# -# For more information, see . - -# Azure discovery allows to use Azure API in order to perform discovery. -# -# You have to install the cloud-azure plugin for enabling the Azure discovery. -# -# For more information, see . - -################################## Slow Log ################################## - -# Shard level query and fetch threshold logging. - -#index.search.slowlog.threshold.query.warn: 10s -#index.search.slowlog.threshold.query.info: 5s -#index.search.slowlog.threshold.query.debug: 2s -#index.search.slowlog.threshold.query.trace: 500ms - -#index.search.slowlog.threshold.fetch.warn: 1s -#index.search.slowlog.threshold.fetch.info: 800ms -#index.search.slowlog.threshold.fetch.debug: 500ms -#index.search.slowlog.threshold.fetch.trace: 200ms - -#index.indexing.slowlog.threshold.index.warn: 10s -#index.indexing.slowlog.threshold.index.info: 5s -#index.indexing.slowlog.threshold.index.debug: 2s -#index.indexing.slowlog.threshold.index.trace: 500ms - -################################## GC Logging ################################ - -#monitor.jvm.gc.young.warn: 1000ms -#monitor.jvm.gc.young.info: 700ms -#monitor.jvm.gc.young.debug: 400ms - -#monitor.jvm.gc.old.warn: 10s -#monitor.jvm.gc.old.info: 5s -#monitor.jvm.gc.old.debug: 2s diff --git a/titan-es/src/test/config/networkNodeUsingYaml.yml b/titan-es/src/test/config/networkNodeUsingYaml.yml deleted file mode 100644 index cdda1211cd..0000000000 --- a/titan-es/src/test/config/networkNodeUsingYaml.yml +++ /dev/null @@ -1,378 +0,0 @@ -##################### Elasticsearch Configuration Example ##################### - -# This file contains an overview of various configuration settings, -# targeted at operations staff. Application developers should -# consult the guide at . -# -# The installation procedure is covered at -# . -# -# Elasticsearch comes with reasonable defaults for most settings, -# so you can try it out without bothering with configuration. -# -# Most of the time, these defaults are just fine for running a production -# cluster. If you're fine-tuning your cluster, or wondering about the -# effect of certain configuration option, please _do ask_ on the -# mailing list or IRC channel [http://elasticsearch.org/community]. - -# Any element in the configuration can be replaced with environment variables -# by placing them in ${...} notation. For example: -# -#node.rack: ${RACK_ENV_VAR} - -# For information on supported formats and syntax for the config file, see -# - - -################################### Cluster ################################### - -# Cluster name identifies your cluster for auto-discovery. If you're running -# multiple clusters on the same network, make sure you're using unique names. -# -cluster.name: networkNodeUsingYaml - - -#################################### Node ##################################### - -# Node names are generated dynamically on startup, so you're relieved -# from configuring them manually. You can tie this node to a specific name: -# -#node.name: "Franz Kafka" - -# Every node can be configured to allow or deny being eligible as the master, -# and to allow or deny to store the data. -# -# Allow this node to be eligible as a master node (enabled by default): -# -#node.master: true -# -# Allow this node to store data (enabled by default): -# -#node.data: true - -# You can exploit these settings to design advanced cluster topologies. -# -# 1. You want this node to never become a master node, only to hold data. -# This will be the "workhorse" of your cluster. -# -#node.master: false -#node.data: true -# -# 2. You want this node to only serve as a master: to not store any data and -# to have free resources. This will be the "coordinator" of your cluster. -# -#node.master: true -#node.data: false -# -# 3. You want this node to be neither master nor data node, but -# to act as a "search load balancer" (fetching data from nodes, -# aggregating results, etc.) -# -#node.master: false -#node.data: false - -# Use the Cluster Health API [http://localhost:9200/_cluster/health], the -# Node Info API [http://localhost:9200/_nodes] or GUI tools -# such as , -# , -# and -# to inspect the cluster state. - -# A node can have generic attributes associated with it, which can later be used -# for customized shard allocation filtering, or allocation awareness. An attribute -# is a simple key value pair, similar to node.key: value, here is an example: -# -#node.rack: rack314 - -# By default, multiple nodes are allowed to start from the same installation location -# to disable it, set the following: -#node.max_local_storage_nodes: 1 - - -#################################### Index #################################### - -# You can set a number of options (such as shard/replica options, mapping -# or analyzer definitions, translog settings, ...) for indices globally, -# in this file. -# -# Note, that it makes more sense to configure index settings specifically for -# a certain index, either when creating it or by using the index templates API. -# -# See and -# -# for more information. - -# Set the number of shards (splits) of an index (5 by default): -# -#index.number_of_shards: 5 - -# Set the number of replicas (additional copies) of an index (1 by default): -# -#index.number_of_replicas: 1 - -# Note, that for development on a local machine, with small indices, it usually -# makes sense to "disable" the distributed features: -# -#index.number_of_shards: 1 -#index.number_of_replicas: 0 - -# These settings directly affect the performance of index and search operations -# in your cluster. Assuming you have enough machines to hold shards and -# replicas, the rule of thumb is: -# -# 1. Having more *shards* enhances the _indexing_ performance and allows to -# _distribute_ a big index across machines. -# 2. Having more *replicas* enhances the _search_ performance and improves the -# cluster _availability_. -# -# The "number_of_shards" is a one-time setting for an index. -# -# The "number_of_replicas" can be increased or decreased anytime, -# by using the Index Update Settings API. -# -# Elasticsearch takes care about load balancing, relocating, gathering the -# results from nodes, etc. Experiment with different settings to fine-tune -# your setup. - -# Use the Index Status API () to inspect -# the index status. - - -#################################### Paths #################################### - -# Path to directory containing configuration (this file and logging.yml): -# -path.conf: $MAVEN{project.basedir}/config - -# Path to directory where to store index data allocated for this node. -# -path.data: $MAVEN{project.build.directory}/es-data - -# -# Can optionally include more than one location, causing data to be striped across -# the locations (a la RAID 0) on a file level, favouring locations with most free -# space on creation. For example: -# -#path.data: /path/to/data1,/path/to/data2 - -# Path to temporary files: -# -path.work: $MAVEN{project.build.directory}/es-work - -# Path to log files: -# -path.logs: $MAVEN{project.build.directory}/es-logs - -# Path to where plugins are installed: -# -#path.plugins: /path/to/plugins - - -#################################### Plugin ################################### - -# If a plugin listed here is not installed for current node, the node will not start. -# -#plugin.mandatory: mapper-attachments,lang-groovy - - -################################### Memory #################################### - -# Elasticsearch performs poorly when JVM starts swapping: you should ensure that -# it _never_ swaps. -# -# Set this property to true to lock the memory: -# -#bootstrap.mlockall: true - -# Make sure that the ES_MIN_MEM and ES_MAX_MEM environment variables are set -# to the same value, and that the machine has enough memory to allocate -# for Elasticsearch, leaving enough memory for the operating system itself. -# -# You should also make sure that the Elasticsearch process is allowed to lock -# the memory, eg. by using `ulimit -l unlimited`. - - -############################## Network And HTTP ############################### - -# Elasticsearch, by default, binds itself to the 0.0.0.0 address, and listens -# on port [9200-9300] for HTTP traffic and on port [9300-9400] for node-to-node -# communication. (the range means that if the port is busy, it will automatically -# try the next port). - -# Set the bind address specifically (IPv4 or IPv6): -# -network.bind_host: 127.0.0.1 - -# Set the address other nodes will use to communicate with this node. If not -# set, it is automatically derived. It must point to an actual IP address. -# -network.publish_host: 127.0.0.1 - -# Set both 'bind_host' and 'publish_host': -# -network.host: 127.0.0.1 - -# Set a custom port for the node to node communication (9300 by default): -# -#transport.tcp.port: 9300 - -# Enable compression for all communication between nodes (disabled by default): -# -#transport.tcp.compress: true - -# Set a custom port to listen for HTTP traffic: -# -#http.port: 9200 - -# Set a custom allowed content length: -# -#http.max_content_length: 100mb - -# Disable HTTP completely: -# -#http.enabled: false - - -################################### Gateway ################################### - -# The gateway allows for persisting the cluster state between full cluster -# restarts. Every change to the state (such as adding an index) will be stored -# in the gateway, and when the cluster starts up for the first time, -# it will read its state from the gateway. - -# There are several types of gateway implementations. For more information, see -# . - -# The default gateway type is the "local" gateway (recommended): -# -#gateway.type: local - -# Settings below control how and when to start the initial recovery process on -# a full cluster restart (to reuse as much local data as possible when using shared -# gateway). - -# Allow recovery process after N nodes in a cluster are up: -# -#gateway.recover_after_nodes: 1 - -# Set the timeout to initiate the recovery process, once the N nodes -# from previous setting are up (accepts time value): -# -#gateway.recover_after_time: 5m - -# Set how many nodes are expected in this cluster. Once these N nodes -# are up (and recover_after_nodes is met), begin recovery process immediately -# (without waiting for recover_after_time to expire): -# -#gateway.expected_nodes: 2 - - -############################# Recovery Throttling ############################# - -# These settings allow to control the process of shards allocation between -# nodes during initial recovery, replica allocation, rebalancing, -# or when adding and removing nodes. - -# Set the number of concurrent recoveries happening on a node: -# -# 1. During the initial recovery -# -#cluster.routing.allocation.node_initial_primaries_recoveries: 4 -# -# 2. During adding/removing nodes, rebalancing, etc -# -#cluster.routing.allocation.node_concurrent_recoveries: 2 - -# Set to throttle throughput when recovering (eg. 100mb, by default 20mb): -# -#indices.recovery.max_bytes_per_sec: 20mb - -# Set to limit the number of open concurrent streams when -# recovering a shard from a peer: -# -#indices.recovery.concurrent_streams: 5 - - -################################## Discovery ################################## - -# Discovery infrastructure ensures nodes can be found within a cluster -# and master node is elected. Multicast discovery is the default. - -# Set to ensure a node sees N other master eligible nodes to be considered -# operational within the cluster. Its recommended to set it to a higher value -# than 1 when running more than 2 nodes in the cluster. -# -#discovery.zen.minimum_master_nodes: 1 - -# Set the time to wait for ping responses from other nodes when discovering. -# Set this option to a higher value on a slow or congested network -# to minimize discovery failures: -# -#discovery.zen.ping.timeout: 3s - -# For more information, see -# - -# Unicast discovery allows to explicitly control which nodes will be used -# to discover the cluster. It can be used when multicast is not present, -# or to restrict the cluster communication-wise. -# -# 1. Disable multicast discovery (enabled by default): -# -#discovery.zen.ping.multicast.enabled: false -# -# 2. Configure an initial list of master nodes in the cluster -# to perform discovery when new nodes (master or data) are started: -# -#discovery.zen.ping.unicast.hosts: ["host1", "host2:port"] - -# EC2 discovery allows to use AWS EC2 API in order to perform discovery. -# -# You have to install the cloud-aws plugin for enabling the EC2 discovery. -# -# For more information, see -# -# -# See -# for a step-by-step tutorial. - -# GCE discovery allows to use Google Compute Engine API in order to perform discovery. -# -# You have to install the cloud-gce plugin for enabling the GCE discovery. -# -# For more information, see . - -# Azure discovery allows to use Azure API in order to perform discovery. -# -# You have to install the cloud-azure plugin for enabling the Azure discovery. -# -# For more information, see . - -################################## Slow Log ################################## - -# Shard level query and fetch threshold logging. - -#index.search.slowlog.threshold.query.warn: 10s -#index.search.slowlog.threshold.query.info: 5s -#index.search.slowlog.threshold.query.debug: 2s -#index.search.slowlog.threshold.query.trace: 500ms - -#index.search.slowlog.threshold.fetch.warn: 1s -#index.search.slowlog.threshold.fetch.info: 800ms -#index.search.slowlog.threshold.fetch.debug: 500ms -#index.search.slowlog.threshold.fetch.trace: 200ms - -#index.indexing.slowlog.threshold.index.warn: 10s -#index.indexing.slowlog.threshold.index.info: 5s -#index.indexing.slowlog.threshold.index.debug: 2s -#index.indexing.slowlog.threshold.index.trace: 500ms - -################################## GC Logging ################################ - -#monitor.jvm.gc.young.warn: 1000ms -#monitor.jvm.gc.young.info: 700ms -#monitor.jvm.gc.young.debug: 400ms - -#monitor.jvm.gc.old.warn: 10s -#monitor.jvm.gc.old.info: 5s -#monitor.jvm.gc.old.debug: 2s diff --git a/titan-es/src/test/config/transportClient.yml b/titan-es/src/test/config/transportClient.yml deleted file mode 100644 index 36cc947166..0000000000 --- a/titan-es/src/test/config/transportClient.yml +++ /dev/null @@ -1,378 +0,0 @@ -##################### Elasticsearch Configuration Example ##################### - -# This file contains an overview of various configuration settings, -# targeted at operations staff. Application developers should -# consult the guide at . -# -# The installation procedure is covered at -# . -# -# Elasticsearch comes with reasonable defaults for most settings, -# so you can try it out without bothering with configuration. -# -# Most of the time, these defaults are just fine for running a production -# cluster. If you're fine-tuning your cluster, or wondering about the -# effect of certain configuration option, please _do ask_ on the -# mailing list or IRC channel [http://elasticsearch.org/community]. - -# Any element in the configuration can be replaced with environment variables -# by placing them in ${...} notation. For example: -# -#node.rack: ${RACK_ENV_VAR} - -# For information on supported formats and syntax for the config file, see -# - - -################################### Cluster ################################### - -# Cluster name identifies your cluster for auto-discovery. If you're running -# multiple clusters on the same network, make sure you're using unique names. -# -cluster.name: transportClient - - -#################################### Node ##################################### - -# Node names are generated dynamically on startup, so you're relieved -# from configuring them manually. You can tie this node to a specific name: -# -#node.name: "Franz Kafka" - -# Every node can be configured to allow or deny being eligible as the master, -# and to allow or deny to store the data. -# -# Allow this node to be eligible as a master node (enabled by default): -# -#node.master: true -# -# Allow this node to store data (enabled by default): -# -#node.data: true - -# You can exploit these settings to design advanced cluster topologies. -# -# 1. You want this node to never become a master node, only to hold data. -# This will be the "workhorse" of your cluster. -# -#node.master: false -#node.data: true -# -# 2. You want this node to only serve as a master: to not store any data and -# to have free resources. This will be the "coordinator" of your cluster. -# -#node.master: true -#node.data: false -# -# 3. You want this node to be neither master nor data node, but -# to act as a "search load balancer" (fetching data from nodes, -# aggregating results, etc.) -# -#node.master: false -#node.data: false - -# Use the Cluster Health API [http://localhost:9200/_cluster/health], the -# Node Info API [http://localhost:9200/_nodes] or GUI tools -# such as , -# , -# and -# to inspect the cluster state. - -# A node can have generic attributes associated with it, which can later be used -# for customized shard allocation filtering, or allocation awareness. An attribute -# is a simple key value pair, similar to node.key: value, here is an example: -# -#node.rack: rack314 - -# By default, multiple nodes are allowed to start from the same installation location -# to disable it, set the following: -#node.max_local_storage_nodes: 1 - - -#################################### Index #################################### - -# You can set a number of options (such as shard/replica options, mapping -# or analyzer definitions, translog settings, ...) for indices globally, -# in this file. -# -# Note, that it makes more sense to configure index settings specifically for -# a certain index, either when creating it or by using the index templates API. -# -# See and -# -# for more information. - -# Set the number of shards (splits) of an index (5 by default): -# -#index.number_of_shards: 5 - -# Set the number of replicas (additional copies) of an index (1 by default): -# -#index.number_of_replicas: 1 - -# Note, that for development on a local machine, with small indices, it usually -# makes sense to "disable" the distributed features: -# -#index.number_of_shards: 1 -#index.number_of_replicas: 0 - -# These settings directly affect the performance of index and search operations -# in your cluster. Assuming you have enough machines to hold shards and -# replicas, the rule of thumb is: -# -# 1. Having more *shards* enhances the _indexing_ performance and allows to -# _distribute_ a big index across machines. -# 2. Having more *replicas* enhances the _search_ performance and improves the -# cluster _availability_. -# -# The "number_of_shards" is a one-time setting for an index. -# -# The "number_of_replicas" can be increased or decreased anytime, -# by using the Index Update Settings API. -# -# Elasticsearch takes care about load balancing, relocating, gathering the -# results from nodes, etc. Experiment with different settings to fine-tune -# your setup. - -# Use the Index Status API () to inspect -# the index status. - - -#################################### Paths #################################### - -# Path to directory containing configuration (this file and logging.yml): -# -path.conf: $MAVEN{project.basedir}/config - -# Path to directory where to store index data allocated for this node. -# -path.data: $MAVEN{project.build.directory}/es-data - -# -# Can optionally include more than one location, causing data to be striped across -# the locations (a la RAID 0) on a file level, favouring locations with most free -# space on creation. For example: -# -#path.data: /path/to/data1,/path/to/data2 - -# Path to temporary files: -# -path.work: $MAVEN{project.build.directory}/es-work - -# Path to log files: -# -path.logs: $MAVEN{project.build.directory}/es-logs - -# Path to where plugins are installed: -# -#path.plugins: /path/to/plugins - - -#################################### Plugin ################################### - -# If a plugin listed here is not installed for current node, the node will not start. -# -#plugin.mandatory: mapper-attachments,lang-groovy - - -################################### Memory #################################### - -# Elasticsearch performs poorly when JVM starts swapping: you should ensure that -# it _never_ swaps. -# -# Set this property to true to lock the memory: -# -#bootstrap.mlockall: true - -# Make sure that the ES_MIN_MEM and ES_MAX_MEM environment variables are set -# to the same value, and that the machine has enough memory to allocate -# for Elasticsearch, leaving enough memory for the operating system itself. -# -# You should also make sure that the Elasticsearch process is allowed to lock -# the memory, eg. by using `ulimit -l unlimited`. - - -############################## Network And HTTP ############################### - -# Elasticsearch, by default, binds itself to the 0.0.0.0 address, and listens -# on port [9200-9300] for HTTP traffic and on port [9300-9400] for node-to-node -# communication. (the range means that if the port is busy, it will automatically -# try the next port). - -# Set the bind address specifically (IPv4 or IPv6): -# -network.bind_host: 127.0.0.1 - -# Set the address other nodes will use to communicate with this node. If not -# set, it is automatically derived. It must point to an actual IP address. -# -network.publish_host: 127.0.0.1 - -# Set both 'bind_host' and 'publish_host': -# -network.host: 127.0.0.1 - -# Set a custom port for the node to node communication (9300 by default): -# -#transport.tcp.port: 9300 - -# Enable compression for all communication between nodes (disabled by default): -# -#transport.tcp.compress: true - -# Set a custom port to listen for HTTP traffic: -# -#http.port: 9200 - -# Set a custom allowed content length: -# -#http.max_content_length: 100mb - -# Disable HTTP completely: -# -#http.enabled: false - - -################################### Gateway ################################### - -# The gateway allows for persisting the cluster state between full cluster -# restarts. Every change to the state (such as adding an index) will be stored -# in the gateway, and when the cluster starts up for the first time, -# it will read its state from the gateway. - -# There are several types of gateway implementations. For more information, see -# . - -# The default gateway type is the "local" gateway (recommended): -# -#gateway.type: local - -# Settings below control how and when to start the initial recovery process on -# a full cluster restart (to reuse as much local data as possible when using shared -# gateway). - -# Allow recovery process after N nodes in a cluster are up: -# -#gateway.recover_after_nodes: 1 - -# Set the timeout to initiate the recovery process, once the N nodes -# from previous setting are up (accepts time value): -# -#gateway.recover_after_time: 5m - -# Set how many nodes are expected in this cluster. Once these N nodes -# are up (and recover_after_nodes is met), begin recovery process immediately -# (without waiting for recover_after_time to expire): -# -#gateway.expected_nodes: 2 - - -############################# Recovery Throttling ############################# - -# These settings allow to control the process of shards allocation between -# nodes during initial recovery, replica allocation, rebalancing, -# or when adding and removing nodes. - -# Set the number of concurrent recoveries happening on a node: -# -# 1. During the initial recovery -# -#cluster.routing.allocation.node_initial_primaries_recoveries: 4 -# -# 2. During adding/removing nodes, rebalancing, etc -# -#cluster.routing.allocation.node_concurrent_recoveries: 2 - -# Set to throttle throughput when recovering (eg. 100mb, by default 20mb): -# -#indices.recovery.max_bytes_per_sec: 20mb - -# Set to limit the number of open concurrent streams when -# recovering a shard from a peer: -# -#indices.recovery.concurrent_streams: 5 - - -################################## Discovery ################################## - -# Discovery infrastructure ensures nodes can be found within a cluster -# and master node is elected. Multicast discovery is the default. - -# Set to ensure a node sees N other master eligible nodes to be considered -# operational within the cluster. Its recommended to set it to a higher value -# than 1 when running more than 2 nodes in the cluster. -# -#discovery.zen.minimum_master_nodes: 1 - -# Set the time to wait for ping responses from other nodes when discovering. -# Set this option to a higher value on a slow or congested network -# to minimize discovery failures: -# -#discovery.zen.ping.timeout: 3s - -# For more information, see -# - -# Unicast discovery allows to explicitly control which nodes will be used -# to discover the cluster. It can be used when multicast is not present, -# or to restrict the cluster communication-wise. -# -# 1. Disable multicast discovery (enabled by default): -# -#discovery.zen.ping.multicast.enabled: false -# -# 2. Configure an initial list of master nodes in the cluster -# to perform discovery when new nodes (master or data) are started: -# -#discovery.zen.ping.unicast.hosts: ["host1", "host2:port"] - -# EC2 discovery allows to use AWS EC2 API in order to perform discovery. -# -# You have to install the cloud-aws plugin for enabling the EC2 discovery. -# -# For more information, see -# -# -# See -# for a step-by-step tutorial. - -# GCE discovery allows to use Google Compute Engine API in order to perform discovery. -# -# You have to install the cloud-gce plugin for enabling the GCE discovery. -# -# For more information, see . - -# Azure discovery allows to use Azure API in order to perform discovery. -# -# You have to install the cloud-azure plugin for enabling the Azure discovery. -# -# For more information, see . - -################################## Slow Log ################################## - -# Shard level query and fetch threshold logging. - -#index.search.slowlog.threshold.query.warn: 10s -#index.search.slowlog.threshold.query.info: 5s -#index.search.slowlog.threshold.query.debug: 2s -#index.search.slowlog.threshold.query.trace: 500ms - -#index.search.slowlog.threshold.fetch.warn: 1s -#index.search.slowlog.threshold.fetch.info: 800ms -#index.search.slowlog.threshold.fetch.debug: 500ms -#index.search.slowlog.threshold.fetch.trace: 200ms - -#index.indexing.slowlog.threshold.index.warn: 10s -#index.indexing.slowlog.threshold.index.info: 5s -#index.indexing.slowlog.threshold.index.debug: 2s -#index.indexing.slowlog.threshold.index.trace: 500ms - -################################## GC Logging ################################ - -#monitor.jvm.gc.young.warn: 1000ms -#monitor.jvm.gc.young.info: 700ms -#monitor.jvm.gc.young.debug: 400ms - -#monitor.jvm.gc.old.warn: 10s -#monitor.jvm.gc.old.info: 5s -#monitor.jvm.gc.old.debug: 2s diff --git a/titan-es/src/test/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchConfigTest.java b/titan-es/src/test/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchConfigTest.java index 31feaafc7d..a521bc2355 100644 --- a/titan-es/src/test/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchConfigTest.java +++ b/titan-es/src/test/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchConfigTest.java @@ -1,7 +1,6 @@ package com.thinkaurelius.titan.diskstorage.es; import com.google.common.base.Joiner; - import com.thinkaurelius.titan.core.TitanFactory; import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.attribute.Text; @@ -13,15 +12,15 @@ import com.thinkaurelius.titan.diskstorage.configuration.backend.CommonsConfiguration; import com.thinkaurelius.titan.diskstorage.indexing.*; import com.thinkaurelius.titan.diskstorage.util.StandardBaseTransactionConfig; - import com.thinkaurelius.titan.diskstorage.util.time.TimestampProviders; import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration; import com.thinkaurelius.titan.graphdb.query.condition.PredicateCondition; import com.thinkaurelius.titan.util.system.IOUtils; + import org.apache.commons.configuration.BaseConfiguration; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; -import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeBuilder; import org.junit.Assert; @@ -35,7 +34,6 @@ import static com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration.INDEX_CONF_FILE; import static com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration.INDEX_DIRECTORY; import static com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration.INDEX_HOSTS; - import static org.junit.Assert.*; /** @@ -77,7 +75,7 @@ public void testTitanFactoryBuilder() @Test public void testTransportClient() throws BackendException, InterruptedException { - ElasticsearchRunner esr = new ElasticsearchRunner(".", "transportClient.yml"); + ElasticsearchRunner esr = new ElasticsearchRunner("."); esr.start(); ModifiableConfiguration config = GraphDatabaseConfiguration.buildGraphConfiguration(); config.set(INTERFACE, ElasticSearchSetup.TRANSPORT_CLIENT.toString(), INDEX_NAME); @@ -174,12 +172,11 @@ public void testLocalNodeUsingYaml() throws BackendException, InterruptedExcepti @Test public void testNetworkNodeUsingExt() throws BackendException, InterruptedException { - ElasticsearchRunner esr = new ElasticsearchRunner(".", "networkNodeUsingExt.yml"); + ElasticsearchRunner esr = new ElasticsearchRunner("."); esr.start(); CommonsConfiguration cc = new CommonsConfiguration(new BaseConfiguration()); cc.set("index." + INDEX_NAME + ".elasticsearch.ext.node.data", "false"); cc.set("index." + INDEX_NAME + ".elasticsearch.ext.node.client", "true"); - cc.set("index." + INDEX_NAME + ".elasticsearch.ext.cluster.name", "networkNodeUsingExt"); cc.set("index." + INDEX_NAME + ".elasticsearch.ext.discovery.zen.ping.multicast.enabled", "false"); cc.set("index." + INDEX_NAME + ".elasticsearch.ext.discovery.zen.ping.unicast.hosts", "localhost,127.0.0.1:9300"); ModifiableConfiguration config = @@ -211,7 +208,7 @@ public void testNetworkNodeUsingExt() throws BackendException, InterruptedExcept @Test public void testNetworkNodeUsingYaml() throws BackendException, InterruptedException { - ElasticsearchRunner esr = new ElasticsearchRunner(".", "networkNodeUsingYaml.yml"); + ElasticsearchRunner esr = new ElasticsearchRunner("."); esr.start(); ModifiableConfiguration config = GraphDatabaseConfiguration.buildGraphConfiguration(); config.set(INTERFACE, ElasticSearchSetup.NODE.toString(), INDEX_NAME); @@ -242,27 +239,32 @@ public void testNetworkNodeUsingYaml() throws BackendException, InterruptedExcep @Test public void testIndexCreationOptions() throws InterruptedException, BackendException { - final int shards = 77; - ElasticsearchRunner esr = new ElasticsearchRunner(".", "indexCreationOptions.yml"); + String baseDir = Joiner.on(File.separator).join("target", "es", "jvmlocal_opts"); + + assertFalse(new File(baseDir + File.separator + "data").exists()); + + final int shards = 7; + + ElasticsearchRunner esr = new ElasticsearchRunner("."); esr.start(); CommonsConfiguration cc = new CommonsConfiguration(new BaseConfiguration()); cc.set("index." + INDEX_NAME + ".elasticsearch.create.ext.number_of_shards", String.valueOf(shards)); - cc.set("index." + INDEX_NAME + ".elasticsearch.ext.cluster.name", "indexCreationOptions"); ModifiableConfiguration config = new ModifiableConfiguration(GraphDatabaseConfiguration.ROOT_NS, cc, BasicConfiguration.Restriction.NONE); config.set(INTERFACE, ElasticSearchSetup.NODE.toString(), INDEX_NAME); + config.set(GraphDatabaseConfiguration.INDEX_DIRECTORY, baseDir, INDEX_NAME); Configuration indexConfig = config.restrictTo(INDEX_NAME); IndexProvider idx = new ElasticSearchIndex(indexConfig); simpleWriteAndQuery(idx); - ImmutableSettings.Builder settingsBuilder = ImmutableSettings.settingsBuilder(); + Settings.Builder settingsBuilder = Settings.settingsBuilder(); settingsBuilder.put("discovery.zen.ping.multicast.enabled", "false"); settingsBuilder.put("discovery.zen.ping.unicast.hosts", "localhost,127.0.0.1:9300"); - settingsBuilder.put("cluster.name", "indexCreationOptions"); + settingsBuilder.put("path.home", baseDir); NodeBuilder nodeBuilder = NodeBuilder.nodeBuilder().settings(settingsBuilder.build()); nodeBuilder.client(true).data(false).local(false); Node n = nodeBuilder.build().start(); @@ -271,7 +273,7 @@ public void testIndexCreationOptions() throws InterruptedException, BackendExcep assertEquals(String.valueOf(shards), response.getSetting("titan", "index.number_of_shards")); idx.close(); - n.stop(); + n.close(); esr.stop(); } diff --git a/titan-es/src/test/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchIndexTest.java b/titan-es/src/test/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchIndexTest.java index d62df4e9fa..90f2786fdd 100644 --- a/titan-es/src/test/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchIndexTest.java +++ b/titan-es/src/test/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchIndexTest.java @@ -84,6 +84,16 @@ public void testSupport() { assertTrue(index.supports(of(UUID.class, Cardinality.SINGLE), Cmp.EQUAL)); assertTrue(index.supports(of(UUID.class, Cardinality.SINGLE), Cmp.NOT_EQUAL)); + + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE))); + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE), Geo.WITHIN)); + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE), Geo.INTERSECT)); + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE), Geo.DISJOINT)); + assertFalse(index.supports(of(Geoshape.class, Cardinality.SINGLE), Geo.CONTAINS)); + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE, new Parameter("mapping",Mapping.PREFIX_TREE)), Geo.WITHIN)); + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE, new Parameter("mapping",Mapping.PREFIX_TREE)), Geo.INTERSECT)); + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE, new Parameter("mapping",Mapping.PREFIX_TREE)), Geo.CONTAINS)); + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE, new Parameter("mapping",Mapping.PREFIX_TREE)), Geo.DISJOINT)); } @Test diff --git a/titan-es/src/test/java/com/thinkaurelius/titan/diskstorage/es/ElasticsearchRunner.java b/titan-es/src/test/java/com/thinkaurelius/titan/diskstorage/es/ElasticsearchRunner.java index f2b73acf9a..6df8a5e4bf 100644 --- a/titan-es/src/test/java/com/thinkaurelius/titan/diskstorage/es/ElasticsearchRunner.java +++ b/titan-es/src/test/java/com/thinkaurelius/titan/diskstorage/es/ElasticsearchRunner.java @@ -20,7 +20,6 @@ public class ElasticsearchRunner extends DaemonRunner { LoggerFactory.getLogger(ElasticsearchRunner.class); public static final String ES_PID_FILE = "/tmp/titan-test-es.pid"; - private String configFile = "elasticsearch.yml"; public ElasticsearchRunner() { this.homedir = "."; @@ -30,11 +29,6 @@ public ElasticsearchRunner(String esHome) { this.homedir = esHome; } - public ElasticsearchRunner(String esHome, String configFile) { - this(esHome); - this.configFile = configFile; - } - @Override protected String getDaemonShortName() { @@ -77,7 +71,7 @@ protected ElasticsearchStatus startImpl() throws IOException { FileUtils.deleteDirectory(logs); } - runCommand(homedir + File.separator + "bin/elasticsearch", "-d", "-p", ES_PID_FILE, "-Des.config=" + homedir + File.separator + "config" + File.separator + configFile); + runCommand(homedir + File.separator + "bin/elasticsearch", "-d", "-p", ES_PID_FILE); try { watchLog(" started", 60L, TimeUnit.SECONDS); } catch (InterruptedException e) { diff --git a/titan-es/src/test/resources/es_cfg_bogus_nodeclient.yml b/titan-es/src/test/resources/es_cfg_bogus_nodeclient.yml index be59d45860..29b13bca5e 100644 --- a/titan-es/src/test/resources/es_cfg_bogus_nodeclient.yml +++ b/titan-es/src/test/resources/es_cfg_bogus_nodeclient.yml @@ -2,4 +2,3 @@ node.data: false node.client: true discovery.zen.ping.multicast.enabled: false discovery.zen.ping.unicast.hosts: [ "10.11.12.13" ] -cluster.name: networkNodeUsingYaml \ No newline at end of file diff --git a/titan-es/src/test/resources/es_cfg_nodeclient.yml b/titan-es/src/test/resources/es_cfg_nodeclient.yml index e0f4f02475..22c6e28c20 100644 --- a/titan-es/src/test/resources/es_cfg_nodeclient.yml +++ b/titan-es/src/test/resources/es_cfg_nodeclient.yml @@ -2,4 +2,3 @@ node.data: false node.client: true discovery.zen.ping.multicast.enabled: false discovery.zen.ping.unicast.hosts: [ "localhost", "127.0.0.1:9300" ] -cluster.name: networkNodeUsingYaml \ No newline at end of file diff --git a/titan-es/src/test/resources/es_jvmlocal.yml b/titan-es/src/test/resources/es_jvmlocal.yml index 995e2aa592..9e6f4260d3 100644 --- a/titan-es/src/test/resources/es_jvmlocal.yml +++ b/titan-es/src/test/resources/es_jvmlocal.yml @@ -4,4 +4,3 @@ node.local: true path.data: ${project.build.directory}/es/jvmlocal_yml/data path.work: ${project.build.directory}/es/jvmlocal_yml/work path.logs: ${project.build.directory}/es/jvmlocal_yml/logs -cluster.name: jvmlocal \ No newline at end of file diff --git a/titan-hadoop-parent/pom.xml b/titan-hadoop-parent/pom.xml index b97a17b544..0329473961 100644 --- a/titan-hadoop-parent/pom.xml +++ b/titan-hadoop-parent/pom.xml @@ -77,6 +77,20 @@ net.jpountz.lz4 lz4 + + com.fasterxml.jackson.module + jackson-module-scala_2.10 + + + + + com.fasterxml.jackson.module + jackson-module-scala_2.10 + + + org.scala-lang + scala-library + diff --git a/titan-hadoop-parent/titan-hadoop-1/pom.xml b/titan-hadoop-parent/titan-hadoop-1/pom.xml index d114ddbc38..25e0b45890 100644 --- a/titan-hadoop-parent/titan-hadoop-1/pom.xml +++ b/titan-hadoop-parent/titan-hadoop-1/pom.xml @@ -12,6 +12,8 @@ ${basedir}/../.. + + true diff --git a/titan-hadoop-parent/titan-hadoop-core/src/main/java/com/thinkaurelius/titan/hadoop/serialize/TitanKryoRegistrator.java b/titan-hadoop-parent/titan-hadoop-core/src/main/java/com/thinkaurelius/titan/hadoop/serialize/TitanKryoRegistrator.java new file mode 100644 index 0000000000..07a846cdce --- /dev/null +++ b/titan-hadoop-parent/titan-hadoop-core/src/main/java/com/thinkaurelius/titan/hadoop/serialize/TitanKryoRegistrator.java @@ -0,0 +1,58 @@ +package com.thinkaurelius.titan.hadoop.serialize; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; + +import org.apache.commons.io.output.ByteArrayOutputStream; +import org.apache.spark.serializer.KryoRegistrator; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.Serializer; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; +import com.thinkaurelius.titan.core.attribute.Geoshape; +import com.thinkaurelius.titan.core.attribute.Geoshape.GeoshapeBinarySerializer; + +/** + * Register Titan classes requiring custom Kryo serialization for Spark. + * + */ +public class TitanKryoRegistrator implements KryoRegistrator { + + @Override + public void registerClasses(Kryo kryo) { + kryo.register(Geoshape.class, new GeoShapeKryoSerializer()); + } + + /** + * Geoshape serializer for Kryo. + */ + public static class GeoShapeKryoSerializer extends Serializer { + @Override + public void write(Kryo kryo, Output output, Geoshape geoshape) { + try { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + GeoshapeBinarySerializer.write(outputStream, geoshape); + byte[] bytes = outputStream.toByteArray(); + output.write(bytes.length); + output.write(bytes); + } catch (IOException e) { + throw new RuntimeException("I/O exception writing geoshape"); + } + } + + @Override + public Geoshape read(Kryo kryo, Input input, Class aClass) { + int length = input.read(); + assert length>0; + InputStream inputStream = new ByteArrayInputStream(input.readBytes(length)); + try { + return GeoshapeBinarySerializer.read(inputStream); + } catch (IOException e) { + throw new RuntimeException("I/O exception reding geoshape"); + } + } + } + +} diff --git a/titan-hadoop-parent/titan-hadoop-core/src/test/java/com/thinkaurelius/titan/hadoop/CassandraInputFormatIT.java b/titan-hadoop-parent/titan-hadoop-core/src/test/java/com/thinkaurelius/titan/hadoop/CassandraInputFormatIT.java index 727a98d5ef..e50b9ee8f8 100644 --- a/titan-hadoop-parent/titan-hadoop-core/src/test/java/com/thinkaurelius/titan/hadoop/CassandraInputFormatIT.java +++ b/titan-hadoop-parent/titan-hadoop-core/src/test/java/com/thinkaurelius/titan/hadoop/CassandraInputFormatIT.java @@ -7,6 +7,8 @@ import com.thinkaurelius.titan.diskstorage.configuration.WriteConfiguration; import com.thinkaurelius.titan.example.GraphOfTheGodsFactory; import com.thinkaurelius.titan.graphdb.TitanGraphBaseTest; + +import org.apache.commons.io.FileUtils; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer; import org.apache.tinkerpop.gremlin.structure.Direction; @@ -14,8 +16,11 @@ import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.util.GraphFactory; +import org.junit.Before; import org.junit.Test; +import java.io.File; +import java.io.IOException; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -31,6 +36,10 @@ public class CassandraInputFormatIT extends TitanGraphBaseTest { + @Before + public void setup() throws IOException { + FileUtils.deleteDirectory(new File("output")); + } @Test public void testReadGraphOfTheGods() { diff --git a/titan-hadoop-parent/titan-hadoop-core/src/test/resources/cassandra-read.properties b/titan-hadoop-parent/titan-hadoop-core/src/test/resources/cassandra-read.properties index 4a4a6ddf7c..5f6b4cfaec 100644 --- a/titan-hadoop-parent/titan-hadoop-core/src/test/resources/cassandra-read.properties +++ b/titan-hadoop-parent/titan-hadoop-core/src/test/resources/cassandra-read.properties @@ -28,5 +28,6 @@ giraph.maxMessagesInMemory=100000 spark.master=local[4] spark.executor.memory=1g spark.serializer=org.apache.spark.serializer.KryoSerializer +spark.kryo.registrator=com.thinkaurelius.titan.hadoop.serialize.TitanKryoRegistrator cassandra.input.partitioner.class=org.apache.cassandra.dht.Murmur3Partitioner cassandra.input.widerows=true diff --git a/titan-hbase-parent/titan-hbase-core/src/test/java/com/thinkaurelius/titan/HBaseStatus.java b/titan-hbase-parent/titan-hbase-core/src/test/java/com/thinkaurelius/titan/HBaseStatus.java index 265fa68585..ed1426c828 100644 --- a/titan-hbase-parent/titan-hbase-core/src/test/java/com/thinkaurelius/titan/HBaseStatus.java +++ b/titan-hbase-parent/titan-hbase-core/src/test/java/com/thinkaurelius/titan/HBaseStatus.java @@ -6,10 +6,10 @@ import java.io.FileReader; import java.io.IOException; -import org.elasticsearch.common.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Preconditions; import com.thinkaurelius.titan.util.system.IOUtils; public class HBaseStatus { diff --git a/titan-lucene/src/main/java/com/thinkaurelius/titan/diskstorage/lucene/LuceneIndex.java b/titan-lucene/src/main/java/com/thinkaurelius/titan/diskstorage/lucene/LuceneIndex.java index 893d95af38..3dd2b691b6 100644 --- a/titan-lucene/src/main/java/com/thinkaurelius/titan/diskstorage/lucene/LuceneIndex.java +++ b/titan-lucene/src/main/java/com/thinkaurelius/titan/diskstorage/lucene/LuceneIndex.java @@ -18,7 +18,9 @@ import com.thinkaurelius.titan.graphdb.database.serialize.AttributeUtil; import com.thinkaurelius.titan.graphdb.query.TitanPredicate; import com.thinkaurelius.titan.graphdb.query.condition.*; +import com.thinkaurelius.titan.graphdb.types.ParameterType; import com.thinkaurelius.titan.util.system.IOUtils; + import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -33,23 +35,29 @@ import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.*; import org.apache.lucene.spatial.SpatialStrategy; +import org.apache.lucene.spatial.prefix.PrefixTreeStrategy; +import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; +import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; +import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; import org.apache.lucene.spatial.query.SpatialArgs; import org.apache.lucene.spatial.query.SpatialOperation; import org.apache.lucene.spatial.vector.PointVectorStrategy; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.Version; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.nio.file.Path; import java.time.Instant; import java.util.*; +import java.util.AbstractMap.SimpleEntry; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * @author Matthias Broecheler (me@matthiasb.com) @@ -63,10 +71,20 @@ public class LuceneIndex implements IndexProvider { private static final String GEOID = "_____geo"; private static final int MAX_STRING_FIELD_LEN = 256; - private static final Version LUCENE_VERSION = Version.LUCENE_4_10_4; + private static final Version LUCENE_VERSION = Version.LUCENE_5_5_2; private static final IndexFeatures LUCENE_FEATURES = new IndexFeatures.Builder().supportedStringMappings(Mapping.TEXT, Mapping.STRING).supportsCardinality(Cardinality.SINGLE).supportsNanoseconds().build(); - private static final int GEO_MAX_LEVELS = 11; + /** + * Default tree levels used when creating the prefix tree. + */ + public static final int DEFAULT_GEO_MAX_LEVELS = 20; + + /** + * Default measure of shape precision used when creating the prefix tree. + */ + public static final double DEFAULT_GEO_DIST_ERROR_PCT = 0.025; + + private static Map SPATIAL_PREDICATES = spatialPredicates(); private final Analyzer analyzer = new StandardAnalyzer(); @@ -74,7 +92,7 @@ public class LuceneIndex implements IndexProvider { private final ReentrantLock writerLock = new ReentrantLock(); private Map spatial = new ConcurrentHashMap(12); - private SpatialContext ctx = SpatialContext.GEO; + private SpatialContext ctx = Geoshape.CTX; private final String basePath; @@ -97,7 +115,7 @@ private Directory getStoreDirectory(String store) throws BackendException { if (!path.exists() || !path.isDirectory() || !path.canWrite()) throw new PermanentBackendException("Cannot access or write to directory: " + dir); log.debug("Opening store directory [{}]", path); - return FSDirectory.open(path); + return FSDirectory.open(path.toPath()); } catch (IOException e) { throw new PermanentBackendException("Could not open directory: " + dir, e); } @@ -107,7 +125,7 @@ private IndexWriter getWriter(String store) throws BackendException { Preconditions.checkArgument(writerLock.isHeldByCurrentThread()); IndexWriter writer = writers.get(store); if (writer == null) { - IndexWriterConfig iwc = new IndexWriterConfig(LUCENE_VERSION, analyzer); + IndexWriterConfig iwc = new IndexWriterConfig(analyzer); iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); try { writer = new IndexWriter(getStoreDirectory(store), iwc); @@ -119,14 +137,23 @@ private IndexWriter getWriter(String store) throws BackendException { return writer; } - private SpatialStrategy getSpatialStrategy(String key) { + private SpatialStrategy getSpatialStrategy(String key, KeyInformation ki) { SpatialStrategy strategy = spatial.get(key); + Mapping mapping = Mapping.getMapping(ki); + int maxLevels = (int) ParameterType.INDEX_GEO_MAX_LEVELS.findParameter(ki.getParameters(), DEFAULT_GEO_MAX_LEVELS); + double distErrorPct = (double) ParameterType.INDEX_GEO_DIST_ERROR_PCT.findParameter(ki.getParameters(), DEFAULT_GEO_DIST_ERROR_PCT); if (strategy == null) { synchronized (spatial) { if (!spatial.containsKey(key)) { // SpatialPrefixTree grid = new GeohashPrefixTree(ctx, GEO_MAX_LEVELS); // strategy = new RecursivePrefixTreeStrategy(grid, key); - strategy = new PointVectorStrategy(ctx, key); + if (mapping == Mapping.DEFAULT) { + strategy = new PointVectorStrategy(ctx, key); + } else { + SpatialPrefixTree grid = new QuadPrefixTree(ctx, maxLevels); + strategy = new RecursivePrefixTreeStrategy(grid, key); + ((PrefixTreeStrategy) strategy).setDistErrPct(distErrorPct); + } spatial.put(key, strategy); } else return spatial.get(key); } @@ -134,12 +161,23 @@ private SpatialStrategy getSpatialStrategy(String key) { return strategy; } + private static Map spatialPredicates() { + return Collections.unmodifiableMap(Stream.of( + new SimpleEntry<>(Geo.WITHIN, SpatialOperation.IsWithin), + new SimpleEntry<>(Geo.CONTAINS, SpatialOperation.Contains), + new SimpleEntry<>(Geo.INTERSECT, SpatialOperation.Intersects), + new SimpleEntry<>(Geo.DISJOINT, SpatialOperation.IsDisjointTo)) + .collect(Collectors.toMap((e) -> e.getKey(), (e) -> e.getValue()))); + } + @Override public void register(String store, String key, KeyInformation information, BaseTransaction tx) throws BackendException { Class dataType = information.getDataType(); Mapping map = Mapping.getMapping(information); - Preconditions.checkArgument(map == Mapping.DEFAULT || AttributeUtil.isString(dataType), - "Specified illegal mapping [%s] for data type [%s]", map, dataType); } + Preconditions.checkArgument(map == Mapping.DEFAULT || AttributeUtil.isString(dataType) || + (map == Mapping.PREFIX_TREE && AttributeUtil.isGeo(dataType)), + "Specified illegal mapping [%s] for data type [%s]", map, dataType); + } @Override public void mutate(Map> mutations, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException { @@ -266,7 +304,7 @@ private Pair> retrieveOrCreate(String docID, IndexS for (IndexableField field : doc.getFields()) { if (field.stringValue().startsWith(GEOID)) { try { - geofields.put(field.name(), ctx.readShapeFromWkt(field.stringValue().substring(GEOID.length()))); + geofields.put(field.name(), Geoshape.fromWkt(field.stringValue().substring(GEOID.length())).getShape()); } catch (java.text.ParseException e) { throw new IllegalArgumentException("Geoshape was unparsable"); } @@ -294,12 +332,16 @@ private void addToDocument(String store, if (e.value instanceof Number) { Field field; + Field sortField; if (AttributeUtil.isWholeNumber((Number) e.value)) { field = new LongField(e.field, ((Number) e.value).longValue(), Field.Store.YES); + sortField = new NumericDocValuesField(e.field, ((Number) e.value).longValue()); } else { //double or float field = new DoubleField(e.field, ((Number) e.value).doubleValue(), Field.Store.YES); + sortField = new DoubleDocValuesField(e.field, ((Number) e.value).doubleValue()); } doc.add(field); + doc.add(sortField); } else if (AttributeUtil.isString(e.value)) { String str = (String) e.value; Mapping mapping = Mapping.getMapping(store, e.field, informations); @@ -316,10 +358,9 @@ private void addToDocument(String store, } doc.add(field); } else if (e.value instanceof Geoshape) { - Shape shape = ((Geoshape) e.value).convert2Spatial4j(); + Shape shape = ((Geoshape) e.value).getShape(); geofields.put(e.field, shape); - doc.add(new StoredField(e.field, GEOID + toWkt(shape))); - + doc.add(new StoredField(e.field, GEOID + e.value.toString())); } else if (e.value instanceof Date) { doc.add(new LongField(e.field, (((Date) e.value).getTime()), Field.Store.YES)); } else if (e.value instanceof Instant) { @@ -339,17 +380,14 @@ private void addToDocument(String store, if (log.isTraceEnabled()) log.trace("Updating geo-indexes for key {}", geo.getKey()); - for (IndexableField f : getSpatialStrategy(geo.getKey()).createIndexableFields(geo.getValue())) + KeyInformation ki = informations.get(store, geo.getKey()); + SpatialStrategy spatialStrategy = getSpatialStrategy(geo.getKey(), ki); + for (IndexableField f : spatialStrategy.createIndexableFields(geo.getValue())) { doc.add(f); - } - } - - private String toWkt(Shape shape) { - if(shape instanceof Point) { - return "POINT(" + ((Point) shape).getX() + " " + ((Point) shape).getY() + ")"; - } - else { - throw new IllegalArgumentException("Only points are supported"); + if (spatialStrategy instanceof PointVectorStrategy) { + doc.add(new DoubleDocValuesField(f.name(), f.numericValue().doubleValue())); + } + } } } @@ -488,10 +526,11 @@ private final SearchParams convertQuery(Condition condition, KeyInformation.S } else throw new IllegalArgumentException("Relation is not supported for string value: " + titanPredicate); } else if (value instanceof Geoshape) { - Preconditions.checkArgument(titanPredicate == Geo.WITHIN, "Relation is not supported for geo value: " + titanPredicate); - Shape shape = ((Geoshape) value).convert2Spatial4j(); - SpatialArgs args = new SpatialArgs(SpatialOperation.IsWithin, shape); - params.addFilter(getSpatialStrategy(key).makeFilter(args)); + Preconditions.checkArgument(titanPredicate instanceof Geo, "Relation not supported on geo types: " + titanPredicate); + Shape shape = ((Geoshape) value).getShape(); + SpatialOperation spatialOp = SPATIAL_PREDICATES.get((Geo) titanPredicate); + SpatialArgs args = new SpatialArgs(spatialOp, shape); + params.addQuery(getSpatialStrategy(key, informations.get(key)).makeQuery(args)); } else if (value instanceof Date) { Preconditions.checkArgument(titanPredicate instanceof Cmp, "Relation not supported on date types: " + titanPredicate); params.addFilter(numericFilter(key, (Cmp) titanPredicate, ((Date) value).getTime())); @@ -588,12 +627,13 @@ public boolean supports(KeyInformation information, TitanPredicate titanPredicat if (information.getCardinality()!= Cardinality.SINGLE) return false; Class dataType = information.getDataType(); Mapping mapping = Mapping.getMapping(information); - if (mapping!=Mapping.DEFAULT && !AttributeUtil.isString(dataType)) return false; + if (mapping!=Mapping.DEFAULT && !AttributeUtil.isString(dataType) && + !(mapping==Mapping.PREFIX_TREE && AttributeUtil.isGeo(dataType))) return false; if (Number.class.isAssignableFrom(dataType)) { if (titanPredicate instanceof Cmp) return true; } else if (dataType == Geoshape.class) { - return titanPredicate == Geo.WITHIN; + return titanPredicate == Geo.INTERSECT || titanPredicate == Geo.WITHIN || titanPredicate == Geo.CONTAINS; } else if (AttributeUtil.isString(dataType)) { switch(mapping) { case DEFAULT: @@ -617,10 +657,12 @@ public boolean supports(KeyInformation information) { if (information.getCardinality()!= Cardinality.SINGLE) return false; Class dataType = information.getDataType(); Mapping mapping = Mapping.getMapping(information); - if (Number.class.isAssignableFrom(dataType) || dataType == Geoshape.class || dataType == Date.class || dataType == Instant.class || dataType == Boolean.class || dataType == UUID.class) { + if (Number.class.isAssignableFrom(dataType) || dataType == Date.class || dataType == Instant.class || dataType == Boolean.class || dataType == UUID.class) { if (mapping==Mapping.DEFAULT) return true; } else if (AttributeUtil.isString(dataType)) { if (mapping==Mapping.DEFAULT || mapping==Mapping.STRING || mapping==Mapping.TEXT) return true; + } else if (AttributeUtil.isGeo(dataType)) { + if (mapping==Mapping.DEFAULT || mapping==Mapping.PREFIX_TREE) return true; } return false; } diff --git a/titan-lucene/src/test/java/com/thinkaurelius/titan/diskstorage/lucene/LuceneExample.java b/titan-lucene/src/test/java/com/thinkaurelius/titan/diskstorage/lucene/LuceneExample.java index f4cb84a02d..75edc1ae3e 100644 --- a/titan-lucene/src/test/java/com/thinkaurelius/titan/diskstorage/lucene/LuceneExample.java +++ b/titan-lucene/src/test/java/com/thinkaurelius/titan/diskstorage/lucene/LuceneExample.java @@ -64,9 +64,9 @@ private SpatialStrategy getSpatialStrategy(String key) { @Test public void example1() throws Exception { - Directory dir = FSDirectory.open(path); + Directory dir = FSDirectory.open(path.toPath()); Analyzer analyzer = new StandardAnalyzer(); - IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_4_10_4, analyzer); + IndexWriterConfig iwc = new IndexWriterConfig(analyzer); iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); IndexWriter writer = new IndexWriter(dir, iwc); @@ -91,7 +91,7 @@ public void example1() throws Exception { writer.close(); //Search - IndexReader reader = DirectoryReader.open(FSDirectory.open(path)); + IndexReader reader = DirectoryReader.open(FSDirectory.open(path.toPath())); IndexSearcher searcher = new IndexSearcher(reader); analyzer = new StandardAnalyzer(); @@ -99,7 +99,7 @@ public void example1() throws Exception { BooleanFilter filter = new BooleanFilter(); //filter.add(new TermsFilter(new Term("name_txt","know")), BooleanClause.Occur.MUST); - SpatialArgs args = new SpatialArgs(SpatialOperation.Intersects,Geoshape.circle(51.666167,6.58905,450).convert2Spatial4j()); + SpatialArgs args = new SpatialArgs(SpatialOperation.Intersects,Geoshape.circle(51.666167,6.58905,450).getShape()); //filter.add(getSpatialStrategy("location").makeFilter(args), BooleanClause.Occur.MUST); filter.add(NumericRangeFilter.newLongRange("time",(long)1000342034,(long)1000342034,true,true), BooleanClause.Occur.MUST); @@ -152,7 +152,7 @@ void indexDocs(IndexWriter writer, String docid, Map docMap) thro field = new StringField(key+STR_SUFFIX, str, Field.Store.NO); doc.add(field); } else if (value instanceof Geoshape) { - Shape shape = ((Geoshape)value).convert2Spatial4j(); + Shape shape = ((Geoshape)value).getShape(); for (IndexableField f : getSpatialStrategy(key).createIndexableFields(shape)) { doc.add(f); } diff --git a/titan-lucene/src/test/java/com/thinkaurelius/titan/diskstorage/lucene/LuceneIndexTest.java b/titan-lucene/src/test/java/com/thinkaurelius/titan/diskstorage/lucene/LuceneIndexTest.java index 6fa596b43f..d6edc00410 100644 --- a/titan-lucene/src/test/java/com/thinkaurelius/titan/diskstorage/lucene/LuceneIndexTest.java +++ b/titan-lucene/src/test/java/com/thinkaurelius/titan/diskstorage/lucene/LuceneIndexTest.java @@ -94,6 +94,15 @@ public void testSupport() { assertTrue(index.supports(of(UUID.class, Cardinality.SINGLE), Cmp.EQUAL)); assertTrue(index.supports(of(UUID.class, Cardinality.SINGLE), Cmp.NOT_EQUAL)); + + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE))); + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE), Geo.WITHIN)); + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE), Geo.INTERSECT)); + assertFalse(index.supports(of(Geoshape.class, Cardinality.SINGLE), Geo.DISJOINT)); + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE, new Parameter("mapping",Mapping.PREFIX_TREE)), Geo.WITHIN)); + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE, new Parameter("mapping",Mapping.PREFIX_TREE)), Geo.CONTAINS)); + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE, new Parameter("mapping",Mapping.PREFIX_TREE)), Geo.INTERSECT)); + assertFalse(index.supports(of(Geoshape.class, Cardinality.SINGLE, new Parameter("mapping",Mapping.PREFIX_TREE)), Geo.DISJOINT)); } // @Override diff --git a/titan-solr/pom.xml b/titan-solr/pom.xml index e8badfdbaf..0eb0ed9afe 100644 --- a/titan-solr/pom.xml +++ b/titan-solr/pom.xml @@ -87,11 +87,6 @@ com.spatial4j spatial4j - - com.vividsolutions - jts - 1.13 - ${basedir}/target diff --git a/titan-solr/src/main/java/com/thinkaurelius/titan/diskstorage/solr/SolrIndex.java b/titan-solr/src/main/java/com/thinkaurelius/titan/diskstorage/solr/SolrIndex.java index c18104d79a..d6c35b0574 100644 --- a/titan-solr/src/main/java/com/thinkaurelius/titan/diskstorage/solr/SolrIndex.java +++ b/titan-solr/src/main/java/com/thinkaurelius/titan/diskstorage/solr/SolrIndex.java @@ -3,7 +3,6 @@ import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.Sets; - import com.thinkaurelius.titan.core.Cardinality; import com.thinkaurelius.titan.graphdb.internal.Order; import com.thinkaurelius.titan.core.TitanElement; @@ -49,6 +48,9 @@ import java.text.SimpleDateFormat; import java.time.Instant; import java.util.*; +import java.util.AbstractMap.SimpleEntry; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration.*; @@ -148,6 +150,8 @@ public static Mode parse(String mode) { private static final IndexFeatures SOLR_FEATURES = new IndexFeatures.Builder().supportsDocumentTTL() .setDefaultStringMapping(Mapping.TEXT).supportedStringMappings(Mapping.TEXT, Mapping.STRING).supportsCardinality(Cardinality.SINGLE).build(); + private static Map SPATIAL_PREDICATES = spatialPredicates(); + private final SolrClient solrClient; private final Configuration configuration; private final Mode mode; @@ -566,27 +570,27 @@ public String buildQueryFilter(Condition condition, KeyInformation throw new IllegalArgumentException("Relation is not supported for string value: " + titanPredicate); } } else if (value instanceof Geoshape) { + Mapping map = Mapping.getMapping(informations.get(key)); + Preconditions.checkArgument(titanPredicate instanceof Geo && titanPredicate != Geo.DISJOINT, "Relation not supported on geo types: " + titanPredicate); + Preconditions.checkArgument(map == Mapping.PREFIX_TREE || titanPredicate == Geo.WITHIN || titanPredicate == Geo.INTERSECT, "Relation not supported on geopoint types: " + titanPredicate); Geoshape geo = (Geoshape)value; - if (geo.getType() == Geoshape.Type.CIRCLE) { + if (geo.getType() == Geoshape.Type.CIRCLE && (titanPredicate == Geo.INTERSECT || map == Mapping.DEFAULT)) { Geoshape.Point center = geo.getPoint(); return ("{!geofilt sfield=" + key + " pt=" + center.getLatitude() + "," + center.getLongitude() + " d=" + geo.getRadius() + "} distErrPct=0"); //distance in kilometers - } else if (geo.getType() == Geoshape.Type.BOX) { + } else if (geo.getType() == Geoshape.Type.BOX && (titanPredicate == Geo.INTERSECT || map == Mapping.DEFAULT)) { Geoshape.Point southwest = geo.getPoint(0); Geoshape.Point northeast = geo.getPoint(1); return (key + ":[" + southwest.getLatitude() + "," + southwest.getLongitude() + " TO " + northeast.getLatitude() + "," + northeast.getLongitude() + "]"); - } else if (geo.getType() == Geoshape.Type.POLYGON) { - List coordinates = getPolygonPoints(geo); - StringBuilder poly = new StringBuilder(key + ":\"IsWithin(POLYGON(("); - for (Geoshape.Point coordinate : coordinates) { - poly.append(coordinate.getLongitude()).append(" ").append(coordinate.getLatitude()).append(", "); - } - //close the polygon with the first coordinate - poly.append(coordinates.get(0).getLongitude()).append(" ").append(coordinates.get(0).getLatitude()); - poly.append(")))\" distErrPct=0"); - return (poly.toString()); + } else if (map == Mapping.PREFIX_TREE) { + StringBuilder builder = new StringBuilder(key + ":\""); + builder.append(SPATIAL_PREDICATES.get((Geo) titanPredicate) + "("); + builder.append(geo + ")\" distErrPct=0"); + return builder.toString(); + } else { + throw new IllegalArgumentException("Unsupported or invalid search shape type: " + geo.getType()); } } else if (value instanceof Date || value instanceof Instant) { String s = value.toString(); @@ -669,7 +673,6 @@ public String buildQueryFilter(Condition condition, KeyInformation } else { throw new IllegalArgumentException("Invalid condition: " + condition); } - return null; } private String toIsoDate(Date value) { @@ -679,24 +682,6 @@ private String toIsoDate(Date value) { return df.format(value); } - private List getPolygonPoints(Geoshape polygon) { - List locations = new ArrayList(); - - int index = 0; - boolean hasCoordinates = true; - while (hasCoordinates) { - try { - locations.add(polygon.getPoint(index)); - } catch (ArrayIndexOutOfBoundsException ignore) { - //just means we asked for a point past the size of the list - //of known coordinates - hasCoordinates = false; - } - } - - return locations; - } - /** * Solr handles all transactions on the server-side. That means all * commit, optimize, or rollback applies since the last commit/optimize/rollback. @@ -752,7 +737,8 @@ public void clearStorage() throws BackendException { public boolean supports(KeyInformation information, TitanPredicate titanPredicate) { Class dataType = information.getDataType(); Mapping mapping = Mapping.getMapping(information); - if (mapping!=Mapping.DEFAULT && !AttributeUtil.isString(dataType)) return false; + if (mapping!=Mapping.DEFAULT && !AttributeUtil.isString(dataType) && + !(mapping==Mapping.PREFIX_TREE && AttributeUtil.isGeo(dataType))) return false; if(information.getCardinality() != Cardinality.SINGLE) { return false; @@ -761,7 +747,12 @@ public boolean supports(KeyInformation information, TitanPredicate titanPredicat if (Number.class.isAssignableFrom(dataType)) { return titanPredicate instanceof Cmp; } else if (dataType == Geoshape.class) { - return titanPredicate == Geo.WITHIN; + switch(mapping) { + case DEFAULT: + return titanPredicate == Geo.WITHIN || titanPredicate == Geo.INTERSECT; + case PREFIX_TREE: + return titanPredicate == Geo.INTERSECT || titanPredicate == Geo.WITHIN || titanPredicate == Geo.CONTAINS; + } } else if (AttributeUtil.isString(dataType)) { switch(mapping) { case DEFAULT: @@ -789,10 +780,12 @@ public boolean supports(KeyInformation information) { } Class dataType = information.getDataType(); Mapping mapping = Mapping.getMapping(information); - if (Number.class.isAssignableFrom(dataType) || dataType == Geoshape.class || dataType == Date.class || dataType == Instant.class || dataType == Boolean.class || dataType == UUID.class) { + if (Number.class.isAssignableFrom(dataType) || dataType == Date.class || dataType == Instant.class || dataType == Boolean.class || dataType == UUID.class) { if (mapping==Mapping.DEFAULT) return true; } else if (AttributeUtil.isString(dataType)) { if (mapping==Mapping.DEFAULT || mapping==Mapping.TEXT || mapping==Mapping.STRING) return true; + } else if (AttributeUtil.isGeo(dataType)) { + if (mapping==Mapping.DEFAULT || mapping==Mapping.PREFIX_TREE) return true; } return false; } @@ -845,6 +838,15 @@ private static Mapping getStringMapping(KeyInformation information) { return map; } + private static Map spatialPredicates() { + return Collections.unmodifiableMap(Stream.of( + new SimpleEntry<>(Geo.WITHIN, "IsWithin"), + new SimpleEntry<>(Geo.CONTAINS, "Contains"), + new SimpleEntry<>(Geo.INTERSECT, "Intersects"), + new SimpleEntry<>(Geo.DISJOINT, "IsDisjointTo")) + .collect(Collectors.toMap((e) -> e.getKey(), (e) -> e.getValue()))); + } + private UpdateRequest newUpdateRequest() { UpdateRequest req = new UpdateRequest(); if(waitSearcher) { diff --git a/titan-solr/src/main/java/com/thinkaurelius/titan/diskstorage/solr/transform/GeoToWktConverter.java b/titan-solr/src/main/java/com/thinkaurelius/titan/diskstorage/solr/transform/GeoToWktConverter.java index e38ed1928f..0d9472fa54 100644 --- a/titan-solr/src/main/java/com/thinkaurelius/titan/diskstorage/solr/transform/GeoToWktConverter.java +++ b/titan-solr/src/main/java/com/thinkaurelius/titan/diskstorage/solr/transform/GeoToWktConverter.java @@ -2,19 +2,13 @@ import com.thinkaurelius.titan.core.attribute.Geoshape; import com.thinkaurelius.titan.diskstorage.BackendException; -import com.thinkaurelius.titan.diskstorage.PermanentBackendException; public class GeoToWktConverter { /** - * {@link com.thinkaurelius.titan.core.attribute.Geoshape} stores Points in the String format: point[X.0,Y.0]. - * Solr needs it to be in Well-Known Text format: POINT(X.0 Y.0) + * Get Well-Known Text format (e.g. POINT(X.0 Y.0)) for geoshape for indexing in solr. + * Only points, lines and polygons are supported. */ public static String convertToWktString(Geoshape fieldValue) throws BackendException { - if (fieldValue.getType() == Geoshape.Type.POINT) { - Geoshape.Point point = fieldValue.getPoint(); - return "POINT(" + point.getLongitude() + " " + point.getLatitude() + ")"; - } else { - throw new PermanentBackendException("Cannot index " + fieldValue.getType()); - } + return fieldValue.toString(); } } diff --git a/titan-solr/src/test/java/com/thinkaurelius/titan/diskstorage/solr/SolrIndexTest.java b/titan-solr/src/test/java/com/thinkaurelius/titan/diskstorage/solr/SolrIndexTest.java index 37e52ddb25..3f95eea7ce 100644 --- a/titan-solr/src/test/java/com/thinkaurelius/titan/diskstorage/solr/SolrIndexTest.java +++ b/titan-solr/src/test/java/com/thinkaurelius/titan/diskstorage/solr/SolrIndexTest.java @@ -75,7 +75,6 @@ public void testSupport() { assertTrue(index.supports(of(Short.class, Cardinality.SINGLE))); assertTrue(index.supports(of(Byte.class, Cardinality.SINGLE))); assertTrue(index.supports(of(Float.class, Cardinality.SINGLE))); - assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE))); assertFalse(index.supports(of(Object.class, Cardinality.SINGLE))); assertFalse(index.supports(of(Exception.class, Cardinality.SINGLE))); @@ -96,11 +95,19 @@ public void testSupport() { assertTrue(index.supports(of(Double.class, Cardinality.SINGLE), Cmp.LESS_THAN)); assertTrue(index.supports(of(Double.class, Cardinality.SINGLE, new Parameter("mapping",Mapping.DEFAULT)), Cmp.LESS_THAN)); assertFalse(index.supports(of(Double.class, Cardinality.SINGLE, new Parameter("mapping",Mapping.TEXT)), Cmp.LESS_THAN)); + + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE))); assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE), Geo.WITHIN)); + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE), Geo.INTERSECT)); + assertFalse(index.supports(of(Geoshape.class, Cardinality.SINGLE), Geo.CONTAINS)); + assertFalse(index.supports(of(Geoshape.class, Cardinality.SINGLE), Geo.DISJOINT)); + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE, new Parameter("mapping",Mapping.PREFIX_TREE)), Geo.CONTAINS)); + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE, new Parameter("mapping",Mapping.PREFIX_TREE)), Geo.WITHIN)); + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE, new Parameter("mapping",Mapping.PREFIX_TREE)), Geo.INTERSECT)); + assertFalse(index.supports(of(Geoshape.class, Cardinality.SINGLE, new Parameter("mapping",Mapping.PREFIX_TREE)), Geo.DISJOINT)); assertFalse(index.supports(of(Double.class, Cardinality.SINGLE), Geo.INTERSECT)); assertFalse(index.supports(of(Long.class, Cardinality.SINGLE), Text.CONTAINS)); - assertFalse(index.supports(of(Geoshape.class, Cardinality.SINGLE), Geo.DISJOINT)); assertTrue(index.supports(of(Date.class, Cardinality.SINGLE), Cmp.EQUAL)); assertTrue(index.supports(of(Date.class, Cardinality.SINGLE), Cmp.LESS_THAN_EQUAL)); diff --git a/titan-solr/src/test/java/com/thinkaurelius/titan/diskstorage/solr/transform/GeoToWktConverterTest.java b/titan-solr/src/test/java/com/thinkaurelius/titan/diskstorage/solr/transform/GeoToWktConverterTest.java index 7862ebb0cd..562a877400 100644 --- a/titan-solr/src/test/java/com/thinkaurelius/titan/diskstorage/solr/transform/GeoToWktConverterTest.java +++ b/titan-solr/src/test/java/com/thinkaurelius/titan/diskstorage/solr/transform/GeoToWktConverterTest.java @@ -2,38 +2,104 @@ import com.thinkaurelius.titan.core.attribute.Geoshape; import com.thinkaurelius.titan.diskstorage.BackendException; +import com.vividsolutions.jts.geom.Coordinate; +import com.vividsolutions.jts.geom.GeometryFactory; +import com.vividsolutions.jts.geom.LineString; +import com.vividsolutions.jts.geom.LinearRing; +import com.vividsolutions.jts.geom.Polygon; + import org.junit.Test; import static org.junit.Assert.assertEquals; +import java.text.ParseException; + /** * @author Jared Holmberg (jholmberg@bericotechnologies.com) */ public class GeoToWktConverterTest { /** - * Titan Geoshapes are converted to a string that gets sent to its respective index. Unfortunately, the string format - * is not compatible with Solr 4. The GeoToWktConverter transforms the Geoshape's string value into a Well-Known Text + * The GeoToWktConverter transforms the Geoshape's string value into a Well-Known Text * format understood by Solr. */ @Test - public void testConvertGeoshapePointToWktString() throws BackendException { + public void testConvertGeoshapePointToWktString() throws BackendException, ParseException { Geoshape p1 = Geoshape.point(35.4, 48.9); //no spaces, no negative values Geoshape p2 = Geoshape.point(-35.4,48.9); //negative longitude value Geoshape p3 = Geoshape.point(35.4, -48.9); //negative latitude value String wkt1 = "POINT(48.9 35.4)"; - String actualWkt1 = GeoToWktConverter.convertToWktString(p1); + assertEquals(p1.getPoint().getLongitude(), Geoshape.fromWkt(wkt1).getPoint().getLongitude(), 1e-5); + assertEquals(p1.getPoint().getLatitude(), Geoshape.fromWkt(wkt1).getPoint().getLatitude(), 1e-5); String wkt2 = "POINT(48.9 -35.4)"; - String actualWkt2 = GeoToWktConverter.convertToWktString(p2); + assertEquals(p2.getPoint().getLongitude(), Geoshape.fromWkt(wkt2).getPoint().getLongitude(), 1e-5); + assertEquals(p2.getPoint().getLatitude(), Geoshape.fromWkt(wkt2).getPoint().getLatitude(), 1e-5); String wkt3 = "POINT(-48.9 35.4)"; - String actualWkt3 = GeoToWktConverter.convertToWktString(p3); + assertEquals(p3.getPoint().getLongitude(), Geoshape.fromWkt(wkt3).getPoint().getLongitude(), 1e-5); + assertEquals(p3.getPoint().getLatitude(), Geoshape.fromWkt(wkt3).getPoint().getLatitude(), 1e-5); + } + + @Test + public void testConvertGeoshapeLineToWktString() throws BackendException { + Geoshape l1 = Geoshape.line(35.4, 48.9, 35.6, 49.1); + String wkt1 = "LINESTRING (48.9 35.4, 49.1 35.6)"; + String actualWkt1 = GeoToWktConverter.convertToWktString(l1); assertEquals(wkt1, actualWkt1); + } + + @Test + public void testConvertGeoshapePolygonToWktString() throws BackendException { + GeometryFactory gf = new GeometryFactory(); + Geoshape p1 = Geoshape.polygon(35.4, 48.9, 35.6, 48.9, 35.6, 49.1, 35.4, 49.1, 35.4, 48.9); + Geoshape p2 = Geoshape.geoshape(gf.createPolygon(gf.createLinearRing(new Coordinate[] { + new Coordinate(10,10), new Coordinate(20,10), new Coordinate(20,20), new Coordinate(10,20), new Coordinate(10,10)}), + new LinearRing[] { gf.createLinearRing(new Coordinate[] { + new Coordinate(13,13), new Coordinate(17,13), new Coordinate(17,17), new Coordinate(13,17), new Coordinate(13,13)})})); + + String wkt1 = "POLYGON ((48.9 35.4, 48.9 35.6, 49.1 35.6, 49.1 35.4, 48.9 35.4))"; + String actualWkt1 = GeoToWktConverter.convertToWktString(p1); + assertEquals(wkt1, actualWkt1); + + String wkt2 = "POLYGON ((10 10, 20 10, 20 20, 10 20, 10 10), (13 13, 17 13, 17 17, 13 17, 13 13))"; + String actualWkt2 = GeoToWktConverter.convertToWktString(p2); assertEquals(wkt2, actualWkt2); - assertEquals(wkt3, actualWkt3); + } + + @Test + public void testConvertGeoshapeMultiPointToWktString() throws BackendException { + GeometryFactory gf = new GeometryFactory(); + Geoshape g = Geoshape.geoshape(gf.createMultiPoint(new Coordinate[] {new Coordinate(10,10), new Coordinate(20,20)})); + + String wkt1 = "MULTIPOINT ((10 10), (20 20))"; + String actualWkt1 = GeoToWktConverter.convertToWktString(g); + assertEquals(wkt1, actualWkt1); + } + + @Test + public void testConvertGeoshapeMultiLineToWktString() throws BackendException { + GeometryFactory gf = new GeometryFactory(); + Geoshape g = Geoshape.geoshape(gf.createMultiLineString(new LineString[] { + gf.createLineString(new Coordinate[] {new Coordinate(10,10), new Coordinate(20,20)}), + gf.createLineString(new Coordinate[] {new Coordinate(30,30), new Coordinate(40,40)})})); + + String wkt1 = "MULTILINESTRING ((10 10, 20 20), (30 30, 40 40))"; + String actualWkt1 = GeoToWktConverter.convertToWktString(g); + assertEquals(wkt1, actualWkt1); + } + @Test + public void testConvertGeoshapeMultiPolygonToWktString() throws BackendException { + GeometryFactory gf = new GeometryFactory(); + Geoshape g = Geoshape.geoshape(gf.createMultiPolygon(new Polygon[] { + gf.createPolygon(new Coordinate[] {new Coordinate(0,0), new Coordinate(0,10), new Coordinate(10,10), new Coordinate(0,0)}), + gf.createPolygon(new Coordinate[] {new Coordinate(20,20), new Coordinate(20,30), new Coordinate(30,30), new Coordinate(20,20)})})); + + String wkt1 = "MULTIPOLYGON (((0 0, 0 10, 10 10, 0 0)), ((20 20, 20 30, 30 30, 20 20)))"; + String actualWkt1 = GeoToWktConverter.convertToWktString(g); + assertEquals(wkt1, actualWkt1); } } diff --git a/titan-solr/src/test/resources/solr/core-template/schema.xml b/titan-solr/src/test/resources/solr/core-template/schema.xml index 540ab42f0d..517a1e67fb 100644 --- a/titan-solr/src/test/resources/solr/core-template/schema.xml +++ b/titan-solr/src/test/resources/solr/core-template/schema.xml @@ -534,7 +534,7 @@ @@ -551,6 +551,7 @@ + diff --git a/titan-test/src/main/java/com/thinkaurelius/titan/diskstorage/indexing/IndexProviderTest.java b/titan-test/src/main/java/com/thinkaurelius/titan/diskstorage/indexing/IndexProviderTest.java index 19184aecfd..ec9848bd2f 100644 --- a/titan-test/src/main/java/com/thinkaurelius/titan/diskstorage/indexing/IndexProviderTest.java +++ b/titan-test/src/main/java/com/thinkaurelius/titan/diskstorage/indexing/IndexProviderTest.java @@ -28,7 +28,6 @@ import java.util.*; import static org.junit.Assert.*; -import static org.junit.Assert.assertFalse; /** * @author Matthias Broecheler (me@matthiasb.com) @@ -48,7 +47,7 @@ public abstract class IndexProviderTest { protected Map allKeys; protected KeyInformation.IndexRetriever indexRetriever; - public static final String TEXT = "text", TIME = "time", WEIGHT = "weight", LOCATION = "location", NAME = "name", PHONE_LIST = "phone_list", PHONE_SET = "phone_set", DATE = "date"; + public static final String TEXT = "text", TIME = "time", WEIGHT = "weight", LOCATION = "location", BOUNDARY = "boundary", NAME = "name", PHONE_LIST = "phone_list", PHONE_SET = "phone_set", DATE = "date"; public static StandardKeyInformation of(Class clazz, Cardinality cardinality, Parameter... paras) { return new StandardKeyInformation(clazz, cardinality, paras); @@ -85,6 +84,7 @@ public static final Map getMapping(final IndexFeatures in put(TIME,new StandardKeyInformation(Long.class, Cardinality.SINGLE)); put(WEIGHT,new StandardKeyInformation(Double.class, Cardinality.SINGLE, new Parameter("mapping",Mapping.DEFAULT))); put(LOCATION,new StandardKeyInformation(Geoshape.class, Cardinality.SINGLE)); + put(BOUNDARY,new StandardKeyInformation(Geoshape.class, Cardinality.SINGLE, new Parameter("mapping",Mapping.PREFIX_TREE))); put(NAME,new StandardKeyInformation(String.class, Cardinality.SINGLE, new Parameter("mapping", indexFeatures.supportsStringMapping(Mapping.STRING)?Mapping.STRING:Mapping.TEXTSTRING))); if(indexFeatures.supportsCardinality(Cardinality.LIST)) { @@ -163,9 +163,9 @@ public void multipleStores() throws Exception { private void storeTest(String... stores) throws Exception { - Multimap doc1 = getDocument("Hello world", 1001, 5.2, Geoshape.point(48.0, 0.0), Arrays.asList("1", "2", "3"), Sets.newHashSet("1", "2"), Instant.ofEpochSecond(1)); - Multimap doc2 = getDocument("Tomorrow is the world", 1010, 8.5, Geoshape.point(49.0, 1.0), Arrays.asList("4", "5", "6"), Sets.newHashSet("4", "5"), Instant.ofEpochSecond(2)); - Multimap doc3 = getDocument("Hello Bob, are you there?", -500, 10.1, Geoshape.point(47.0, 10.0), Arrays.asList("7", "8", "9"), Sets.newHashSet("7", "8"), Instant.ofEpochSecond(3)); + Multimap doc1 = getDocument("Hello world", 1001, 5.2, Geoshape.point(48.0, 0.0), Geoshape.polygon(47.9,-0.1,48.1,-0.1,48.1,0.1,47.9,0.1,47.9,-0.1),Arrays.asList("1", "2", "3"), Sets.newHashSet("1", "2"), Instant.ofEpochSecond(1)); + Multimap doc2 = getDocument("Tomorrow is the world", 1010, 8.5, Geoshape.point(49.0, 1.0), Geoshape.line(48.9,0.9,49.1,0.9,49.1,1.1,48.9,1.1), Arrays.asList("4", "5", "6"), Sets.newHashSet("4", "5"), Instant.ofEpochSecond(2)); + Multimap doc3 = getDocument("Hello Bob, are you there?", -500, 10.1, Geoshape.point(47.0, 10.0), Geoshape.polygon(46.9,9.9,47.1,9.9,47.1,10.1,46.9,10.1,46.9,9.9), Arrays.asList("7", "8", "9"), Sets.newHashSet("7", "8"), Instant.ofEpochSecond(3)); for (String store : stores) { initialize(store); @@ -291,18 +291,59 @@ private void storeTest(String... stores) throws Exception { assertEquals(1, result.size()); assertEquals("doc2", result.get(0)); + result = tx.query(new IndexQuery(store, PredicateCondition.of(LOCATION, Geo.WITHIN, Geoshape.box(46.5, -0.5, 50.5, 10.5)))); + assertEquals(3,result.size()); + assertEquals(ImmutableSet.of("doc1", "doc2", "doc3"), ImmutableSet.copyOf(result)); + result = tx.query(new IndexQuery(store, PredicateCondition.of(LOCATION, Geo.WITHIN, Geoshape.circle(48.5, 0.5, 200.00)))); assertEquals(2, result.size()); assertEquals(ImmutableSet.of("doc1", "doc2"), ImmutableSet.copyOf(result)); - result = tx.query(new IndexQuery(store, And.of(PredicateCondition.of(TEXT, Text.CONTAINS, "tomorrow"), PredicateCondition.of(LOCATION, Geo.WITHIN, Geoshape.circle(48.5, 0.5, 200.00))))); - assertEquals(ImmutableSet.of("doc2"), ImmutableSet.copyOf(result)); - - result = tx.query(new IndexQuery(store, PredicateCondition.of("location", Geo.WITHIN, Geoshape.box(46.5, -0.5, 50.5, 10.5)))); + result = tx.query(new IndexQuery(store, PredicateCondition.of(BOUNDARY, Geo.WITHIN, Geoshape.box(46.5, -0.5, 50.5, 10.5)))); assertEquals(3,result.size()); assertEquals(ImmutableSet.of("doc1", "doc2", "doc3"), ImmutableSet.copyOf(result)); - result = tx.query(new IndexQuery(store, And.of(PredicateCondition.of(TIME, Cmp.GREATER_THAN_EQUAL, -1000), PredicateCondition.of(TIME, Cmp.LESS_THAN, 1010), PredicateCondition.of(LOCATION, Geo.WITHIN, Geoshape.circle(48.5, 0.5, 1000.00))))); + result = tx.query(new IndexQuery(store, PredicateCondition.of(BOUNDARY, Geo.WITHIN, Geoshape.circle(48.5, 0.5, 200.00)))); + assertEquals(2, result.size()); + assertEquals(ImmutableSet.of("doc1", "doc2"), ImmutableSet.copyOf(result)); + + result = tx.query(new IndexQuery(store, PredicateCondition.of(BOUNDARY, Geo.WITHIN, Geoshape.polygon(48.5,-0.5,47.5,-0.5,47.5,0.5,48.5,0.5,48.5,-0.5)))); + assertEquals(1, result.size()); + assertEquals(ImmutableSet.of("doc1"), ImmutableSet.copyOf(result)); + + if (index.supports(new StandardKeyInformation(Geoshape.class, Cardinality.SINGLE, new Parameter("mapping", Mapping.PREFIX_TREE)), Geo.DISJOINT)) { + result = tx.query(new IndexQuery(store, PredicateCondition.of(BOUNDARY, Geo.DISJOINT, Geoshape.box(46.5, -0.5, 50.5, 10.5)))); + assertEquals(0,result.size()); + + result = tx.query(new IndexQuery(store, PredicateCondition.of(BOUNDARY, Geo.DISJOINT, Geoshape.circle(48.5, 0.5, 200.00)))); + assertEquals(1, result.size()); + assertEquals(ImmutableSet.of("doc3"), ImmutableSet.copyOf(result)); + + result = tx.query(new IndexQuery(store, PredicateCondition.of(BOUNDARY, Geo.DISJOINT, Geoshape.polygon(48.5,-0.5,47.5,-0.5,47.5,0.5,48.5,0.5,48.5,-0.5)))); + assertEquals(2, result.size()); + assertEquals(ImmutableSet.of("doc2","doc3"), ImmutableSet.copyOf(result)); + } + + result = tx.query(new IndexQuery(store, PredicateCondition.of(BOUNDARY, Geo.CONTAINS, Geoshape.point(47,10)))); + assertEquals(1, result.size()); + assertEquals(ImmutableSet.of("doc3"), ImmutableSet.copyOf(result)); + + result = tx.query(new IndexQuery(store, PredicateCondition.of(BOUNDARY, Geo.INTERSECT, Geoshape.box(48,-1,49,2)))); + assertEquals(2,result.size()); + assertEquals(ImmutableSet.of("doc1","doc2"), ImmutableSet.copyOf(result)); + + result = tx.query(new IndexQuery(store, PredicateCondition.of(BOUNDARY, Geo.INTERSECT, Geoshape.circle(48.5, 0.5, 200.00)))); + assertEquals(2, result.size()); + assertEquals(ImmutableSet.of("doc1", "doc2"), ImmutableSet.copyOf(result)); + + result = tx.query(new IndexQuery(store, PredicateCondition.of(BOUNDARY, Geo.INTERSECT, Geoshape.polygon(48,-1,49,-1,49,2,48,2,48,-1)))); + assertEquals(2, result.size()); + assertEquals(ImmutableSet.of("doc1","doc2"), ImmutableSet.copyOf(result)); + + result = tx.query(new IndexQuery(store, And.of(PredicateCondition.of("text", Text.CONTAINS, "tomorrow"), PredicateCondition.of(LOCATION, Geo.WITHIN, Geoshape.circle(48.5, 0.5, 200.00)), PredicateCondition.of(BOUNDARY, Geo.WITHIN, Geoshape.circle(48.5, 0.5, 200.00))))); + assertEquals(ImmutableSet.of("doc2"), ImmutableSet.copyOf(result)); + + result = tx.query(new IndexQuery(store, And.of(PredicateCondition.of(TIME, Cmp.GREATER_THAN_EQUAL, -1000), PredicateCondition.of(TIME, Cmp.LESS_THAN, 1010), PredicateCondition.of(LOCATION, Geo.WITHIN, Geoshape.circle(48.5, 0.5, 1000.00)), PredicateCondition.of(BOUNDARY, Geo.WITHIN, Geoshape.circle(48.5, 0.5, 1000.00))))); assertEquals(ImmutableSet.of("doc1", "doc3"), ImmutableSet.copyOf(result)); result = tx.query(new IndexQuery(store, And.of(PredicateCondition.of(WEIGHT, Cmp.GREATER_THAN, 10.0)))); @@ -353,7 +394,7 @@ private void storeTest(String... stores) throws Exception { //Update some data - add(store, "doc4", getDocument("I'ts all a big Bob", -100, 11.2, Geoshape.point(48.0, 8.0), Arrays.asList("10", "11", "12"), Sets.newHashSet("10", "11"), Instant.ofEpochSecond(4)), true); + add(store, "doc4", getDocument("I'ts all a big Bob", -100, 11.2, Geoshape.point(48.0, 8.0), Geoshape.line(47.5, 7.5, 48.5, 8.5), Arrays.asList("10", "11", "12"), Sets.newHashSet("10", "11"), Instant.ofEpochSecond(4)), true); remove(store, "doc2", doc2, true); remove(store, "doc3", ImmutableMultimap.of(WEIGHT, (Object) 10.1), false); add(store, "doc3", ImmutableMultimap.of(TIME, (Object) 2000, TEXT, "Bob owns the world"), false); @@ -377,12 +418,21 @@ private void storeTest(String... stores) throws Exception { result = tx.query(new IndexQuery(store, PredicateCondition.of(LOCATION, Geo.WITHIN, Geoshape.circle(48.5, 0.5, 200.00)))); assertEquals(ImmutableSet.of("doc1"), ImmutableSet.copyOf(result)); + result = tx.query(new IndexQuery(store, PredicateCondition.of(BOUNDARY, Geo.WITHIN, Geoshape.circle(48.5, 0.5, 200.00)))); + assertEquals(ImmutableSet.of("doc1"), ImmutableSet.copyOf(result)); + result = tx.query(new IndexQuery(store, And.of(PredicateCondition.of(TEXT, Text.CONTAINS, "tomorrow"), PredicateCondition.of(LOCATION, Geo.WITHIN, Geoshape.circle(48.5, 0.5, 200.00))))); assertEquals(ImmutableSet.of(), ImmutableSet.copyOf(result)); + result = tx.query(new IndexQuery(store, And.of(PredicateCondition.of(TEXT, Text.CONTAINS, "tomorrow"), PredicateCondition.of(BOUNDARY, Geo.WITHIN, Geoshape.circle(48.5, 0.5, 200.00))))); + assertEquals(ImmutableSet.of(), ImmutableSet.copyOf(result)); + result = tx.query(new IndexQuery(store, And.of(PredicateCondition.of(TIME, Cmp.GREATER_THAN_EQUAL, -1000), PredicateCondition.of(TIME, Cmp.LESS_THAN, 1010), PredicateCondition.of(LOCATION, Geo.WITHIN, Geoshape.circle(48.5, 0.5, 1000.00))))); assertEquals(ImmutableSet.of("doc1", "doc4"), ImmutableSet.copyOf(result)); + result = tx.query(new IndexQuery(store, And.of(PredicateCondition.of(TIME, Cmp.GREATER_THAN_EQUAL, -1000), PredicateCondition.of(TIME, Cmp.LESS_THAN, 1010), PredicateCondition.of(BOUNDARY, Geo.WITHIN, Geoshape.circle(48.5, 0.5, 1000.00))))); + assertEquals(ImmutableSet.of("doc1", "doc4"), ImmutableSet.copyOf(result)); + result = tx.query(new IndexQuery(store, And.of(PredicateCondition.of(WEIGHT, Cmp.GREATER_THAN, 10.0)))); assertEquals(ImmutableSet.of("doc1", "doc4"), ImmutableSet.copyOf(result)); @@ -427,7 +477,6 @@ public void testCommonSupport() { assertTrue(index.supports(of(Short.class, Cardinality.SINGLE))); assertTrue(index.supports(of(Byte.class, Cardinality.SINGLE))); assertTrue(index.supports(of(Float.class, Cardinality.SINGLE))); - assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE))); assertFalse(index.supports(of(Object.class, Cardinality.SINGLE))); assertFalse(index.supports(of(Exception.class, Cardinality.SINGLE))); @@ -436,11 +485,16 @@ public void testCommonSupport() { assertTrue(index.supports(of(Double.class, Cardinality.SINGLE), Cmp.LESS_THAN)); assertTrue(index.supports(of(Double.class, Cardinality.SINGLE, new Parameter("mapping",Mapping.DEFAULT)), Cmp.LESS_THAN)); assertFalse(index.supports(of(Double.class, Cardinality.SINGLE, new Parameter("mapping",Mapping.TEXT)), Cmp.LESS_THAN)); - assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE), Geo.WITHIN)); assertFalse(index.supports(of(Double.class, Cardinality.SINGLE), Geo.INTERSECT)); assertFalse(index.supports(of(Long.class, Cardinality.SINGLE), Text.CONTAINS)); - assertFalse(index.supports(of(Geoshape.class, Cardinality.SINGLE), Geo.DISJOINT)); + + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE))); + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE), Geo.WITHIN)); + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE), Geo.INTERSECT)); + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE, new Parameter("mapping",Mapping.PREFIX_TREE)), Geo.WITHIN)); + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE, new Parameter("mapping",Mapping.PREFIX_TREE)), Geo.CONTAINS)); + assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE, new Parameter("mapping",Mapping.PREFIX_TREE)), Geo.INTERSECT)); } @Test @@ -848,13 +902,14 @@ private void remove(String store, String docid, Multimap doc, bo } - public Multimap getDocument(final String txt, final long time, final double weight, final Geoshape geo, List phoneList, Set phoneSet, Instant date) { + public Multimap getDocument(final String txt, final long time, final double weight, final Geoshape location, final Geoshape boundary, List phoneList, Set phoneSet, Instant date) { HashMultimap values = HashMultimap.create(); values.put(TEXT, txt); values.put(NAME, txt); values.put(TIME, time); values.put(WEIGHT, weight); - values.put(LOCATION, geo); + values.put(LOCATION, location); + values.put(BOUNDARY, boundary); values.put(DATE, date); if(indexFeatures.supportsCardinality(Cardinality.LIST)) { for (String phone : phoneList) { diff --git a/titan-test/src/main/java/com/thinkaurelius/titan/graphdb/TitanGraphTest.java b/titan-test/src/main/java/com/thinkaurelius/titan/graphdb/TitanGraphTest.java index 67ed5194e9..20a9468084 100644 --- a/titan-test/src/main/java/com/thinkaurelius/titan/graphdb/TitanGraphTest.java +++ b/titan-test/src/main/java/com/thinkaurelius/titan/graphdb/TitanGraphTest.java @@ -1013,7 +1013,9 @@ public void testDataTypes() throws Exception { PropertyKey birthday = makeKey("birthday", Instant.class); - PropertyKey geo = makeKey("geo", Geoshape.class); + PropertyKey location = makeKey("location", Geoshape.class); + + PropertyKey boundary = makeKey("boundary", Geoshape.class); PropertyKey precise = makeKey("precise", Double.class); @@ -1040,7 +1042,8 @@ public void testDataTypes() throws Exception { num = tx.getPropertyKey("num"); barr = tx.getPropertyKey("barr"); birthday = tx.getPropertyKey("birthday"); - geo = tx.getPropertyKey("geo"); + location = tx.getPropertyKey("location"); + boundary = tx.getPropertyKey("boundary"); precise = tx.getPropertyKey("precise"); any = tx.getPropertyKey("any"); @@ -1049,6 +1052,7 @@ public void testDataTypes() throws Exception { assertEquals(Object.class, any.dataType()); final Instant c = Instant.ofEpochSecond(1429225756); + final Geoshape point = Geoshape.point(10.0, 10.0); final Geoshape shape = Geoshape.box(10.0, 10.0, 20.0, 20.0); TitanVertex v = tx.addVertex(); @@ -1056,7 +1060,8 @@ public void testDataTypes() throws Exception { v.property(VertexProperty.Cardinality.single, n(birthday), c); v.property(VertexProperty.Cardinality.single, n(num), new SpecialInt(10)); v.property(VertexProperty.Cardinality.single, n(barr), new byte[]{1, 2, 3, 4}); - v.property(VertexProperty.Cardinality.single, n(geo), shape); + v.property(VertexProperty.Cardinality.single, n(location), point); + v.property(VertexProperty.Cardinality.single, n(boundary), shape); v.property(VertexProperty.Cardinality.single, n(precise), 10.12345); v.property(n(any), "Hello"); v.property(n(any), 10l); @@ -1068,7 +1073,8 @@ public void testDataTypes() throws Exception { assertEquals(10, v.value("num").getValue()); assertEquals(c, v.value("birthday")); assertEquals(4, v.value("barr").length); - assertEquals(shape, v.value("geo")); + assertEquals(point, v.value("location")); + assertEquals(shape, v.value("boundary")); assertEquals(10.12345, v.value("precise").doubleValue(), 0.000001); assertCount(3, v.properties("any")); for (TitanVertexProperty prop : v.query().labels("any").properties()) { @@ -1089,7 +1095,8 @@ else if (value.getClass().isArray()) { assertEquals(10, v.value("num").getValue()); assertEquals(c, v.value("birthday")); assertEquals(4, v.value("barr").length); - assertEquals(shape, v.value("geo")); + assertEquals(point, v.value("location")); + assertEquals(shape, v.value("boundary")); assertEquals(10.12345, v.value("precise").doubleValue(), 0.000001); assertCount(3, v.properties("any")); for (TitanVertexProperty prop : v.query().labels("any").properties()) { diff --git a/titan-test/src/main/java/com/thinkaurelius/titan/graphdb/TitanIndexTest.java b/titan-test/src/main/java/com/thinkaurelius/titan/graphdb/TitanIndexTest.java index b27d9bb359..ec64118e33 100644 --- a/titan-test/src/main/java/com/thinkaurelius/titan/graphdb/TitanIndexTest.java +++ b/titan-test/src/main/java/com/thinkaurelius/titan/graphdb/TitanIndexTest.java @@ -63,6 +63,7 @@ import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import static com.thinkaurelius.titan.graphdb.TitanGraphTest.evaluateQuery; import static com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration.*; @@ -199,6 +200,10 @@ public void testIndexing() { createExternalVertexIndex(location, INDEX); createExternalEdgeIndex(location, INDEX); + PropertyKey boundary = makeKey("boundary", Geoshape.class); + mgmt.addIndexKey(getExternalIndex(Vertex.class,INDEX),boundary, Parameter.of("mapping", Mapping.PREFIX_TREE), Parameter.of("index-geo-dist-error-pct", 0.0025)); + mgmt.addIndexKey(getExternalIndex(Edge.class,INDEX),boundary, Parameter.of("mapping", Mapping.PREFIX_TREE), Parameter.of("index-geo-dist-error-pct", 0.0025)); + PropertyKey time = makeKey("time", Long.class); createExternalVertexIndex(time, INDEX); createExternalEdgeIndex(time, INDEX); @@ -212,7 +217,7 @@ public void testIndexing() { createExternalEdgeIndex(group, INDEX); PropertyKey id = makeVertexIndexedKey("uid", Integer.class); - EdgeLabel knows = ((StandardEdgeLabelMaker) mgmt.makeEdgeLabel("knows")).sortKey(time).signature(location).make(); + EdgeLabel knows = ((StandardEdgeLabelMaker) mgmt.makeEdgeLabel("knows")).sortKey(time).signature(location,boundary).make(); finishSchema(); clopen(); @@ -231,13 +236,22 @@ public void testIndexing() { v.property(VertexProperty.Cardinality.single, "time", i); offset = (i % 2 == 0 ? 1 : -1) * (i * 50.0 / numV); v.property(VertexProperty.Cardinality.single, "location", Geoshape.point(0.0 + offset, 0.0 + offset)); - + if (i % 2 == 0) { + v.property(VertexProperty.Cardinality.single, "boundary", Geoshape.line(offset-0.1, offset-0.1, offset+0.1, offset-0.1, offset+0.1, offset+0.1, offset-0.1, offset+0.1)); + } else { + v.property(VertexProperty.Cardinality.single, "boundary", Geoshape.polygon(offset-0.1, offset-0.1, offset+0.1, offset-0.1, offset+0.1, offset+0.1, offset-0.1, offset+0.1, offset-0.1, offset-0.1)); + } Edge e = v.addEdge("knows", getVertex("uid", Math.max(0, i - 1))); e.property("text", "Vertex " + words[i % words.length]); e.property("time", i); e.property("category", i % numCategories); e.property("group", i % numGroups); e.property("location", Geoshape.point(0.0 + offset, 0.0 + offset)); + if (i % 2 == 0) { + e.property("boundary", Geoshape.line(offset-0.1, offset-0.1, offset+0.1, offset-0.1, offset+0.1, offset+0.1, offset-0.1, offset+0.1)); + } else { + e.property("boundary", Geoshape.polygon(offset-0.1, offset-0.1, offset+0.1, offset-0.1, offset+0.1, offset+0.1, offset-0.1, offset+0.1, offset-0.1, offset-0.1)); + } } for (int i = 0; i < words.length; i++) { @@ -266,11 +280,8 @@ public void testIndexing() { assertCount(i, tx.query().has("time", Cmp.GREATER_THAN_EQUAL, i).has("time", Cmp.LESS_THAN, i + i).edges()); } - for (int i = 0; i < numV; i += 10) { - offset = (i * 50.0 / originalNumV); - distance = Geoshape.point(0.0, 0.0).getPoint().distance(Geoshape.point(offset, offset).getPoint()) + 20; - assertCount(i + 1, tx.query().has("location", Geo.WITHIN, Geoshape.circle(0.0, 0.0, distance)).vertices()); - assertCount(i + 1, tx.query().has("location", Geo.WITHIN, Geoshape.circle(0.0, 0.0, distance)).edges()); + for (int i = 0; i < numV; i += 5) { + testGeo(i, originalNumV, numV, "location", "boundary"); } //Queries combining mixed and composite indexes @@ -281,6 +292,7 @@ public void testIndexing() { offset = (19 * 50.0 / originalNumV); distance = Geoshape.point(0.0, 0.0).getPoint().distance(Geoshape.point(offset, offset).getPoint()) + 20; assertCount(5, tx.query().has("location", Geo.INTERSECT, Geoshape.circle(0.0, 0.0, distance)).has("text", Text.CONTAINS, words[0]).vertices()); + assertCount(5, tx.query().has("boundary", Geo.INTERSECT, Geoshape.circle(0.0, 0.0, distance)).has("text", Text.CONTAINS, words[0]).vertices()); assertCount(numV, tx.query().vertices()); assertCount(numV, tx.query().edges()); @@ -319,11 +331,8 @@ public void testIndexing() { assertCount(i, tx.query().has("time", Cmp.GREATER_THAN_EQUAL, i).has("time", Cmp.LESS_THAN, i + i).edges()); } - for (int i = 0; i < numV; i += 10) { - offset = (i * 50.0 / originalNumV); - distance = Geoshape.point(0.0, 0.0).getPoint().distance(Geoshape.point(offset, offset).getPoint()) + 20; - assertCount(i + 1, tx.query().has("location", Geo.WITHIN, Geoshape.circle(0.0, 0.0, distance)).vertices()); - assertCount(i + 1, tx.query().has("location", Geo.WITHIN, Geoshape.circle(0.0, 0.0, distance)).edges()); + for (int i = 0; i < numV; i += 5) { + testGeo(i, originalNumV, numV, "location", "boundary"); } //Queries combining mixed and composite indexes @@ -334,6 +343,7 @@ public void testIndexing() { offset = (19 * 50.0 / originalNumV); distance = Geoshape.point(0.0, 0.0).getPoint().distance(Geoshape.point(offset, offset).getPoint()) + 20; assertCount(5, tx.query().has("location", Geo.INTERSECT, Geoshape.circle(0.0, 0.0, distance)).has("text", Text.CONTAINS, words[0]).vertices()); + assertCount(5, tx.query().has("boundary", Geo.INTERSECT, Geoshape.circle(0.0, 0.0, distance)).has("text", Text.CONTAINS, words[0]).vertices()); assertCount(numV, tx.query().vertices()); assertCount(numV, tx.query().edges()); @@ -358,17 +368,15 @@ public void testIndexing() { assertCount(i, tx.query().has("time", Cmp.GREATER_THAN_EQUAL, i).has("time", Cmp.LESS_THAN, i + i).edges()); } - for (int i = 0; i < numV; i += 10) { - offset = (i * 50.0 / originalNumV); - distance = Geoshape.point(0.0, 0.0).getPoint().distance(Geoshape.point(offset, offset).getPoint()) + 20; - assertCount(i + 1, tx.query().has("location", Geo.WITHIN, Geoshape.circle(0.0, 0.0, distance)).vertices()); - assertCount(i + 1, tx.query().has("location", Geo.WITHIN, Geoshape.circle(0.0, 0.0, distance)).edges()); + for (int i = 0; i < numV; i += 5) { + testGeo(i, originalNumV, numV, "location", "boundary"); } assertCount(5, tx.query().has("time", Cmp.GREATER_THAN_EQUAL, 10).has("time", Cmp.LESS_THAN, 30).has("text", Text.CONTAINS, words[0]).vertices()); offset = (19 * 50.0 / originalNumV); distance = Geoshape.point(0.0, 0.0).getPoint().distance(Geoshape.point(offset, offset).getPoint()) + 20; assertCount(5, tx.query().has("location", Geo.INTERSECT, Geoshape.circle(0.0, 0.0, distance)).has("text", Text.CONTAINS, words[0]).vertices()); + assertCount(5, tx.query().has("boundary", Geo.INTERSECT, Geoshape.circle(0.0, 0.0, distance)).has("text", Text.CONTAINS, words[0]).vertices()); assertCount(numV, tx.query().vertices()); assertCount(numV, tx.query().edges()); @@ -1667,15 +1675,15 @@ private void testIndexing(Cardinality cardinality) { PropertyKey intProperty = mgmt.makePropertyKey("age").dataType(Integer.class).cardinality(cardinality).make(); PropertyKey longProperty = mgmt.makePropertyKey("long").dataType(Long.class).cardinality(cardinality).make(); PropertyKey uuidProperty = mgmt.makePropertyKey("uuid").dataType(UUID.class).cardinality(cardinality).make(); - PropertyKey geoProperty = mgmt.makePropertyKey("geo").dataType(Geoshape.class).cardinality(cardinality).make(); - mgmt.buildIndex("collectionIndex", Vertex.class).addKey(stringProperty, getStringMapping()).addKey(intProperty).addKey(longProperty).addKey(uuidProperty).addKey(geoProperty).buildMixedIndex(INDEX); + PropertyKey geopointProperty = mgmt.makePropertyKey("geopoint").dataType(Geoshape.class).cardinality(cardinality).make(); + mgmt.buildIndex("collectionIndex", Vertex.class).addKey(stringProperty, getStringMapping()).addKey(intProperty).addKey(longProperty).addKey(uuidProperty).addKey(geopointProperty).buildMixedIndex(INDEX); finishSchema(); testCollection(cardinality, "name", "Totoro", "Hiro"); testCollection(cardinality, "age", 1, 2); testCollection(cardinality, "long", 1L, 2L); testCollection(cardinality, "uuid", UUID.randomUUID(), UUID.randomUUID()); - testCollection(cardinality, "geo", Geoshape.point(1.0, 1.0), Geoshape.point(2.0, 2.0)); + testCollection(cardinality, "geopoint", Geoshape.point(1.0, 1.0), Geoshape.point(2.0, 2.0)); } else { try { PropertyKey stringProperty = mgmt.makePropertyKey("name").dataType(String.class).cardinality(cardinality).make(); @@ -1761,4 +1769,59 @@ private void testCollection(Cardinality cardinality, String property, Object val } + private void testGeo(int i, int origNumV, int numV, String geoPointProperty, String geoShapeProperty) { + double offset = (i * 50.0 / origNumV); + double bufferKm = 20; + double distance = Geoshape.point(0.0, 0.0).getPoint().distance(Geoshape.point(offset, offset).getPoint()) + bufferKm; + + assertCount(i + 1, tx.query().has(geoPointProperty, Geo.WITHIN, Geoshape.circle(0.0, 0.0, distance)).vertices()); + assertCount(i + 1, tx.query().has(geoPointProperty, Geo.WITHIN, Geoshape.circle(0.0, 0.0, distance)).edges()); + assertCount(i + 1, tx.query().has(geoPointProperty, Geo.INTERSECT, Geoshape.circle(0.0, 0.0, distance)).vertices()); + assertCount(i + 1, tx.query().has(geoPointProperty, Geo.INTERSECT, Geoshape.circle(0.0, 0.0, distance)).edges()); + assertCount(numV-(i + 1), tx.query().has(geoPointProperty, Geo.DISJOINT, Geoshape.circle(0.0, 0.0, distance)).vertices()); + assertCount(numV-(i + 1), tx.query().has(geoPointProperty, Geo.DISJOINT, Geoshape.circle(0.0, 0.0, distance)).edges()); + assertCount(i + 1, tx.query().has(geoShapeProperty, Geo.INTERSECT, Geoshape.circle(0.0, 0.0, distance)).vertices()); + assertCount(i + 1, tx.query().has(geoShapeProperty, Geo.INTERSECT, Geoshape.circle(0.0, 0.0, distance)).edges()); + if (i > 0) { + assertCount(i, tx.query().has(geoShapeProperty, Geo.WITHIN, Geoshape.circle(0.0, 0.0, distance-bufferKm)).vertices()); + assertCount(i, tx.query().has(geoShapeProperty, Geo.WITHIN, Geoshape.circle(0.0, 0.0, distance-bufferKm)).edges()); + } + assertCount(numV-(i + 1), tx.query().has(geoShapeProperty, Geo.DISJOINT, Geoshape.circle(0.0, 0.0, distance)).vertices()); + assertCount(numV-(i + 1), tx.query().has(geoShapeProperty, Geo.DISJOINT, Geoshape.circle(0.0, 0.0, distance)).edges()); + + assertCount(i % 2, tx.query().has(geoShapeProperty, Geo.CONTAINS, Geoshape.point(-offset,-offset)).vertices()); + assertCount(i % 2, tx.query().has(geoShapeProperty, Geo.CONTAINS, Geoshape.point(-offset,-offset)).edges()); + + double buffer = bufferKm/111.; + double min = -Math.abs(offset); + double max = Math.abs(offset); + Geoshape bufferedBox = Geoshape.box(min-buffer, min-buffer, max+buffer, max+buffer); + assertCount(i + 1, tx.query().has(geoPointProperty, Geo.WITHIN, bufferedBox).vertices()); + assertCount(i + 1, tx.query().has(geoPointProperty, Geo.WITHIN, bufferedBox).edges()); + assertCount(i + 1, tx.query().has(geoPointProperty, Geo.INTERSECT, bufferedBox).vertices()); + assertCount(i + 1, tx.query().has(geoPointProperty, Geo.INTERSECT, bufferedBox).edges()); + assertCount(numV-(i + 1), tx.query().has(geoPointProperty, Geo.DISJOINT, bufferedBox).vertices()); + assertCount(numV-(i + 1), tx.query().has(geoPointProperty, Geo.DISJOINT, bufferedBox).edges()); + if (i > 0) { + Geoshape exactBox = Geoshape.box(min, min, max, max); + assertCount(i, tx.query().has(geoShapeProperty, Geo.WITHIN, exactBox).vertices()); + assertCount(i, tx.query().has(geoShapeProperty, Geo.WITHIN, exactBox).edges()); + } + assertCount(i + 1, tx.query().has(geoShapeProperty, Geo.INTERSECT, bufferedBox).vertices()); + assertCount(i + 1, tx.query().has(geoShapeProperty, Geo.INTERSECT, bufferedBox).edges()); + assertCount(numV-(i + 1), tx.query().has(geoShapeProperty, Geo.DISJOINT, bufferedBox).vertices()); + assertCount(numV-(i + 1), tx.query().has(geoShapeProperty, Geo.DISJOINT, bufferedBox).edges()); + + Geoshape bufferedPoly = Geoshape.polygon(min-buffer, min-buffer, max+buffer, min-buffer, max+buffer, max+buffer, min-buffer, max+buffer, min-buffer, min-buffer); + if (i > 0) { + Geoshape exactPoly = Geoshape.polygon(min, min, max, min, max, max, min, max, min, min); + assertCount(i, tx.query().has(geoShapeProperty, Geo.WITHIN, exactPoly).vertices()); + assertCount(i, tx.query().has(geoShapeProperty, Geo.WITHIN, exactPoly).edges()); + } + assertCount(i + 1, tx.query().has(geoShapeProperty, Geo.INTERSECT, bufferedPoly).vertices()); + assertCount(i + 1, tx.query().has(geoShapeProperty, Geo.INTERSECT, bufferedPoly).edges()); + assertCount(numV-(i + 1), tx.query().has(geoShapeProperty, Geo.DISJOINT, bufferedPoly).vertices()); + assertCount(numV-(i + 1), tx.query().has(geoShapeProperty, Geo.DISJOINT, bufferedPoly).edges()); + } + } diff --git a/titan-test/src/main/java/com/thinkaurelius/titan/graphdb/TitanIoTest.java b/titan-test/src/main/java/com/thinkaurelius/titan/graphdb/TitanIoTest.java index 4259aa1c1c..a0eb7246dc 100644 --- a/titan-test/src/main/java/com/thinkaurelius/titan/graphdb/TitanIoTest.java +++ b/titan-test/src/main/java/com/thinkaurelius/titan/graphdb/TitanIoTest.java @@ -1,14 +1,27 @@ package com.thinkaurelius.titan.graphdb; +import com.thinkaurelius.titan.core.TitanTransaction; +import com.thinkaurelius.titan.core.attribute.Geoshape; +import com.thinkaurelius.titan.core.schema.TitanManagement; import com.thinkaurelius.titan.example.GraphOfTheGodsFactory; +import com.vividsolutions.jts.geom.Coordinate; +import com.vividsolutions.jts.geom.GeometryFactory; +import com.vividsolutions.jts.geom.LineString; +import com.vividsolutions.jts.geom.Polygon; + import org.apache.tinkerpop.gremlin.structure.io.GraphReader; import org.apache.tinkerpop.gremlin.structure.io.GraphWriter; import org.apache.tinkerpop.gremlin.structure.io.IoCore; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.Before; import org.junit.Test; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.util.function.Function; /** * Tests Titan specific serialization classes not covered by the TinkerPop suite. @@ -17,9 +30,41 @@ */ public abstract class TitanIoTest extends TitanGraphBaseTest { - @Test - public void testGeoShapeSerializationReadWriteAsGraphSONEmbedded() throws Exception { + private GeometryFactory gf; + + @Before + public void setup() { + gf = new GeometryFactory(); GraphOfTheGodsFactory.loadWithoutMixedIndex(graph, true); + TitanManagement mgmt = graph.openManagement(); + mgmt.makePropertyKey("shape").dataType(Geoshape.class).make(); + mgmt.commit(); + } + + @Test + public void testSerializationReadWriteAsGraphSONEmbedded() throws Exception { + testSerializationReadWriteAsGraphSONEmbedded(null); + testSerializationReadWriteAsGraphSONEmbedded(makeLine); + testSerializationReadWriteAsGraphSONEmbedded(makePoly); + testSerializationReadWriteAsGraphSONEmbedded(makeMultiPoint); + testSerializationReadWriteAsGraphSONEmbedded(makeMultiLine); + testSerializationReadWriteAsGraphSONEmbedded(makeMultiPolygon); + } + + @Test + public void testSerializationReadWriteAsGryo() throws Exception { + testSerializationReadWriteAsGryo(null); + testSerializationReadWriteAsGryo(makeLine); + testSerializationReadWriteAsGryo(makePoly); + testSerializationReadWriteAsGryo(makeMultiPoint); + testSerializationReadWriteAsGryo(makeMultiLine); + testSerializationReadWriteAsGryo(makeMultiPolygon); + } + + public void testSerializationReadWriteAsGraphSONEmbedded(Function makeGeoshape) throws Exception { + if (makeGeoshape != null) { + addGeoshape(makeGeoshape); + } GraphSONMapper m = graph.io(IoCore.graphson()).mapper().embedTypes(true).create(); GraphWriter writer = graph.io(IoCore.graphson()).writer().mapper(m).create(); FileOutputStream fos = new FileOutputStream("/tmp/test.json"); @@ -33,11 +78,15 @@ public void testGeoShapeSerializationReadWriteAsGraphSONEmbedded() throws Except reader.readGraph(fis, graph); TitanIndexTest.assertGraphOfTheGods(graph); + if (makeGeoshape != null) { + assertGeoshape(makeGeoshape); + } } - @Test - public void testGeoShapeSerializationReadWriteAsGryo() throws Exception { - GraphOfTheGodsFactory.loadWithoutMixedIndex(graph, true); + private void testSerializationReadWriteAsGryo(Function makeGeoshape) throws Exception { + if (makeGeoshape != null) { + addGeoshape(makeGeoshape); + } graph.io(IoCore.gryo()).writeGraph("/tmp/test.kryo"); clearGraph(config); @@ -46,5 +95,62 @@ public void testGeoShapeSerializationReadWriteAsGryo() throws Exception { graph.io(IoCore.gryo()).readGraph("/tmp/test.kryo"); TitanIndexTest.assertGraphOfTheGods(graph); + if (makeGeoshape != null) { + assertGeoshape(makeGeoshape); + } + } + + private void addGeoshape(Function makeGeoshape) { + TitanTransaction tx = graph.newTransaction(); + graph.traversal().E().has("place").toList().stream().forEach(e-> { + Geoshape place = (Geoshape) e.property("place").value(); + e.property("shape", makeGeoshape.apply(place)); + }); + tx.commit(); + } + + private void assertGeoshape(Function makeGeoshape) { + graph.traversal().E().has("place").toList().stream().forEach(e-> { + assertTrue(e.property("shape").isPresent()); + Geoshape place = (Geoshape) e.property("place").value(); + Geoshape expected = makeGeoshape.apply(place); + Geoshape actual = (Geoshape) e.property("shape").value(); + assertEquals(expected, actual); + }); } + + private Function makePoly = place -> { + double x = Math.floor(place.getPoint().getLongitude()); + double y = Math.floor(place.getPoint().getLatitude()); + return Geoshape.polygon(x,y,x,y+1,x+1,y+1,x+1,y,x,y,x,y); + }; + + private Function makeLine = place -> { + double x = Math.floor(place.getPoint().getLongitude()); + double y = Math.floor(place.getPoint().getLatitude()); + return Geoshape.line(x,y,x,y+1,x+1,y+1,x+1,y); + }; + + private Function makeMultiPoint = place -> { + double x = Math.floor(place.getPoint().getLongitude()); + double y = Math.floor(place.getPoint().getLatitude()); + return Geoshape.geoshape(gf.createMultiPoint(new Coordinate[] {new Coordinate(x,y), new Coordinate(x+1,y+1)})); + }; + + private Function makeMultiLine = place -> { + double x = Math.floor(place.getPoint().getLongitude()); + double y = Math.floor(place.getPoint().getLatitude()); + return Geoshape.geoshape(gf.createMultiLineString(new LineString[] { + gf.createLineString(new Coordinate[] {new Coordinate(x,y), new Coordinate(x+1,y+1)}), + gf.createLineString(new Coordinate[] {new Coordinate(x-1,y-1), new Coordinate(x,y)})})); + }; + + private Function makeMultiPolygon = place -> { + double x = Math.floor(place.getPoint().getLongitude()); + double y = Math.floor(place.getPoint().getLatitude()); + return Geoshape.geoshape(gf.createMultiPolygon(new Polygon[] { + gf.createPolygon(new Coordinate[] {new Coordinate(x,y), new Coordinate(x+1,y), new Coordinate(x+1,y+1), new Coordinate(x,y)}), + gf.createPolygon(new Coordinate[] {new Coordinate(x+2,y+2), new Coordinate(x+2,y+3), new Coordinate(x+3,y+3), new Coordinate(x+2,y+2)})})); + }; + } diff --git a/titan-test/src/test/java/com/thinkaurelius/titan/graphdb/attribute/GeoshapeTest.java b/titan-test/src/test/java/com/thinkaurelius/titan/graphdb/attribute/GeoshapeTest.java index b3e9f55467..eb3a98c8df 100644 --- a/titan-test/src/test/java/com/thinkaurelius/titan/graphdb/attribute/GeoshapeTest.java +++ b/titan-test/src/test/java/com/thinkaurelius/titan/graphdb/attribute/GeoshapeTest.java @@ -1,8 +1,15 @@ package com.thinkaurelius.titan.graphdb.attribute; import com.thinkaurelius.titan.core.attribute.Geoshape; +import com.vividsolutions.jts.geom.Coordinate; +import com.vividsolutions.jts.geom.GeometryFactory; +import com.vividsolutions.jts.geom.LineString; +import com.vividsolutions.jts.geom.LinearRing; +import com.vividsolutions.jts.geom.Polygon; + import org.apache.tinkerpop.shaded.jackson.databind.ObjectMapper; import org.apache.tinkerpop.shaded.jackson.databind.module.SimpleModule; +import org.junit.Before; import org.junit.Test; import java.util.Arrays; @@ -18,6 +25,13 @@ public class GeoshapeTest { + private GeometryFactory gf; + + @Before + public void setup() { + gf = new GeometryFactory(); + } + @Test public void testDistance() { Geoshape p1 = Geoshape.point(37.759, -122.536); @@ -25,20 +39,22 @@ public void testDistance() { double distance = 1496; assertEquals(distance,p1.getPoint().distance(p2.getPoint()),5.0); - - p1 = Geoshape.point(0.0,0.0); - p2 = Geoshape.point(10.0,10.0); - //System.out.println(p1.getPoint().distance(p2.getPoint())); } @Test public void testIntersection() { for (int i=0;i<50;i++) { Geoshape point = Geoshape.point(i,i); + Geoshape line = Geoshape.line(i-1,i-1,i,i,i+1,i+1); + Geoshape polygon = Geoshape.polygon(i-1,i-1,i,i-1,i+1,i-i,i+1,i+1,i-1,i+1,i-1,i-1); Geoshape circle = Geoshape.circle(0.0,0.0,point.getPoint().distance(Geoshape.point(0,0).getPoint())+10); assertTrue(circle.intersect(point)); assertTrue(point.intersect(circle)); assertTrue(circle.intersect(circle)); + assertTrue(polygon.intersect(circle)); + assertTrue(circle.intersect(polygon)); + assertTrue(line.intersect(circle)); + assertTrue(circle.intersect(line)); } } @@ -46,14 +62,28 @@ public void testIntersection() { public void testEquality() { Geoshape c = Geoshape.circle(10.0,12.5,100); Geoshape b = Geoshape.box(20.0, 22.5, 40.5, 60.5); + Geoshape l = Geoshape.line(10.5,20.5,10.5,22.5,12.5,22.5); + Geoshape p = Geoshape.polygon(10.5,20.5,8.0,21.75,10.5,22.5,11.75,25.0,12.5,22.5,15.0,21.0,12.5,20.5,11.75,18.0,10.5,20.5); assertEquals(Geoshape.circle(10.0,12.5,100),c); assertEquals(Geoshape.box(20.0,22.5,40.5,60.5),b); + assertEquals(Geoshape.line(10.5,20.5,10.5,22.5,12.5,22.5),l); + assertEquals(Geoshape.polygon(10.5,20.5,8.0,21.75,10.5,22.5,11.75,25.0,12.5,22.5,15.0,21.0,12.5,20.5,11.75,18.0,10.5,20.5),p); assertEquals(Geoshape.circle(10.0,12.5,100).hashCode(),c.hashCode()); assertEquals(Geoshape.box(20.0,22.5,40.5,60.5).hashCode(),b.hashCode()); + assertEquals(Geoshape.line(10.5,20.5,10.5,22.5,12.5,22.5).hashCode(),l.hashCode()); + assertEquals(Geoshape.polygon(10.5,20.5,8.0,21.75,10.5,22.5,11.75,25.0,12.5,22.5,15.0,21.0,12.5,20.5,11.75,18.0,10.5,20.5).hashCode(),p.hashCode()); assertNotSame(c.hashCode(),b.hashCode()); + assertNotSame(c.hashCode(),l.hashCode()); + assertNotSame(c.hashCode(),p.hashCode()); + assertNotSame(b.hashCode(),l.hashCode()); + assertNotSame(b.hashCode(),p.hashCode()); + assertNotSame(l.hashCode(),p.hashCode()); assertNotSame(c,b); - System.out.println(c); - System.out.println(b); + assertNotSame(c,l); + assertNotSame(c,p); + assertNotSame(b,l); + assertNotSame(b,p); + assertNotSame(l,p); } @@ -140,7 +170,7 @@ public void testGeoJsonCircleMissingRadius() throws IOException { } @Test - public void testGeoJsonPolygon() throws IOException { + public void testGeoJsonBox() throws IOException { Geoshape.GeoshapeSerializer s = new Geoshape.GeoshapeSerializer(); Map json = new ObjectMapper().readValue("{\n" + " \"type\": \"Feature\",\n" + @@ -169,7 +199,7 @@ public void testGeoJsonPolygon() throws IOException { } @Test(expected = IllegalArgumentException.class) - public void testGeoJsonPolygonNotBox1() throws IOException { + public void testGeoJsonInvalidBox1() throws IOException { Geoshape.GeoshapeSerializer s = new Geoshape.GeoshapeSerializer(); Map json = new ObjectMapper().readValue("{\n" + " \"type\": \"Feature\",\n" + @@ -186,7 +216,7 @@ public void testGeoJsonPolygonNotBox1() throws IOException { } @Test(expected = IllegalArgumentException.class) - public void testGeoJsonPolygonNotBox2() throws IOException { + public void testGeoJsonInvalidBox2() throws IOException { Geoshape.GeoshapeSerializer s = new Geoshape.GeoshapeSerializer(); Map json = new ObjectMapper().readValue("{\n" + " \"type\": \"Feature\",\n" + @@ -202,6 +232,81 @@ public void testGeoJsonPolygonNotBox2() throws IOException { } + @Test + public void testGeoJsonLine() throws IOException { + Geoshape.GeoshapeSerializer s = new Geoshape.GeoshapeSerializer(); + Map json = new ObjectMapper().readValue("{\n" + + " \"type\": \"Feature\",\n" + + " \"geometry\": {\n" + + " \"type\": \"LineString\",\n" + + " \"coordinates\": [[20.5, 10.5],[22.5, 10.5],[22.5, 12.5]]\n" + + " },\n" + + " \"properties\": {\n" + + " \"name\": \"Dinagat Islands\"\n" + + " }\n" + + "}", HashMap.class); + assertEquals(Geoshape.line(10.5,20.5,10.5,22.5,12.5,22.5), s.convert(json)); + } + + @Test + public void testGeoJsonPolygon() throws IOException { + Geoshape.GeoshapeSerializer s = new Geoshape.GeoshapeSerializer(); + Map json = new ObjectMapper().readValue("{\n" + + " \"type\": \"Feature\",\n" + + " \"geometry\": {\n" + + " \"type\": \"Polygon\",\n" + + " \"coordinates\": [[[20.5,10.5],[21.75,8.0],[22.5,10.5],[25.0,11.75],[22.5,12.5],[21.0,15.0],[20.5,12.5],[18.0,11.75],[20.5,10.5]]]\n" + + " }" + + "}", HashMap.class); + assertEquals(Geoshape.polygon(10.5,20.5,8.0,21.75,10.5,22.5,11.75,25.0,12.5,22.5,15.0,21.0,12.5,20.5,11.75,18.0,10.5,20.5), s.convert(json)); + } + + @Test + public void testGeoJsonMultiPoint() throws IOException { + Geoshape.GeoshapeSerializer s = new Geoshape.GeoshapeSerializer(); + Map json = new ObjectMapper().readValue("{\n" + + " \"type\": \"Feature\",\n" + + " \"geometry\": {\n" + + " \"type\": \"MultiPoint\",\n" + + " \"coordinates\": [[100.0, 0.0],[101.0, 1.0]]\n" + + " }" + + "}", HashMap.class); + assertEquals(Geoshape.geoshape(gf.createMultiPoint(new Coordinate[] {new Coordinate(100,0), new Coordinate(101,1)})), s.convert(json)); + } + + @Test + public void testGeoJsonMultiLineString() throws IOException { + Geoshape.GeoshapeSerializer s = new Geoshape.GeoshapeSerializer(); + Map json = new ObjectMapper().readValue("{\n" + + " \"type\": \"Feature\",\n" + + " \"geometry\": {\n" + + " \"type\": \"MultiLineString\",\n" + + " \"coordinates\": [[[100.0,0.0],[101.0, 1.0]],[[102.0,2.0],[103.0,3.0]]]\n" + + " }" + + "}", HashMap.class); + assertEquals(Geoshape.geoshape(gf.createMultiLineString(new LineString[] { + gf.createLineString(new Coordinate[] {new Coordinate(100,0), new Coordinate(101,1)}), + gf.createLineString(new Coordinate[] {new Coordinate(102,2), new Coordinate(103,3)})})), s.convert(json)); + } + + @Test + public void testGeoJsonMultiPolygon() throws IOException { + Geoshape.GeoshapeSerializer s = new Geoshape.GeoshapeSerializer(); + Map json = new ObjectMapper().readValue("{\n" + + " \"type\": \"Feature\",\n" + + " \"geometry\": {\n" + + " \"type\": \"MultiPolygon\",\n" + + " \"coordinates\": [[[[102.0,2.0],[103.0,2.0],[103.0,3.0],[102.0,3.0],[102.0,2.0]]]," + + "[[[100.0,0.0],[101.0,0.0],[101.0,1.0],[100.0,1.0],[100.0,0.0]],[[100.2,0.2],[100.8,0.2],[100.8,0.8],[100.2,0.8],[100.2,0.2]]]]\n" + + " }" + + "}", HashMap.class); + assertEquals(Geoshape.geoshape(gf.createMultiPolygon(new Polygon[] { + gf.createPolygon(new Coordinate[] {new Coordinate(102,2), new Coordinate(103,2), new Coordinate(103,3), new Coordinate(102,3), new Coordinate(102,2)}), + gf.createPolygon(gf.createLinearRing(new Coordinate[] {new Coordinate(100,0), new Coordinate(101,0), new Coordinate(101,1), new Coordinate(100,1), new Coordinate(100,0)}), + new LinearRing[] { gf.createLinearRing(new Coordinate[] {new Coordinate(100.2,0.2), new Coordinate(100.8,0.2), new Coordinate(100.8,0.8), new Coordinate(100.2,0.8), new Coordinate(100.2,0.2) + })})})), s.convert(json)); + } + @Test public void testGeoJsonGeometry() throws IOException { Geoshape.GeoshapeSerializer s = new Geoshape.GeoshapeSerializer(); @@ -210,7 +315,6 @@ public void testGeoJsonGeometry() throws IOException { " \"coordinates\": [20.5, 10.5]\n" + "}", HashMap.class); assertEquals(Geoshape.point(10.5, 20.5), s.convert(json)); - } @@ -221,9 +325,23 @@ public void testGeoJsonSerialization() throws IOException { final ObjectMapper om = new ObjectMapper(); om.registerModule(module); assertEquals("{\"type\":\"Point\",\"coordinates\":[20.5,10.5]}", om.writeValueAsString(Geoshape.point(10.5, 20.5))); - assertEquals("{\"type\":\"Box\",\"coordinates\":[[20.5,10.5],[22.5,10.5],[22.5,12.5],[20.5,12.5]]}", om.writeValueAsString(Geoshape.box(10.5, 20.5, 12.5, 22.5))); - assertEquals("{\"type\":\"Circle\",\"radius\":30.5,\"coordinates\":[20.5,10.5]}", om.writeValueAsString(Geoshape.circle(10.5, 20.5, 30.5))); - + assertEquals("{\"type\":\"Polygon\",\"coordinates\": [[[20.5,10.5],[20.5,12.5],[22.5,12.5],[22.5,10.5],[20.5,10.5]]]}", om.writeValueAsString(Geoshape.box(10.5, 20.5, 12.5, 22.5))); + assertEquals("{\"type\":\"Circle\",\"coordinates\":[20.5,10.5],\"radius\":30.5,\"properties\":{\"radius_units\":\"km\"}}", om.writeValueAsString(Geoshape.circle(10.5, 20.5, 30.5))); + assertEquals("{\"type\":\"LineString\",\"coordinates\":[[20.5,10.5],[22.5,10.5],[22.5,12.5]]}", om.writeValueAsString(Geoshape.line(10.5,20.5,10.5,22.5,12.5,22.5))); + assertEquals("{\"type\":\"Polygon\",\"coordinates\":[[[20.5,10.5],[21.75,8],[22.5,10.5],[25,11.75],[22.5,12.5],[21,15],[20.5,12.5],[18,11.75],[20.5,10.5]]]}", + om.writeValueAsString(Geoshape.polygon(10.5,20.5,8.0,21.75,10.5,22.5,11.75,25.0,12.5,22.5,15.0,21.0,12.5,20.5,11.75,18.0,10.5,20.5))); + assertEquals("{\"type\":\"MultiPoint\",\"coordinates\":[[100,0],[101,1]]}", + om.writeValueAsString(Geoshape.geoshape(Geoshape.CTX.makeShape(gf.createMultiPoint(new Coordinate[] {new Coordinate(100,0), new Coordinate(101,1)}))))); + assertEquals("{\"type\":\"MultiLineString\",\"coordinates\":[[[100,0],[101,1]],[[102,2],[103,3]]]}", + om.writeValueAsString(Geoshape.geoshape(Geoshape.CTX.makeShape(gf.createMultiLineString(new LineString[] { + gf.createLineString(new Coordinate[] {new Coordinate(100,0), new Coordinate(101,1)}), + gf.createLineString(new Coordinate[] {new Coordinate(102,2), new Coordinate(103,3)})}))))); + assertEquals("{\"type\":\"MultiPolygon\",\"coordinates\":[[[[102,2],[103,2],[103,3],[102,3],[102,2]]],[[[100,0],[101,0],[101,1],[100,1],[100,0]],[[100.2,0.2],[100.8,0.2],[100.8,0.8],[100.2,0.8],[100.2,0.2]]]]}", + om.writeValueAsString(Geoshape.geoshape(Geoshape.CTX.makeShape(gf.createMultiPolygon(new Polygon[] { + gf.createPolygon(new Coordinate[] {new Coordinate(102,2), new Coordinate(103,2), new Coordinate(103,3), new Coordinate(102,3), new Coordinate(102,2)}), + gf.createPolygon(gf.createLinearRing(new Coordinate[] {new Coordinate(100,0), new Coordinate(101,0), new Coordinate(101,1), new Coordinate(100,1), new Coordinate(100,0)}), + new LinearRing[] { gf.createLinearRing(new Coordinate[] {new Coordinate(100.2,0.2), new Coordinate(100.8,0.2), + new Coordinate(100.8,0.8), new Coordinate(100.2,0.8), new Coordinate(100.2,0.2)})})}))))); } diff --git a/titan-test/src/test/java/com/thinkaurelius/titan/graphdb/serializer/SerializerTest.java b/titan-test/src/test/java/com/thinkaurelius/titan/graphdb/serializer/SerializerTest.java index 23df924856..0698858d86 100644 --- a/titan-test/src/test/java/com/thinkaurelius/titan/graphdb/serializer/SerializerTest.java +++ b/titan-test/src/test/java/com/thinkaurelius/titan/graphdb/serializer/SerializerTest.java @@ -397,6 +397,14 @@ public static final float randomGeoPoint() { return random.nextFloat()*180.0f-90.0f; } + public static final List randomGeoPoints(int n) { + List points = new ArrayList<>(); + for (int i=0; i TYPES = new HashMap() {{ put(Byte.class, new Factory() { @Override @@ -455,10 +463,19 @@ public Double newInstance() { put(Geoshape.class, new Factory() { @Override public Geoshape newInstance() { - if (random.nextDouble()>0.5) - return Geoshape.box(randomGeoPoint(),randomGeoPoint(),randomGeoPoint(),randomGeoPoint()); - else - return Geoshape.circle(randomGeoPoint(),randomGeoPoint(),random.nextInt(100)+1); + double alpha = random.nextDouble(); + double x0=randomGeoPoint(), y0=randomGeoPoint(), x1=randomGeoPoint(), y1=randomGeoPoint(); + if (alpha>0.75) { + double minx=Math.min(x0,x1), miny=Math.min(y0,y1); + double maxx=minx==x0? x1 : x0, maxy=miny==y0 ? y1 : y0; + return Geoshape.box(miny, minx, maxy, maxx); + } else if (alpha>0.5) { + return Geoshape.circle(y0,x0,random.nextInt(100)+1); + } else if (alpha>0.25) { + return Geoshape.line(y0,x0,y1,x0,y1,x1,y0,x1); + } else { + return Geoshape.polygon(y0,x0,y1,x0,y1,x1,y0,x1,y0,x0); + } } }); put(String.class, STRING_FACTORY);