From fc11a96f80959664e2d9493c994424f5f12d5ddd Mon Sep 17 00:00:00 2001 From: Jim Cahill Date: Wed, 9 Sep 2015 15:55:06 -0400 Subject: [PATCH 1/4] plugin exclusions in eclipse --- titan-core/pom.xml | 40 ++++++++++++++ .../titan/diskstorage/Backend.java | 1 + titan-es/pom.xml | 54 +++++++++++++++++++ titan-hadoop-parent/pom.xml | 31 +++++++++++ titan-hadoop-parent/titan-hadoop-core/pom.xml | 36 +++++++++++++ titan-hbase-parent/pom.xml | 31 +++++++++++ titan-hbase-parent/titan-hbase-094/pom.xml | 54 +++++++++++++++++++ titan-hbase-parent/titan-hbase-096/pom.xml | 54 +++++++++++++++++++ titan-hbase-parent/titan-hbase-098/pom.xml | 54 +++++++++++++++++++ titan-hbase-parent/titan-hbase-10/pom.xml | 54 +++++++++++++++++++ titan-test/pom.xml | 44 +++++++++++++++ 11 files changed, 453 insertions(+) diff --git a/titan-core/pom.xml b/titan-core/pom.xml index b34182de0a..17bad1c997 100644 --- a/titan-core/pom.xml +++ b/titan-core/pom.xml @@ -219,5 +219,45 @@ + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + + org.codehaus.gmaven + + + gmaven-plugin + + + [1.5,) + + + testCompile + compile + generateStubs + + generateTestStubs + + + + + + + + + + + + + diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/diskstorage/Backend.java b/titan-core/src/main/java/com/thinkaurelius/titan/diskstorage/Backend.java index 0d33670531..a89ca4f8d0 100644 --- a/titan-core/src/main/java/com/thinkaurelius/titan/diskstorage/Backend.java +++ b/titan-core/src/main/java/com/thinkaurelius/titan/diskstorage/Backend.java @@ -538,6 +538,7 @@ public BackendTransaction beginTransaction(TransactionConfiguration configuratio maxReadTime, indexTx, threadPool); } + @Override public synchronized void close() throws BackendException { if (!hasAttemptedClose) { hasAttemptedClose = true; diff --git a/titan-es/pom.xml b/titan-es/pom.xml index d60183b187..78e1c8a4ac 100644 --- a/titan-es/pom.xml +++ b/titan-es/pom.xml @@ -246,6 +246,60 @@ + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + + org.apache.maven.plugins + + + maven-dependency-plugin + + + [2.8,) + + + build-classpath + + + + + + + + + + org.apache.maven.plugins + + + maven-assembly-plugin + + + [2.4,) + + + single + + + + + + + + + + + + diff --git a/titan-hadoop-parent/pom.xml b/titan-hadoop-parent/pom.xml index 1098de9341..707e864d88 100644 --- a/titan-hadoop-parent/pom.xml +++ b/titan-hadoop-parent/pom.xml @@ -360,6 +360,37 @@ --> + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + + org.apache.maven.plugins + + + maven-dependency-plugin + + + [2.8,) + + + unpack + + + + + + + + + + diff --git a/titan-hadoop-parent/titan-hadoop-core/pom.xml b/titan-hadoop-parent/titan-hadoop-core/pom.xml index c069dcb315..807d6dc436 100644 --- a/titan-hadoop-parent/titan-hadoop-core/pom.xml +++ b/titan-hadoop-parent/titan-hadoop-core/pom.xml @@ -151,6 +151,42 @@ + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + + org.codehaus.gmaven + + + gmaven-plugin + + + [1.5,) + + + compile + generateStubs + + + + + + + + + + + + diff --git a/titan-hbase-parent/pom.xml b/titan-hbase-parent/pom.xml index 383c44f14e..47710691d7 100644 --- a/titan-hbase-parent/pom.xml +++ b/titan-hbase-parent/pom.xml @@ -193,6 +193,37 @@ + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + + org.apache.maven.plugins + + + maven-dependency-plugin + + + [2.8,) + + + unpack + + + + + + + + + + diff --git a/titan-hbase-parent/titan-hbase-094/pom.xml b/titan-hbase-parent/titan-hbase-094/pom.xml index f260d935f2..e2d9a25a97 100644 --- a/titan-hbase-parent/titan-hbase-094/pom.xml +++ b/titan-hbase-parent/titan-hbase-094/pom.xml @@ -223,6 +223,60 @@ + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + + org.apache.maven.plugins + + + maven-dependency-plugin + + + [2.8,) + + + build-classpath + + + + + + + + + + org.apache.maven.plugins + + + maven-assembly-plugin + + + [2.4,) + + + single + + + + + + + + + + + + diff --git a/titan-hbase-parent/titan-hbase-096/pom.xml b/titan-hbase-parent/titan-hbase-096/pom.xml index e76076f80a..c9583b1f6c 100644 --- a/titan-hbase-parent/titan-hbase-096/pom.xml +++ b/titan-hbase-parent/titan-hbase-096/pom.xml @@ -130,6 +130,60 @@ + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + + org.apache.maven.plugins + + + maven-dependency-plugin + + + [2.8,) + + + build-classpath + + + + + + + + + + org.apache.maven.plugins + + + maven-assembly-plugin + + + [2.4,) + + + single + + + + + + + + + + + + diff --git a/titan-hbase-parent/titan-hbase-098/pom.xml b/titan-hbase-parent/titan-hbase-098/pom.xml index 5423b90647..c00a3360e0 100644 --- a/titan-hbase-parent/titan-hbase-098/pom.xml +++ b/titan-hbase-parent/titan-hbase-098/pom.xml @@ -131,6 +131,60 @@ + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + + org.apache.maven.plugins + + + maven-dependency-plugin + + + [2.8,) + + + build-classpath + + + + + + + + + + org.apache.maven.plugins + + + maven-assembly-plugin + + + [2.4,) + + + single + + + + + + + + + + + + diff --git a/titan-hbase-parent/titan-hbase-10/pom.xml b/titan-hbase-parent/titan-hbase-10/pom.xml index cc9643e8be..dda756883e 100644 --- a/titan-hbase-parent/titan-hbase-10/pom.xml +++ b/titan-hbase-parent/titan-hbase-10/pom.xml @@ -130,6 +130,60 @@ + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + + org.apache.maven.plugins + + + maven-dependency-plugin + + + [2.8,) + + + build-classpath + + + + + + + + + + org.apache.maven.plugins + + + maven-assembly-plugin + + + [2.4,) + + + single + + + + + + + + + + + + diff --git a/titan-test/pom.xml b/titan-test/pom.xml index fb3284ef7f..f0b9a486b8 100644 --- a/titan-test/pom.xml +++ b/titan-test/pom.xml @@ -120,5 +120,49 @@ + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + + org.codehaus.gmavenplus + + + gmavenplus-plugin + + + [1.2,) + + + addSources + generateStubs + + testGenerateStubs + + addTestSources + compile + removeStubs + removeTestStubs + testCompile + + + + + + + + + + + + From 88137972b175dfb18a360438615da0699fbeba6a Mon Sep 17 00:00:00 2001 From: Jim Cahill Date: Fri, 11 Sep 2015 09:09:50 -0400 Subject: [PATCH 2/4] update es to 2.0.0-beta1 and lucene to 5.2.1 --- pom.xml | 4 +- .../diskstorage/es/ElasticSearchIndex.java | 105 ++++++++++-------- .../diskstorage/es/ElasticSearchSetup.java | 38 ++++--- .../es/ElasticSearchConfigTest.java | 10 +- .../com/thinkaurelius/titan/HBaseStatus.java | 2 +- .../titan/diskstorage/lucene/LuceneIndex.java | 9 +- .../diskstorage/lucene/LuceneExample.java | 6 +- 7 files changed, 96 insertions(+), 78 deletions(-) diff --git a/pom.xml b/pom.xml index 4874f58e31..500d10017e 100644 --- a/pom.xml +++ b/pom.xml @@ -83,8 +83,8 @@ version used by the titan-lucene module. When updating the ES version, also consider the version of Lucene, and vice-versa. --> - 4.10.4 - 1.5.1 + 5.2.1 + 2.0.0-beta1 2.0.0 1.7.0 1.6.2 diff --git a/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchIndex.java b/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchIndex.java index 16445dd31a..fe1c7b1d50 100644 --- a/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchIndex.java +++ b/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchIndex.java @@ -25,6 +25,7 @@ import com.thinkaurelius.titan.graphdb.query.TitanPredicate; import com.thinkaurelius.titan.graphdb.query.condition.*; import com.thinkaurelius.titan.util.system.IOUtils; + import org.apache.commons.lang.StringUtils; import org.elasticsearch.Version; import org.elasticsearch.action.ActionFuture; @@ -45,17 +46,17 @@ import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.*; -import org.elasticsearch.indices.IndexMissingException; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeBuilder; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -68,6 +69,8 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.time.Instant; import java.util.*; import java.util.concurrent.ExecutionException; @@ -219,7 +222,7 @@ private void checkForOrCreateIndex(Configuration config) { IndicesExistsResponse response = client.admin().indices().exists(new IndicesExistsRequest(indexName)).actionGet(); if (!response.isExists()) { - ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder(); + Settings.Builder settings = Settings.settingsBuilder(); ElasticSearchSetup.applySettingsFromTitanConf(settings, config, ES_CREATE_EXTRAS_NS); @@ -292,7 +295,7 @@ private ElasticSearchSetup.Connection legacyConfiguration(Configuration config) "Must either configure configuration file or base directory"); if (config.has(INDEX_CONF_FILE)) { String configFile = config.get(INDEX_CONF_FILE); - ImmutableSettings.Builder sb = ImmutableSettings.settingsBuilder(); + Settings.Builder sb = Settings.settingsBuilder(); log.debug("Configuring ES from YML file [{}]", configFile); FileInputStream fis = null; try { @@ -309,15 +312,17 @@ private ElasticSearchSetup.Connection legacyConfiguration(Configuration config) log.debug("Configuring ES with data directory [{}]", dataDirectory); File f = new File(dataDirectory); if (!f.exists()) f.mkdirs(); - ImmutableSettings.Builder b = ImmutableSettings.settingsBuilder(); + Settings.Builder b = Settings.settingsBuilder(); for (String sub : DATA_SUBDIRS) { String subdir = dataDirectory + File.separator + sub; f = new File(subdir); if (!f.exists()) f.mkdirs(); b.put("path." + sub, subdir); } - b.put("script.disable_dynamic", false); + b.put("script.inline", "on"); b.put("indices.ttl.interval", "5s"); + //TODO fix this + b.put("path.home", "/tmp"); builder.settings(b.build()); @@ -331,7 +336,7 @@ private ElasticSearchSetup.Connection legacyConfiguration(Configuration config) } else { log.debug("Configuring ES for network transport"); - ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder(); + Settings.Builder settings = Settings.settingsBuilder(); if (config.has(CLUSTER_NAME)) { String clustername = config.get(CLUSTER_NAME); Preconditions.checkArgument(StringUtils.isNotBlank(clustername), "Invalid cluster name: %s", clustername); @@ -341,8 +346,8 @@ private ElasticSearchSetup.Connection legacyConfiguration(Configuration config) } log.debug("Transport sniffing enabled: {}", config.get(CLIENT_SNIFF)); settings.put("client.transport.sniff", config.get(CLIENT_SNIFF)); - settings.put("script.disable_dynamic", false); - TransportClient tc = new TransportClient(settings.build()); + settings.put("script.inline", "on"); + TransportClient tc = TransportClient.builder().settings(settings.build()).build(); int defaultPort = config.has(INDEX_PORT)?config.get(INDEX_PORT):HOST_PORT_DEFAULT; for (String host : config.get(INDEX_HOSTS)) { String[] hostparts = host.split(":"); @@ -350,7 +355,12 @@ private ElasticSearchSetup.Connection legacyConfiguration(Configuration config) int hostport = defaultPort; if (hostparts.length == 2) hostport = Integer.parseInt(hostparts[1]); log.info("Configured remote host: {} : {}", hostname, hostport); - tc.addTransportAddress(new InetSocketTransportAddress(hostname, hostport)); + try { + tc.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(hostname), hostport)); + } catch (UnknownHostException e) { + log.error("unknown host", e); + throw new RuntimeException(e); + } } client = tc; node = null; @@ -453,7 +463,7 @@ public void register(String store, String key, KeyInformation information, BaseT try { PutMappingResponse response = client.admin().indices().preparePutMapping(indexName). - setIgnoreConflicts(false).setType(store).setSource(mapping).execute().actionGet(); + setType(store).setSource(mapping).execute().actionGet(); } catch (Exception e) { throw convert(e); } @@ -563,7 +573,7 @@ public void mutate(Map> mutations, KeyInforma brb.add(new DeleteRequest(indexName, storename, docid)); } else { String script = getDeletionScript(informations, storename, mutation); - brb.add(client.prepareUpdate(indexName, storename, docid).setScript(script, ScriptService.ScriptType.INLINE)); + brb.add(client.prepareUpdate(indexName, storename, docid).setScript(new Script(script, ScriptService.ScriptType.INLINE, null, null))); log.trace("Adding script {}", script); } @@ -582,7 +592,8 @@ public void mutate(Map> mutations, KeyInforma boolean needUpsert = !mutation.hasDeletions(); String script = getAdditionScript(informations, storename, mutation); - UpdateRequestBuilder update = client.prepareUpdate(indexName, storename, docid).setScript(script, ScriptService.ScriptType.INLINE); + UpdateRequestBuilder update = client.prepareUpdate(indexName, storename, docid).setScript( + new Script(script, ScriptService.ScriptType.INLINE, null, null)); if (needUpsert) { XContentBuilder doc = getNewDocument(mutation.getAdditions(), informations.get(storename), ttl); update.setUpsert(doc); @@ -692,6 +703,7 @@ private static String convertToJsType(Object value) throws PermanentBackendExcep } + @Override public void restore(Map>> documents, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException { BulkRequestBuilder bulk = client.prepareBulk(); int requests = 0; @@ -727,7 +739,7 @@ public void restore(Map>> documents, KeyInfo } } - public FilterBuilder getFilter(Condition condition, KeyInformation.StoreRetriever informations) { + public QueryBuilder getFilter(Condition condition, KeyInformation.StoreRetriever informations) { if (condition instanceof PredicateCondition) { PredicateCondition atom = (PredicateCondition) condition; Object value = atom.getValue(); @@ -740,17 +752,17 @@ public FilterBuilder getFilter(Condition condition, KeyInformation.StoreRetri switch (numRel) { case EQUAL: - return FilterBuilders.inFilter(key, value); + return QueryBuilders.matchQuery(key, value); case NOT_EQUAL: - return FilterBuilders.notFilter(FilterBuilders.inFilter(key, value)); + return QueryBuilders.notQuery(QueryBuilders.matchQuery(key, value)); case LESS_THAN: - return FilterBuilders.rangeFilter(key).lt(value); + return QueryBuilders.rangeQuery(key).lt(value); case LESS_THAN_EQUAL: - return FilterBuilders.rangeFilter(key).lte(value); + return QueryBuilders.rangeQuery(key).lte(value); case GREATER_THAN: - return FilterBuilders.rangeFilter(key).gt(value); + return QueryBuilders.rangeQuery(key).gt(value); case GREATER_THAN_EQUAL: - return FilterBuilders.rangeFilter(key).gte(value); + return QueryBuilders.rangeQuery(key).gte(value); default: throw new IllegalArgumentException("Unexpected relation: " + numRel); } @@ -766,25 +778,25 @@ public FilterBuilder getFilter(Condition condition, KeyInformation.StoreRetri if (titanPredicate == Text.CONTAINS) { value = ((String) value).toLowerCase(); - AndFilterBuilder b = FilterBuilders.andFilter(); + BoolQueryBuilder b = QueryBuilders.boolQuery(); for (String term : Text.tokenize((String)value)) { - b.add(FilterBuilders.termFilter(fieldName, term)); + b.must(QueryBuilders.termQuery(fieldName, term)); } return b; } else if (titanPredicate == Text.CONTAINS_PREFIX) { value = ((String) value).toLowerCase(); - return FilterBuilders.prefixFilter(fieldName, (String) value); + return QueryBuilders.prefixQuery(fieldName, (String) value); } else if (titanPredicate == Text.CONTAINS_REGEX) { value = ((String) value).toLowerCase(); - return FilterBuilders.regexpFilter(fieldName, (String) value); + return QueryBuilders.regexpQuery(fieldName, (String) value); } else if (titanPredicate == Text.PREFIX) { - return FilterBuilders.prefixFilter(fieldName, (String) value); + return QueryBuilders.prefixQuery(fieldName, (String) value); } else if (titanPredicate == Text.REGEX) { - return FilterBuilders.regexpFilter(fieldName, (String) value); + return QueryBuilders.regexpQuery(fieldName, (String) value); } else if (titanPredicate == Cmp.EQUAL) { - return FilterBuilders.termFilter(fieldName, (String) value); + return QueryBuilders.termQuery(fieldName, (String) value); } else if (titanPredicate == Cmp.NOT_EQUAL) { - return FilterBuilders.notFilter(FilterBuilders.termFilter(fieldName, (String) value)); + return QueryBuilders.notQuery(QueryBuilders.termQuery(fieldName, (String) value)); } else throw new IllegalArgumentException("Predicate is not supported for string value: " + titanPredicate); } else if (value instanceof Geoshape) { @@ -792,11 +804,11 @@ public FilterBuilder getFilter(Condition condition, KeyInformation.StoreRetri Geoshape shape = (Geoshape) value; if (shape.getType() == Geoshape.Type.CIRCLE) { Geoshape.Point center = shape.getPoint(); - return FilterBuilders.geoDistanceFilter(key).lat(center.getLatitude()).lon(center.getLongitude()).distance(shape.getRadius(), DistanceUnit.KILOMETERS); + return QueryBuilders.geoDistanceQuery(key).lat(center.getLatitude()).lon(center.getLongitude()).distance(shape.getRadius(), DistanceUnit.KILOMETERS); } else if (shape.getType() == Geoshape.Type.BOX) { Geoshape.Point southwest = shape.getPoint(0); Geoshape.Point northeast = shape.getPoint(1); - return FilterBuilders.geoBoundingBoxFilter(key).bottomRight(southwest.getLatitude(), northeast.getLongitude()).topLeft(northeast.getLatitude(), southwest.getLongitude()); + return QueryBuilders.geoBoundingBoxQuery(key).bottomRight(southwest.getLatitude(), northeast.getLongitude()).topLeft(northeast.getLatitude(), southwest.getLongitude()); } else throw new IllegalArgumentException("Unsupported or invalid search shape type: " + shape.getType()); } else if (value instanceof Date || value instanceof Instant) { @@ -805,17 +817,17 @@ public FilterBuilder getFilter(Condition condition, KeyInformation.StoreRetri switch (numRel) { case EQUAL: - return FilterBuilders.inFilter(key, value); + return QueryBuilders.matchQuery(key, value); case NOT_EQUAL: - return FilterBuilders.notFilter(FilterBuilders.inFilter(key, value)); + return QueryBuilders.notQuery(QueryBuilders.matchQuery(key, value)); case LESS_THAN: - return FilterBuilders.rangeFilter(key).lt(value); + return QueryBuilders.rangeQuery(key).lt(value); case LESS_THAN_EQUAL: - return FilterBuilders.rangeFilter(key).lte(value); + return QueryBuilders.rangeQuery(key).lte(value); case GREATER_THAN: - return FilterBuilders.rangeFilter(key).gt(value); + return QueryBuilders.rangeQuery(key).gt(value); case GREATER_THAN_EQUAL: - return FilterBuilders.rangeFilter(key).gte(value); + return QueryBuilders.rangeQuery(key).gte(value); default: throw new IllegalArgumentException("Unexpected relation: " + numRel); } @@ -823,34 +835,35 @@ public FilterBuilder getFilter(Condition condition, KeyInformation.StoreRetri Cmp numRel = (Cmp) titanPredicate; switch (numRel) { case EQUAL: - return FilterBuilders.inFilter(key, value); + return QueryBuilders.matchQuery(key, value); case NOT_EQUAL: - return FilterBuilders.notFilter(FilterBuilders.inFilter(key, value)); + return QueryBuilders.notQuery(QueryBuilders.matchQuery(key, value)); default: throw new IllegalArgumentException("Boolean types only support EQUAL or NOT_EQUAL"); } } else if (value instanceof UUID) { if (titanPredicate == Cmp.EQUAL) { - return FilterBuilders.termFilter(key, value); + return QueryBuilders.termQuery(key, value); } else if (titanPredicate == Cmp.NOT_EQUAL) { - return FilterBuilders.notFilter(FilterBuilders.termFilter(key, value)); + return QueryBuilders.notQuery(QueryBuilders.termQuery(key, value)); } else { throw new IllegalArgumentException("Only equal or not equal is supported for UUIDs: " + titanPredicate); } } else throw new IllegalArgumentException("Unsupported type: " + value); } else if (condition instanceof Not) { - return FilterBuilders.notFilter(getFilter(((Not) condition).getChild(),informations)); + return QueryBuilders.notQuery(getFilter(((Not) condition).getChild(),informations)); } else if (condition instanceof And) { - AndFilterBuilder b = FilterBuilders.andFilter(); + BoolQueryBuilder b = QueryBuilders.boolQuery(); for (Condition c : condition.getChildren()) { - b.add(getFilter(c,informations)); + b.must(getFilter(c,informations)); } return b; } else if (condition instanceof Or) { - OrFilterBuilder b = FilterBuilders.orFilter(); + BoolQueryBuilder b = QueryBuilders.boolQuery(); + b.minimumNumberShouldMatch(1); for (Condition c : condition.getChildren()) { - b.add(getFilter(c,informations)); + b.should(getFilter(c,informations)); } return b; } else throw new IllegalArgumentException("Invalid condition: " + condition); @@ -1024,7 +1037,7 @@ public void clearStorage() throws BackendException { .delete(new DeleteIndexRequest(indexName)).actionGet(); // We wait for one second to let ES delete the river Thread.sleep(1000); - } catch (IndexMissingException e) { + } catch (IndexNotFoundException e) { // Index does not exist... Fine } } catch (Exception e) { diff --git a/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchSetup.java b/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchSetup.java index 0bdf47bba5..941968528e 100644 --- a/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchSetup.java +++ b/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchSetup.java @@ -6,10 +6,11 @@ import com.thinkaurelius.titan.diskstorage.configuration.ConfigOption; import com.thinkaurelius.titan.diskstorage.configuration.Configuration; import com.thinkaurelius.titan.util.system.IOUtils; + import org.apache.commons.lang.StringUtils; import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeBuilder; @@ -18,6 +19,7 @@ import java.io.*; import java.lang.reflect.Array; +import java.net.InetAddress; import java.util.List; import java.util.Map; @@ -68,7 +70,7 @@ public enum ElasticSearchSetup { public Connection connect(Configuration config) throws IOException { log.debug("Configuring TransportClient"); - ImmutableSettings.Builder settingsBuilder = settingsBuilder(config); + Settings.Builder settingsBuilder = settingsBuilder(config); if (config.has(ElasticSearchIndex.CLIENT_SNIFF)) { String k = "client.transport.sniff"; @@ -76,7 +78,9 @@ public Connection connect(Configuration config) throws IOException { log.debug("Set {}: {}", k, config.get(ElasticSearchIndex.CLIENT_SNIFF)); } - TransportClient tc = new TransportClient(settingsBuilder.build()); + makeLocalDirsIfNecessary(settingsBuilder, config); + + TransportClient tc = TransportClient.builder().settings(settingsBuilder.build()).build(); int defaultPort = config.has(INDEX_PORT) ? config.get(INDEX_PORT) : ElasticSearchIndex.HOST_PORT_DEFAULT; for (String host : config.get(INDEX_HOSTS)) { String[] hostparts = host.split(":"); @@ -84,7 +88,7 @@ public Connection connect(Configuration config) throws IOException { int hostport = defaultPort; if (hostparts.length == 2) hostport = Integer.parseInt(hostparts[1]); log.info("Configured remote host: {} : {}", hostname, hostport); - tc.addTransportAddress(new InetSocketTransportAddress(hostname, hostport)); + tc.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(hostname), hostport)); } return new Connection(null, tc); } @@ -99,7 +103,7 @@ public Connection connect(Configuration config) throws IOException { log.debug("Configuring Node Client"); - ImmutableSettings.Builder settingsBuilder = settingsBuilder(config); + Settings.Builder settingsBuilder = settingsBuilder(config); if (config.has(ElasticSearchIndex.TTL_INTERVAL)) { String k = "indices.ttl.interval"; @@ -145,7 +149,7 @@ public Connection connect(Configuration config) throws IOException { *
  • If ignore-cluster-name is set, copy that value to client.transport.ignore_cluster_name in the settings builder
    • *
  • If client-sniff is set, copy that value to client.transport.sniff in the settings builder
    • *
  • If ttl-interval is set, copy that volue to indices.ttl.interval in the settings builder
    • - *
  • Unconditionally set script.disable_dynamic to false (i.e. enable dynamic scripting)
    • + *
  • Unconditionally set script.inline to on (i.e. enable inline scripting)
    • * * * This method then returns the builder. @@ -154,9 +158,9 @@ public Connection connect(Configuration config) throws IOException { * @return ES settings builder configured according to the {@code config} parameter * @throws java.io.IOException if conf-file was set but could not be read */ - private static ImmutableSettings.Builder settingsBuilder(Configuration config) throws IOException { + private static Settings.Builder settingsBuilder(Configuration config) throws IOException { - ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder(); + Settings.Builder settings = Settings.settingsBuilder(); // Set Titan defaults settings.put("client.transport.ignore_cluster_name", true); @@ -185,20 +189,22 @@ private static ImmutableSettings.Builder settingsBuilder(Configuration config) t } // Force-enable dynamic scripting. This is probably only useful in Node mode. - String disableScriptsKey = "script.disable_dynamic"; + String disableScriptsKey = "script.inline"; String disableScriptsVal = settings.get(disableScriptsKey); - if (null != disableScriptsVal && !"false".equals(disableScriptsVal)) { - log.warn("Titan requires Elasticsearch dynamic scripting. Setting {} to false. " + + if (null != disableScriptsVal && !"on".equals(disableScriptsVal)) { + log.warn("Titan requires Elasticsearch dynamic scripting. Setting {} to 'on'. " + "Dynamic scripting must be allowed in the Elasticsearch cluster configuration.", disableScriptsKey); } - settings.put(disableScriptsKey, false); - log.debug("Set {}: {}", disableScriptsKey, false); + settings.put(disableScriptsKey, "on"); + //TODO fix this + settings.put("path.home", "/tmp"); + log.debug("Set {}: {}", disableScriptsKey, "on"); return settings; } - static void applySettingsFromFile(ImmutableSettings.Builder settings, + static void applySettingsFromFile(Settings.Builder settings, Configuration config, ConfigOption confFileOption) throws FileNotFoundException { if (config.has(confFileOption)) { @@ -216,7 +222,7 @@ static void applySettingsFromFile(ImmutableSettings.Builder settings, } } - static void applySettingsFromTitanConf(ImmutableSettings.Builder settings, + static void applySettingsFromTitanConf(Settings.Builder settings, Configuration config, ConfigNamespace rootNS) { int keysLoaded = 0; @@ -248,7 +254,7 @@ static void applySettingsFromTitanConf(ImmutableSettings.Builder settings, } - private static void makeLocalDirsIfNecessary(ImmutableSettings.Builder settingsBuilder, Configuration config) { + private static void makeLocalDirsIfNecessary(Settings.Builder settingsBuilder, Configuration config) { if (config.has(INDEX_DIRECTORY)) { String dataDirectory = config.get(INDEX_DIRECTORY); File f = new File(dataDirectory); diff --git a/titan-es/src/test/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchConfigTest.java b/titan-es/src/test/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchConfigTest.java index 31feaafc7d..254ff27f73 100644 --- a/titan-es/src/test/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchConfigTest.java +++ b/titan-es/src/test/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchConfigTest.java @@ -1,7 +1,6 @@ package com.thinkaurelius.titan.diskstorage.es; import com.google.common.base.Joiner; - import com.thinkaurelius.titan.core.TitanFactory; import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.attribute.Text; @@ -13,15 +12,15 @@ import com.thinkaurelius.titan.diskstorage.configuration.backend.CommonsConfiguration; import com.thinkaurelius.titan.diskstorage.indexing.*; import com.thinkaurelius.titan.diskstorage.util.StandardBaseTransactionConfig; - import com.thinkaurelius.titan.diskstorage.util.time.TimestampProviders; import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration; import com.thinkaurelius.titan.graphdb.query.condition.PredicateCondition; import com.thinkaurelius.titan.util.system.IOUtils; + import org.apache.commons.configuration.BaseConfiguration; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; -import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeBuilder; import org.junit.Assert; @@ -35,7 +34,6 @@ import static com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration.INDEX_CONF_FILE; import static com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration.INDEX_DIRECTORY; import static com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration.INDEX_HOSTS; - import static org.junit.Assert.*; /** @@ -259,7 +257,7 @@ public void testIndexCreationOptions() throws InterruptedException, BackendExcep - ImmutableSettings.Builder settingsBuilder = ImmutableSettings.settingsBuilder(); + Settings.Builder settingsBuilder = Settings.settingsBuilder(); settingsBuilder.put("discovery.zen.ping.multicast.enabled", "false"); settingsBuilder.put("discovery.zen.ping.unicast.hosts", "localhost,127.0.0.1:9300"); settingsBuilder.put("cluster.name", "indexCreationOptions"); @@ -271,7 +269,7 @@ public void testIndexCreationOptions() throws InterruptedException, BackendExcep assertEquals(String.valueOf(shards), response.getSetting("titan", "index.number_of_shards")); idx.close(); - n.stop(); + n.close(); esr.stop(); } diff --git a/titan-hbase-parent/titan-hbase-core/src/test/java/com/thinkaurelius/titan/HBaseStatus.java b/titan-hbase-parent/titan-hbase-core/src/test/java/com/thinkaurelius/titan/HBaseStatus.java index 265fa68585..ed1426c828 100644 --- a/titan-hbase-parent/titan-hbase-core/src/test/java/com/thinkaurelius/titan/HBaseStatus.java +++ b/titan-hbase-parent/titan-hbase-core/src/test/java/com/thinkaurelius/titan/HBaseStatus.java @@ -6,10 +6,10 @@ import java.io.FileReader; import java.io.IOException; -import org.elasticsearch.common.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Preconditions; import com.thinkaurelius.titan.util.system.IOUtils; public class HBaseStatus { diff --git a/titan-lucene/src/main/java/com/thinkaurelius/titan/diskstorage/lucene/LuceneIndex.java b/titan-lucene/src/main/java/com/thinkaurelius/titan/diskstorage/lucene/LuceneIndex.java index 893d95af38..aae13f655c 100644 --- a/titan-lucene/src/main/java/com/thinkaurelius/titan/diskstorage/lucene/LuceneIndex.java +++ b/titan-lucene/src/main/java/com/thinkaurelius/titan/diskstorage/lucene/LuceneIndex.java @@ -19,6 +19,7 @@ import com.thinkaurelius.titan.graphdb.query.TitanPredicate; import com.thinkaurelius.titan.graphdb.query.condition.*; import com.thinkaurelius.titan.util.system.IOUtils; + import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -39,12 +40,12 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.Version; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.nio.file.Path; import java.time.Instant; import java.util.*; import java.util.concurrent.ConcurrentHashMap; @@ -63,7 +64,7 @@ public class LuceneIndex implements IndexProvider { private static final String GEOID = "_____geo"; private static final int MAX_STRING_FIELD_LEN = 256; - private static final Version LUCENE_VERSION = Version.LUCENE_4_10_4; + private static final Version LUCENE_VERSION = Version.LUCENE_5_2_1; private static final IndexFeatures LUCENE_FEATURES = new IndexFeatures.Builder().supportedStringMappings(Mapping.TEXT, Mapping.STRING).supportsCardinality(Cardinality.SINGLE).supportsNanoseconds().build(); private static final int GEO_MAX_LEVELS = 11; @@ -97,7 +98,7 @@ private Directory getStoreDirectory(String store) throws BackendException { if (!path.exists() || !path.isDirectory() || !path.canWrite()) throw new PermanentBackendException("Cannot access or write to directory: " + dir); log.debug("Opening store directory [{}]", path); - return FSDirectory.open(path); + return FSDirectory.open(path.toPath()); } catch (IOException e) { throw new PermanentBackendException("Could not open directory: " + dir, e); } @@ -107,7 +108,7 @@ private IndexWriter getWriter(String store) throws BackendException { Preconditions.checkArgument(writerLock.isHeldByCurrentThread()); IndexWriter writer = writers.get(store); if (writer == null) { - IndexWriterConfig iwc = new IndexWriterConfig(LUCENE_VERSION, analyzer); + IndexWriterConfig iwc = new IndexWriterConfig(analyzer); iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); try { writer = new IndexWriter(getStoreDirectory(store), iwc); diff --git a/titan-lucene/src/test/java/com/thinkaurelius/titan/diskstorage/lucene/LuceneExample.java b/titan-lucene/src/test/java/com/thinkaurelius/titan/diskstorage/lucene/LuceneExample.java index f4cb84a02d..feae619977 100644 --- a/titan-lucene/src/test/java/com/thinkaurelius/titan/diskstorage/lucene/LuceneExample.java +++ b/titan-lucene/src/test/java/com/thinkaurelius/titan/diskstorage/lucene/LuceneExample.java @@ -64,9 +64,9 @@ private SpatialStrategy getSpatialStrategy(String key) { @Test public void example1() throws Exception { - Directory dir = FSDirectory.open(path); + Directory dir = FSDirectory.open(path.toPath()); Analyzer analyzer = new StandardAnalyzer(); - IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_4_10_4, analyzer); + IndexWriterConfig iwc = new IndexWriterConfig(analyzer); iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); IndexWriter writer = new IndexWriter(dir, iwc); @@ -91,7 +91,7 @@ public void example1() throws Exception { writer.close(); //Search - IndexReader reader = DirectoryReader.open(FSDirectory.open(path)); + IndexReader reader = DirectoryReader.open(FSDirectory.open(path.toPath())); IndexSearcher searcher = new IndexSearcher(reader); analyzer = new StandardAnalyzer(); From 576cd88e997ddeb072628ac4ff675b82afb05700 Mon Sep 17 00:00:00 2001 From: Jim Cahill Date: Mon, 14 Sep 2015 11:39:04 -0400 Subject: [PATCH 3/4] use java.io.tmpdir for path.home in es client (where it shouldn't really even be needed --- .../thinkaurelius/titan/diskstorage/es/ElasticSearchIndex.java | 3 +-- .../thinkaurelius/titan/diskstorage/es/ElasticSearchSetup.java | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchIndex.java b/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchIndex.java index fe1c7b1d50..d0db646046 100644 --- a/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchIndex.java +++ b/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchIndex.java @@ -321,8 +321,7 @@ private ElasticSearchSetup.Connection legacyConfiguration(Configuration config) } b.put("script.inline", "on"); b.put("indices.ttl.interval", "5s"); - //TODO fix this - b.put("path.home", "/tmp"); + b.put("path.home", System.getProperty("java.io.tmpdir")); builder.settings(b.build()); diff --git a/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchSetup.java b/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchSetup.java index 941968528e..d4565ca0f1 100644 --- a/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchSetup.java +++ b/titan-es/src/main/java/com/thinkaurelius/titan/diskstorage/es/ElasticSearchSetup.java @@ -197,8 +197,7 @@ private static Settings.Builder settingsBuilder(Configuration config) throws IOE disableScriptsKey); } settings.put(disableScriptsKey, "on"); - //TODO fix this - settings.put("path.home", "/tmp"); + settings.put("path.home", System.getProperty("java.io.tmpdir")); log.debug("Set {}: {}", disableScriptsKey, "on"); return settings; From c567d65596de8ef6393534fe85c23b6702aa4cf1 Mon Sep 17 00:00:00 2001 From: Jim Cahill Date: Tue, 15 Sep 2015 11:44:23 -0400 Subject: [PATCH 4/4] Revert "plugin exclusions in eclipse" This reverts commit fc11a96f80959664e2d9493c994424f5f12d5ddd. --- titan-core/pom.xml | 40 -------------- .../titan/diskstorage/Backend.java | 1 - titan-es/pom.xml | 54 ------------------- titan-hadoop-parent/pom.xml | 31 ----------- titan-hadoop-parent/titan-hadoop-core/pom.xml | 36 ------------- titan-hbase-parent/pom.xml | 31 ----------- titan-hbase-parent/titan-hbase-094/pom.xml | 54 ------------------- titan-hbase-parent/titan-hbase-096/pom.xml | 54 ------------------- titan-hbase-parent/titan-hbase-098/pom.xml | 54 ------------------- titan-hbase-parent/titan-hbase-10/pom.xml | 54 ------------------- titan-test/pom.xml | 44 --------------- 11 files changed, 453 deletions(-) diff --git a/titan-core/pom.xml b/titan-core/pom.xml index 17bad1c997..b34182de0a 100644 --- a/titan-core/pom.xml +++ b/titan-core/pom.xml @@ -219,45 +219,5 @@ - - - - - org.eclipse.m2e - lifecycle-mapping - 1.0.0 - - - - - - - org.codehaus.gmaven - - - gmaven-plugin - - - [1.5,) - - - testCompile - compile - generateStubs - - generateTestStubs - - - - - - - - - - - - - diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/diskstorage/Backend.java b/titan-core/src/main/java/com/thinkaurelius/titan/diskstorage/Backend.java index a89ca4f8d0..0d33670531 100644 --- a/titan-core/src/main/java/com/thinkaurelius/titan/diskstorage/Backend.java +++ b/titan-core/src/main/java/com/thinkaurelius/titan/diskstorage/Backend.java @@ -538,7 +538,6 @@ public BackendTransaction beginTransaction(TransactionConfiguration configuratio maxReadTime, indexTx, threadPool); } - @Override public synchronized void close() throws BackendException { if (!hasAttemptedClose) { hasAttemptedClose = true; diff --git a/titan-es/pom.xml b/titan-es/pom.xml index 78e1c8a4ac..d60183b187 100644 --- a/titan-es/pom.xml +++ b/titan-es/pom.xml @@ -246,60 +246,6 @@ - - - - - org.eclipse.m2e - lifecycle-mapping - 1.0.0 - - - - - - - org.apache.maven.plugins - - - maven-dependency-plugin - - - [2.8,) - - - build-classpath - - - - - - - - - - org.apache.maven.plugins - - - maven-assembly-plugin - - - [2.4,) - - - single - - - - - - - - - - - - diff --git a/titan-hadoop-parent/pom.xml b/titan-hadoop-parent/pom.xml index 4ff2d86c72..f0d6ab54f7 100644 --- a/titan-hadoop-parent/pom.xml +++ b/titan-hadoop-parent/pom.xml @@ -353,37 +353,6 @@ --> - - - org.eclipse.m2e - lifecycle-mapping - 1.0.0 - - - - - - - org.apache.maven.plugins - - - maven-dependency-plugin - - - [2.8,) - - - unpack - - - - - - - - - - diff --git a/titan-hadoop-parent/titan-hadoop-core/pom.xml b/titan-hadoop-parent/titan-hadoop-core/pom.xml index 807d6dc436..c069dcb315 100644 --- a/titan-hadoop-parent/titan-hadoop-core/pom.xml +++ b/titan-hadoop-parent/titan-hadoop-core/pom.xml @@ -151,42 +151,6 @@ - - - - - org.eclipse.m2e - lifecycle-mapping - 1.0.0 - - - - - - - org.codehaus.gmaven - - - gmaven-plugin - - - [1.5,) - - - compile - generateStubs - - - - - - - - - - - - diff --git a/titan-hbase-parent/pom.xml b/titan-hbase-parent/pom.xml index 47710691d7..383c44f14e 100644 --- a/titan-hbase-parent/pom.xml +++ b/titan-hbase-parent/pom.xml @@ -193,37 +193,6 @@ - - - org.eclipse.m2e - lifecycle-mapping - 1.0.0 - - - - - - - org.apache.maven.plugins - - - maven-dependency-plugin - - - [2.8,) - - - unpack - - - - - - - - - - diff --git a/titan-hbase-parent/titan-hbase-094/pom.xml b/titan-hbase-parent/titan-hbase-094/pom.xml index e2d9a25a97..f260d935f2 100644 --- a/titan-hbase-parent/titan-hbase-094/pom.xml +++ b/titan-hbase-parent/titan-hbase-094/pom.xml @@ -223,60 +223,6 @@ - - - - - org.eclipse.m2e - lifecycle-mapping - 1.0.0 - - - - - - - org.apache.maven.plugins - - - maven-dependency-plugin - - - [2.8,) - - - build-classpath - - - - - - - - - - org.apache.maven.plugins - - - maven-assembly-plugin - - - [2.4,) - - - single - - - - - - - - - - - - diff --git a/titan-hbase-parent/titan-hbase-096/pom.xml b/titan-hbase-parent/titan-hbase-096/pom.xml index c9583b1f6c..e76076f80a 100644 --- a/titan-hbase-parent/titan-hbase-096/pom.xml +++ b/titan-hbase-parent/titan-hbase-096/pom.xml @@ -130,60 +130,6 @@ - - - - - org.eclipse.m2e - lifecycle-mapping - 1.0.0 - - - - - - - org.apache.maven.plugins - - - maven-dependency-plugin - - - [2.8,) - - - build-classpath - - - - - - - - - - org.apache.maven.plugins - - - maven-assembly-plugin - - - [2.4,) - - - single - - - - - - - - - - - - diff --git a/titan-hbase-parent/titan-hbase-098/pom.xml b/titan-hbase-parent/titan-hbase-098/pom.xml index c00a3360e0..5423b90647 100644 --- a/titan-hbase-parent/titan-hbase-098/pom.xml +++ b/titan-hbase-parent/titan-hbase-098/pom.xml @@ -131,60 +131,6 @@ - - - - - org.eclipse.m2e - lifecycle-mapping - 1.0.0 - - - - - - - org.apache.maven.plugins - - - maven-dependency-plugin - - - [2.8,) - - - build-classpath - - - - - - - - - - org.apache.maven.plugins - - - maven-assembly-plugin - - - [2.4,) - - - single - - - - - - - - - - - - diff --git a/titan-hbase-parent/titan-hbase-10/pom.xml b/titan-hbase-parent/titan-hbase-10/pom.xml index dda756883e..cc9643e8be 100644 --- a/titan-hbase-parent/titan-hbase-10/pom.xml +++ b/titan-hbase-parent/titan-hbase-10/pom.xml @@ -130,60 +130,6 @@ - - - - - org.eclipse.m2e - lifecycle-mapping - 1.0.0 - - - - - - - org.apache.maven.plugins - - - maven-dependency-plugin - - - [2.8,) - - - build-classpath - - - - - - - - - - org.apache.maven.plugins - - - maven-assembly-plugin - - - [2.4,) - - - single - - - - - - - - - - - - diff --git a/titan-test/pom.xml b/titan-test/pom.xml index f0b9a486b8..fb3284ef7f 100644 --- a/titan-test/pom.xml +++ b/titan-test/pom.xml @@ -120,49 +120,5 @@ - - - - - org.eclipse.m2e - lifecycle-mapping - 1.0.0 - - - - - - - org.codehaus.gmavenplus - - - gmavenplus-plugin - - - [1.2,) - - - addSources - generateStubs - - testGenerateStubs - - addTestSources - compile - removeStubs - removeTestStubs - testCompile - - - - - - - - - - - -